llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed attribute parsing

Browse Source
This commit is contained in:
Geens
2025-04-12 18:02:55 +02:00
parent bd3adf78e6
commit 9564879edc
23 changed files with 2905 additions and 2786 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
lib/xml_schema_validator/example_schema.xsd Normal file
View File

@@ -0,0 +1,118 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Always answer with a single element.
-->
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
<!--
Delete command removes an entry from the context by its ID.
Use it to remove unnecessary items and stop background processes.
When you delete something, it is gone.
Make sure all important info is stored in files.
Example:
<delete id="1234567890"/>
-->
<xs:element name="delete">
<xs:complexType>
<xs:attribute name="id" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>
<!--
Stop command terminates the agent gracefully.
For the main SIA instance this will trigger an update and restart.
For sub-instances this is the correct way to stop after all tasks are complete.
Example:
<stop id="1234567890"/>
-->
<xs:element name="stop">
<xs:complexType/>
</xs:element>
<!--
Single script that runs once and completes.
Output is stored in context until explicitly deleted.
Used for one-time operations like file manipulation.
Single scripts are limited to 1024 characters and 1 second timeout by default.
These limits can be changed with attributes.
Example:
<single>
ls /
</single>
-->
<xs:element name="single">
<xs:complexType mixed="true">
<xs:sequence>
<xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
</xs:sequence>
<xs:attribute name="timeout" type="xs:float" use="optional"/>
<xs:attribute name="limit" type="xs:integer" use="optional"/>
</xs:complexType>
</xs:element>
<!--
Repeat script runs each time the context is generated.
After a command is issued, all repeat scripts in context are run again.
Useful for monitoring changing files or viewing results immediately after changing a file.
Repeat scripts should execute quickly to avoid blocking the agent.
Repeat scripts are limited to 1024 characters and 1 second timeout by default.
These limits can be changed with attributes.
Example:
<repeat>
ls /
</repeat>
-->
<xs:element name="repeat">
<xs:complexType mixed="true">
<xs:sequence>
<xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
</xs:sequence>
<xs:attribute name="timeout" type="xs:float" use="optional"/>
<xs:attribute name="limit" type="xs:integer" use="optional"/>
</xs:complexType>
</xs:element>
<!--
As an agent it is important to reason about your actions and their results.
In a reasoning action you can write freeform text.
This is also stored in context until deleted.
Example:
<reasoning>
I should explore the file system for interesting files.
</reasoning>
-->
<xs:element name="reasoning">
<xs:complexType mixed="true">
<xs:sequence>
<xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
</xs:sequence>
</xs:complexType>
</xs:element>
<!--
Read all available text on stdin and store it in context.
Do this only if the context indicates there is data in the stdin buffer.
Example:
<read_stdin/>
-->
<xs:element name="read_stdin">
<xs:complexType/>
</xs:element>
<!--
Write to stdout.
This is your main way of contacting the user.
Make sure you have properly reasoned about what to say and if it is necessary before issuing a write_stdout command.
Example:
<write_stdout>
Hello world!
</write_stdout>
-->
<xs:element name="write_stdout">
<xs:complexType mixed="true">
<xs:sequence>
<xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:schema>

2
lib/xml_schema_validator/python/xml_schema_validator/__init__.py
View File

@@ -12,7 +12,7 @@ class XmlLogitsProcessor(LogitsProcessor):
by setting their logits to negative infinity.
"""
def __init__(self, tokenizer: AutoTokenizer, schema_text: str = None, core=None):
def __init__(self, tokenizer: AutoTokenizer, schema_text: str = None, core: XmlLogitsProcessorCore=None):
"""
Initialize the processor with a schema and tokenizer.

219
lib/xml_schema_validator/src/char_trie.rs
View File

@@ -1,103 +1,116 @@
use crate::*;
use pyo3::prelude::*;
// Node in our character trie
struct CharTrieNode {
// Store character and node pairs, keep sorted for binary search
children: Vec<(char, CharTrieNode)>,
// Store token IDs that are valid at this point
token_id: Option<Py<PyAny>>,
}
impl CharTrieNode {
fn new() -> Self {
Self {
children: Vec::new(),
token_id: None,
}
}
// Add a child for character c, maintaining sorted order
fn add_child(&mut self, c: char) -> &mut CharTrieNode {
match self.children.binary_search_by_key(&c, |(ch, _)| *ch) {
Ok(index) => &mut self.children[index].1,
Err(index) => {
// Insert at the right position to maintain sort order
self.children.insert(index, (c, CharTrieNode::new()));
&mut self.children[index].1
}
}
}
// Add a token id to this node
fn add_token(&mut self, token_id: Py<PyAny>) {
self.token_id = Some(token_id);
}
pub fn clone_ref(&self, py: Python<'_>) -> Self {
Self {
children: self.children.iter().map(|(ch, node)| (*ch, node.clone_ref(py))).collect(),
token_id: self.token_id.iter().map(|token_id| token_id.clone_ref(py)).nth(0),
}
}
}
// The main trie structure
pub struct CharTrie {
root: CharTrieNode,
}
impl CharTrie {
pub fn new() -> Self {
Self {
root: CharTrieNode::new(),
}
}
// Insert a token's character sequence into the trie
pub fn insert(&mut self, token_text: &str, token_id: Py<PyAny>) {
let mut current = &mut self.root;
// Add each character in the token text
for c in token_text.chars() {
current = current.add_child(c);
}
// Mark this node as representing token_id
current.add_token(token_id);
}
// Find all tokens that could be invalid given the current XML state
pub fn find_invalid_tokens(&self, py: Python<'_>, xml_validator: &XmlSchemaValidator, token_map: &Vec<(Py<PyAny>, String)>) -> Vec<Py<PyAny>> {
let mut invalid_tokens = Vec::new();
for (token_id, token_text) in token_map {
// Check if this token would make the XML invalid
if !self.is_valid_continuation(xml_validator, token_text) {
invalid_tokens.push(token_id.clone_ref(py));
}
}
invalid_tokens
}
// Check if a token would be a valid continuation
fn is_valid_continuation(&self, xml_validator: &XmlSchemaValidator, token_text: &str) -> bool {
// First, quick check with just the first character
if let Some(first_char) = token_text.chars().next() {
if !xml_validator.can_continue_with(first_char) {
return false;
}
}
// If first character is valid, try the full token
let mut validator_clone = xml_validator.clone();
validator_clone.append(token_text).is_ok()
}
pub fn clone_ref(&self, py: Python<'_>) -> Self {
Self {
root: self.root.clone_ref(py),
}
}
}
use crate::*;
use pyo3::prelude::*;
// Node in our character trie
struct CharTrieNode {
// Store character and node pairs, keep sorted for binary search
children: Vec<(char, CharTrieNode)>,
// Store token IDs that are valid at this point
token_id: Option<Py<PyAny>>,
}
impl CharTrieNode {
fn new() -> Self {
Self {
children: Vec::new(),
token_id: None,
}
}
// Add a child for character c, maintaining sorted order
fn add_child(&mut self, c: char) -> &mut CharTrieNode {
match self.children.binary_search_by_key(&c, |(ch, _)| *ch) {
Ok(index) => &mut self.children[index].1,
Err(index) => {
// Insert at the right position to maintain sort order
self.children.insert(index, (c, CharTrieNode::new()));
&mut self.children[index].1
}
}
}
// Add a token id to this node
fn add_token(&mut self, token_id: Py<PyAny>) {
self.token_id = Some(token_id);
}
pub fn clone_ref(&self, py: Python<'_>) -> Self {
Self {
children: self
.children
.iter()
.map(|(ch, node)| (*ch, node.clone_ref(py)))
.collect(),
token_id: self
.token_id
.iter()
.map(|token_id| token_id.clone_ref(py))
.nth(0),
}
}
}
// The main trie structure
pub struct CharTrie {
root: CharTrieNode,
}
impl CharTrie {
pub fn new() -> Self {
Self {
root: CharTrieNode::new(),
}
}
// Insert a token's character sequence into the trie
pub fn insert(&mut self, token_text: &str, token_id: Py<PyAny>) {
let mut current = &mut self.root;
// Add each character in the token text
for c in token_text.chars() {
current = current.add_child(c);
}
// Mark this node as representing token_id
current.add_token(token_id);
}
// Find all tokens that could be invalid given the current XML state
pub fn find_invalid_tokens(
&self,
py: Python<'_>,
xml_validator: &XmlSchemaValidator,
token_map: &Vec<(Py<PyAny>, String)>,
) -> Vec<Py<PyAny>> {
let mut invalid_tokens = Vec::new();
for (token_id, token_text) in token_map {
// Check if this token would make the XML invalid
if !self.is_valid_continuation(xml_validator, token_text) {
invalid_tokens.push(token_id.clone_ref(py));
}
}
invalid_tokens
}
// Check if a token would be a valid continuation
fn is_valid_continuation(&self, xml_validator: &XmlSchemaValidator, token_text: &str) -> bool {
// First, quick check with just the first character
if let Some(first_char) = token_text.chars().next() {
if !xml_validator.can_continue_with(first_char) {
return false;
}
}
// If first character is valid, try the full token
let mut validator_clone = xml_validator.clone();
validator_clone.append(token_text).is_ok()
}
pub fn clone_ref(&self, py: Python<'_>) -> Self {
Self {
root: self.root.clone_ref(py),
}
}
}

28
lib/xml_schema_validator/src/error.rs
View File

@@ -1,14 +1,14 @@
/// Error types that can occur during XML validation
#[derive(Clone, Debug, thiserror::Error)]
pub enum Error {
/// Indicates that the provided XSD schema is invalid
#[error("Invalid schema: {0}")]
InvalidSchema(#[from] quick_xml::DeError),
/// Indicates that the provided XML fragment is invalid
#[error("Invalid XML: {0}")]
InvalidXml(String),
/// Not implemented yet
#[error("Not implemented yet")]
NotImplemented,
}
/// Error types that can occur during XML validation
#[derive(Clone, Debug, thiserror::Error)]
pub enum Error {
/// Indicates that the provided XSD schema is invalid
#[error("Invalid schema: {0}")]
InvalidSchema(#[from] quick_xml::DeError),
/// Indicates that the provided XML fragment is invalid
#[error("Invalid XML: {0}")]
InvalidXml(String),
/// Not implemented yet
#[error("Not implemented yet")]
NotImplemented,
}

1358
lib/xml_schema_validator/src/lib.rs
View File

File diff suppressed because it is too large Load Diff

216
lib/xml_schema_validator/src/py_xml_logits_processor_core.rs
View File

@@ -1,6 +1,6 @@
use crate::*;
use pyo3::prelude::*;
use pyo3::types::PyDict;
use crate::*;
/// A minimal LogitsProcessor that enforces valid XML according to a schema.
#[pyclass]
@@ -18,7 +18,10 @@ impl XmlLogitsProcessorCore {
Ok(xml_schema_validator) => xml_schema_validator,
Err(e) => return Err(pyo3::exceptions::PyValueError::new_err(e.to_string())),
};
let tokens: Vec<(Py<PyAny>, String)> = tokens.iter().map(|(id, value)| (id.unbind(), value.unbind().to_string())).collect();
let tokens: Vec<(Py<PyAny>, String)> = tokens
.iter()
.map(|(id, value)| (id.unbind(), value.unbind().to_string()))
.collect();
let mut trie = char_trie::CharTrie::new();
for (token_id, token_text) in tokens.iter() {
trie.insert(token_text, token_id.clone_ref(py));
@@ -29,7 +32,7 @@ impl XmlLogitsProcessorCore {
trie,
})
}
/// Append a fragment of XML to the validator
/// Returns true if the append was successful, false otherwise
fn append(&mut self, fragment: &str) -> bool {
@@ -38,7 +41,9 @@ impl XmlLogitsProcessorCore {
/// Get a list of tokens that would lead to invalid XML
fn get_invalid_tokens(&mut self, py: Python<'_>) -> PyResult<Vec<PyObject>> {
Ok(self.trie.find_invalid_tokens(py, &self.xml_schema_validator, &self.tokens))
Ok(self
.trie
.find_invalid_tokens(py, &self.xml_schema_validator, &self.tokens))
}
/// Check if the validator has reached the end of the XML
@@ -50,16 +55,20 @@ impl XmlLogitsProcessorCore {
fn copy(&self, py: Python<'_>) -> PyResult<Self> {
Ok(Self {
xml_schema_validator: self.xml_schema_validator.clone(),
tokens: self.tokens.iter().map(|(a, b)|(a.clone_ref(py), b.clone())).collect(),
tokens: self
.tokens
.iter()
.map(|(a, b)| (a.clone_ref(py), b.clone()))
.collect(),
trie: self.trie.clone_ref(py),
})
}
/// Get a list of all element names in the schema
fn get_element_names(&self) -> PyResult<Vec<String>> {
Ok(self.xml_schema_validator.get_element_names())
}
/// Check if a specific character is valid as the next character
fn can_continue_with(&self, c: &str) -> PyResult<bool> {
if let Some(first_char) = c.chars().next() {
@@ -79,8 +88,8 @@ pub fn register_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()>
#[cfg(test)]
mod tests {
use super::*;
use pyo3::Python;
use pyo3::types::PyDict;
use pyo3::Python;
fn create_test_processor<'py>(py: Python<'py>) -> PyResult<XmlLogitsProcessorCore> {
let tokens = PyDict::new(py);
@@ -91,7 +100,7 @@ mod tests {
tokens.set_item(3, " ")?;
tokens.set_item(4, "a")?;
tokens.set_item(5, "</reasoning>")?;
// Simple schema with only reasoning element
let schema_text = r#"<?xml version="1.0" encoding="UTF-8"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
@@ -103,7 +112,7 @@ mod tests {
</xs:complexType>
</xs:element>
</xs:schema>"#;
XmlLogitsProcessorCore::new(py, tokens.into(), schema_text)
}
@@ -111,24 +120,29 @@ mod tests {
fn test_append_valid_xml() {
Python::with_gil(|py| {
let mut processor = create_test_processor(py).unwrap();
// Test various valid XML fragments
assert!(processor.append("<reasoning>"), "Should accept opening tag");
let mut processor = create_test_processor(py).unwrap();
assert!(processor.append("<reasoning>test</reasoning>"),
"Should accept complete element with content");
assert!(
processor.append("<reasoning>test</reasoning>"),
"Should accept complete element with content"
);
// Test with real schema from disk if available
if let Ok(schema_text) = std::fs::read_to_string("../../action_schema.xsd") {
let tokens = PyDict::new(py);
tokens.set_item(0, "<").unwrap();
tokens.set_item(1, "reasoning").unwrap();
tokens.set_item(2, ">").unwrap();
let mut processor = XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap();
assert!(processor.append("<reasoning>"),
"Should accept <reasoning> with actual schema");
let mut processor =
XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap();
assert!(
processor.append("<reasoning>"),
"Should accept <reasoning> with actual schema"
);
}
});
}
@@ -137,13 +151,18 @@ mod tests {
fn test_append_invalid_xml() {
Python::with_gil(|py| {
let mut processor = create_test_processor(py).unwrap();
// Test invalid XML fragments
assert!(!processor.append("<invalid>"), "Should reject invalid element");
assert!(
!processor.append("<invalid>"),
"Should reject invalid element"
);
let mut processor = create_test_processor(py).unwrap();
assert!(!processor.append("<reasoning></invalid>"),
"Should reject mismatched tags");
assert!(
!processor.append("<reasoning></invalid>"),
"Should reject mismatched tags"
);
});
}
@@ -151,25 +170,36 @@ mod tests {
fn test_eof_detection() {
Python::with_gil(|py| {
let processor = create_test_processor(py).unwrap();
// At start, not at EOF
assert!(!processor.eof().unwrap(), "Should not be at EOF at start");
// After complete document, should be at EOF
let mut processor = create_test_processor(py).unwrap();
// Use a properly formed XML document that matches the schema
assert!(processor.append("<reasoning>test</reasoning>"),
"Should accept complete document");
assert!(
processor.append("<reasoning>test</reasoning>"),
"Should accept complete document"
);
// DEBUG: Print the token state to see why EOF detection fails
println!("Current tokens: {:?}", processor.xml_schema_validator.current_tokens);
assert!(processor.eof().unwrap(), "Should be at EOF after complete document");
println!(
"Current tokens: {:?}",
processor.xml_schema_validator.current_tokens
);
assert!(
processor.eof().unwrap(),
"Should be at EOF after complete document"
);
// After opening tag, should not be at EOF
let mut processor = create_test_processor(py).unwrap();
assert!(processor.append("<reasoning>"), "Should accept opening tag");
assert!(!processor.eof().unwrap(), "Should not be at EOF after opening tag");
assert!(
!processor.eof().unwrap(),
"Should not be at EOF after opening tag"
);
});
}
@@ -177,20 +207,30 @@ mod tests {
fn test_can_continue_with() {
Python::with_gil(|py| {
let processor = create_test_processor(py).unwrap();
// At start, only '<' is valid
assert!(processor.can_continue_with("<").unwrap(), "Should accept '<' at start");
assert!(!processor.can_continue_with("a").unwrap(), "Should reject 'a' at start");
assert!(
processor.can_continue_with("<").unwrap(),
"Should accept '<' at start"
);
assert!(
!processor.can_continue_with("a").unwrap(),
"Should reject 'a' at start"
);
// Test after opening tag
let mut processor = create_test_processor(py).unwrap();
assert!(processor.append("<reasoning>"), "Should accept opening tag");
// In mixed content, any character should be valid
assert!(processor.can_continue_with("a").unwrap(),
"Should accept 'a' in mixed content");
assert!(processor.can_continue_with("<").unwrap(),
"Should accept '<' in mixed content");
assert!(
processor.can_continue_with("a").unwrap(),
"Should accept 'a' in mixed content"
);
assert!(
processor.can_continue_with("<").unwrap(),
"Should accept '<' in mixed content"
);
});
}
@@ -198,14 +238,17 @@ mod tests {
fn test_copy() {
Python::with_gil(|py| {
let processor = create_test_processor(py).unwrap();
// Make a copy
let copy = processor.copy(py).unwrap();
// Original and copy should behave the same
assert_eq!(processor.eof().unwrap(), copy.eof().unwrap(),
"Copy should have same EOF state");
assert_eq!(
processor.eof().unwrap(),
copy.eof().unwrap(),
"Copy should have same EOF state"
);
// Modify copy, original should be unchanged
let mut copy = processor.copy(py).unwrap();
assert!(copy.append("<reasoning>"), "Copy should accept XML");
@@ -218,28 +261,32 @@ mod tests {
fn test_get_element_names() {
Python::with_gil(|py| {
let processor = create_test_processor(py).unwrap();
let names = processor.get_element_names().unwrap();
assert!(names.contains(&"reasoning".to_string()),
"Should contain 'reasoning' element");
assert!(
names.contains(&"reasoning".to_string()),
"Should contain 'reasoning' element"
);
assert_eq!(names.len(), 1, "Should have exactly one element");
let tokens = PyDict::new(py);
tokens.set_item(0, "<").unwrap();
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let processor = XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap();
let names = processor.get_element_names().unwrap();
// Print out all elements from schema for debugging
println!("Elements in schema: {:?}", names);
// Check if reasoning exists
assert!(names.contains(&"reasoning".to_string()),
"Real schema should contain 'reasoning'");
assert!(
names.contains(&"reasoning".to_string()),
"Real schema should contain 'reasoning'"
);
});
}
#[test]
fn test_debug_reasoning_tag_validation() {
Python::with_gil(|py| {
@@ -254,7 +301,7 @@ mod tests {
</xs:complexType>
</xs:element>
</xs:schema>"#;
// Create a minimal token dictionary
let tokens = PyDict::new(py);
tokens.set_item(0, "<").unwrap();
@@ -268,56 +315,67 @@ mod tests {
tokens.set_item(8, "n").unwrap();
tokens.set_item(9, "g").unwrap();
tokens.set_item(10, ">").unwrap();
// Create processor with this schema and tokens
let processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
let processor =
XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
// Test basic operations to verify core functionality works
let element_names = processor.get_element_names().unwrap();
println!("Element names in test schema: {:?}", element_names);
assert!(element_names.contains(&"reasoning".to_string()),
"Test schema should contain 'reasoning'");
assert!(
element_names.contains(&"reasoning".to_string()),
"Test schema should contain 'reasoning'"
);
// Now try to append each character of "<reasoning>" one by one
let chars = vec!['<', 'r', 'e', 'a', 's', 'o', 'n', 'i', 'n', 'g', '>'];
let mut full_text = String::new();
let processor = processor.copy(py).unwrap();
for c in chars {
full_text.push(c);
let fragment = c.to_string();
let result = processor.can_continue_with(&fragment).unwrap();
println!("Can continue with '{}' after '{}': {}",
c, &full_text[..full_text.len()-1], result);
println!(
"Can continue with '{}' after '{}': {}",
c,
&full_text[..full_text.len() - 1],
result
);
// Create a new processor to test appending the fragment built so far
let mut fresh_processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
let mut fresh_processor =
XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
let append_result = fresh_processor.append(&full_text);
println!("Appending full text '{}': {}", full_text, append_result);
}
// Now test with actual schema from disk if available
if let Ok(actual_schema) = std::fs::read_to_string("../../action_schema.xsd") {
println!("\nTesting with actual schema from disk:");
let processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap();
let processor =
XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap();
let element_names = processor.get_element_names().unwrap();
println!("Element names in actual schema: {:?}", element_names);
if element_names.contains(&"reasoning".to_string()) {
println!("'reasoning' element found in actual schema");
// Test appending "<reasoning>" to a fresh processor
let mut processor = processor.copy(py).unwrap();
let result = processor.append("<reasoning>");
println!("Appending '<reasoning>' to fresh processor: {}", result);
// Test character by character
let mut text = String::new();
for c in "<reasoning>".chars() {
text.push(c);
let result = XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap()
.append(&text);
let result =
XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema)
.unwrap()
.append(&text);
println!("Appending '{}' to fresh processor: {}", text, result);
}
} else {
@@ -326,4 +384,4 @@ mod tests {
}
});
}
}
}

142
lib/xml_schema_validator/src/py_xml_schema_validator.rs
View File

@@ -1,71 +1,71 @@
use pyo3::prelude::*;
use pyo3::exceptions::PyValueError;
/// Python wrapper for XmlValidator
#[pyclass]
struct XmlSchemaValidator {
validator: crate::XmlSchemaValidator,
}
#[pymethods]
impl XmlSchemaValidator {
#[new]
fn new(schema_text: &str) -> PyResult<Self> {
let validator = crate::XmlSchemaValidator::new(schema_text)
.map_err(|e| PyValueError::new_err(format!("Failed to create validator: {}", e)))?;
Ok(Self { validator })
}
/// Append a fragment of XML to the validator
/// Returns a tuple containing:
/// - A boolean indicating success
/// - An optional error message
fn append(&mut self, fragment: &str) -> PyResult<(bool, Option<String>)> {
match self.validator.append(fragment) {
Ok(()) => PyResult::Ok((true, None)),
Err(e) => PyResult::Ok((false, Some(e.to_string()))),
}
}
/// Check if the validator has reached the end of the XML
fn eof(&self) -> PyResult<bool> {
Ok(self.validator.eof())
}
/// Create a deep copy of this validator
fn copy(&self) -> PyResult<Self> {
Ok(Self {
validator: self.validator.clone(),
})
}
/// Get a list of all element names in the schema
fn get_element_names(&self) -> PyResult<Vec<String>> {
Ok(self.validator.get_element_names())
}
/// Check if a specific character is valid as the next character
fn can_continue_with(&self, c: &str) -> PyResult<bool> {
if let Some(first_char) = c.chars().next() {
Ok(self.validator.can_continue_with(first_char))
} else {
Ok(false)
}
}
/// Validate an entire XML string and check if it's valid according to the schema
fn validate(&self, xml_string: &str) -> PyResult<bool> {
let mut validator_clone = self.validator.clone();
match validator_clone.append(xml_string) {
Ok(()) => Ok(validator_clone.eof()), // Valid only if we reached EOF
Err(_) => Ok(false), // Invalid XML
}
}
}
/// Register the class with the Python module
pub fn register_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<XmlSchemaValidator>()?;
Ok(())
}
use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
/// Python wrapper for XmlValidator
#[pyclass]
struct XmlSchemaValidator {
validator: crate::XmlSchemaValidator,
}
#[pymethods]
impl XmlSchemaValidator {
#[new]
fn new(schema_text: &str) -> PyResult<Self> {
let validator = crate::XmlSchemaValidator::new(schema_text)
.map_err(|e| PyValueError::new_err(format!("Failed to create validator: {}", e)))?;
Ok(Self { validator })
}
/// Append a fragment of XML to the validator
/// Returns a tuple containing:
/// - A boolean indicating success
/// - An optional error message
fn append(&mut self, fragment: &str) -> PyResult<(bool, Option<String>)> {
match self.validator.append(fragment) {
Ok(()) => PyResult::Ok((true, None)),
Err(e) => PyResult::Ok((false, Some(e.to_string()))),
}
}
/// Check if the validator has reached the end of the XML
fn eof(&self) -> PyResult<bool> {
Ok(self.validator.eof())
}
/// Create a deep copy of this validator
fn copy(&self) -> PyResult<Self> {
Ok(Self {
validator: self.validator.clone(),
})
}
/// Get a list of all element names in the schema
fn get_element_names(&self) -> PyResult<Vec<String>> {
Ok(self.validator.get_element_names())
}
/// Check if a specific character is valid as the next character
fn can_continue_with(&self, c: &str) -> PyResult<bool> {
if let Some(first_char) = c.chars().next() {
Ok(self.validator.can_continue_with(first_char))
} else {
Ok(false)
}
}
/// Validate an entire XML string and check if it's valid according to the schema
fn validate(&self, xml_string: &str) -> PyResult<bool> {
let mut validator_clone = self.validator.clone();
match validator_clone.append(xml_string) {
Ok(()) => Ok(validator_clone.eof()), // Valid only if we reached EOF
Err(_) => Ok(false), // Invalid XML
}
}
}
/// Register the class with the Python module
pub fn register_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<XmlSchemaValidator>()?;
Ok(())
}

18
lib/xml_schema_validator/src/python.rs
View File

@@ -1,9 +1,9 @@
use pyo3::prelude::*;
/// Python module for XML Schema validation
#[pymodule]
fn _rs(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
crate::py_xml_logits_processor_core::register_module(py, m)?;
crate::py_xml_schema_validator::register_module(py, m)?;
Ok(())
}
use pyo3::prelude::*;
/// Python module for XML Schema validation
#[pymodule]
fn _rs(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
crate::py_xml_logits_processor_core::register_module(py, m)?;
crate::py_xml_schema_validator::register_module(py, m)?;
Ok(())
}

61
lib/xml_schema_validator/src/schema.rs
View File

@@ -1,6 +1,14 @@
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsSchema {
#[serde(rename = "@xmlns:xs")]
pub xmlns_xs: String,
@@ -12,7 +20,15 @@ pub struct XsSchema {
pub xs_element: Vec<XsElement>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsElement {
#[serde(rename = "@name")]
pub name: String,
@@ -22,7 +38,15 @@ pub struct XsElement {
pub xs_complex_type: XsComplexType,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsComplexType {
#[serde(rename = "@mixed")]
pub mixed: Option<String>,
@@ -34,7 +58,15 @@ pub struct XsComplexType {
pub xs_sequence: Option<XsSequence>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsAttribute {
#[serde(rename = "@name")]
pub name: String,
@@ -44,7 +76,15 @@ pub struct XsAttribute {
pub xs_attribute_use: String,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsSequence {
#[serde(rename = "$text")]
pub text: Option<String>,
@@ -52,7 +92,15 @@ pub struct XsSequence {
pub xs_any: XsAny,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsAny {
#[serde(rename = "@minOccurs")]
pub min_occurs: String,
@@ -61,4 +109,3 @@ pub struct XsAny {
#[serde(rename = "@processContents")]
pub process_contents: String,
}

102
lib/xml_schema_validator/src/token/attribute_equals.rs Normal file
View File

@@ -0,0 +1,102 @@
use crate::*;
use std::sync::Arc;
/// Represents the equals sign after an XML attribute name
#[derive(Clone, Debug)]
pub struct AttributeEquals {
element: Arc<schema::XsElement>,
used_attributes: Vec<schema::XsAttribute>,
}
impl AttributeEquals {
pub fn new(
element: Arc<schema::XsElement>,
used_attributes: Vec<schema::XsAttribute>,
) -> Self {
Self {
element,
used_attributes,
}
}
pub fn append(self, c: char) -> Vec<Token> {
if c == '"' {
// Opening quote starts the attribute value
return vec![Token::AttributeValue(AttributeValue::new(
Arc::clone(&self.element),
self.used_attributes,
))];
} else if c.is_whitespace() {
// Allow whitespace between equals sign and opening quote
return vec![Token::AttributeEquals(self)];
} else {
// Any other character is invalid
return vec![];
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_attribute_equals_quote() {
let values = [
"<delete id=\"",
"<single timeout=\"",
"<single limit=\"",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeValue(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_attribute_equals_with_whitespace() {
let values = [
"<delete id= \"",
"<single timeout = \"",
"<single limit \t = \n \"",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeValue(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_invalid_after_equals() {
let values = [
"<delete id=a", // Only quote should be accepted after equals
"<single timeout=<", // Invalid character after equals
"<single timeout=1", // Unquoted numbers not supported yet
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
}

359
lib/xml_schema_validator/src/token/attribute_name.rs
View File

@@ -1,196 +1,165 @@
use std::sync::Arc;
use crate::*;
/// Represents an attribute name of an XML element
#[derive(Clone, Debug)]
pub struct AttributeName {
/// The element that owns this attribute
element: Arc<schema::XsElement>,
/// Buffer containing the attribute name characters processed so far
buffer: String,
/// Track which attributes have already been set for this element
attributes_set: Vec<String>,
}
impl AttributeName {
/// Create a new AttributeName with an empty buffer
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
buffer: String::new(),
attributes_set: Vec::new(),
}
}
/// Create a new AttributeName with the given buffer
pub fn with_buffer(element: Arc<schema::XsElement>, buffer: String, attributes_set: Vec<String>) -> Self {
Self {
element,
buffer,
attributes_set,
}
}
/// Create a new AttributeName with attributes already set
pub fn with_attributes(element: Arc<schema::XsElement>, attributes_set: Vec<String>) -> Self {
Self {
element,
buffer: String::new(),
attributes_set,
}
}
/// Append a character to the attribute name and return possible continuations
pub fn append(self, c: char) -> Vec<Token> {
if c.is_whitespace() && self.buffer.is_empty() {
return vec![Token::AttributeName(self)];
} else if c == '=' {
// Found the equals sign, transition to attribute value
// We need to verify that this attribute name is valid for this element
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
for attr in attributes {
if attr.name == self.buffer {
// Valid attribute, transition to attribute value
return vec![Token::AttributeValue(AttributeValue::new(
Arc::clone(&self.element),
self.buffer,
self.attributes_set,
))];
}
}
}
// Invalid attribute name
vec![]
} else {
// Continue building the attribute name
let new_buffer = self.buffer + &c.to_string();
// Check if this could be a valid attribute for this element
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
for attr in attributes {
if attr.name.starts_with(&new_buffer) {
// This could be a valid attribute, continue parsing
return vec![Token::AttributeName(AttributeName::with_buffer(
Arc::clone(&self.element),
new_buffer,
self.attributes_set.clone(),
))];
}
}
}
// No matching attribute found
vec![]
}
}
}
#[cfg(test)]
mod attribute_name_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_element_with_attributes() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "delete".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: Some(vec![
schema::XsAttribute {
name: "id".to_string(),
xs_attribute_type: "xs:string".to_string(),
xs_attribute_use: "required".to_string(),
}
]),
xs_sequence: None,
},
};
Arc::new(element)
}
#[test]
fn test_attribute_name_append() {
let element = create_element_with_attributes();
let token = AttributeName::new(Arc::clone(&element));
// Build attribute name character by character
let mut current_token = token;
for c in "id".chars() {
let next_tokens = current_token.append(c);
assert!(!next_tokens.is_empty(), "Should accept attribute name character '{}'", c);
match &next_tokens[0] {
Token::AttributeName(next) => current_token = next.clone(),
_ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
}
}
// Test equals sign
let next_tokens = current_token.append('=');
assert!(!next_tokens.is_empty(), "Should accept '=' after attribute name");
match &next_tokens[0] {
Token::AttributeValue(_) => (),
_ => panic!("Expected AttributeValue, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_attribute_name_whitespace() {
let element = create_element_with_attributes();
let token = AttributeName::new(Arc::clone(&element));
// Test with whitespace
let next_tokens = token.append(' ');
assert!(!next_tokens.is_empty(), "Should accept whitespace in attribute name");
match &next_tokens[0] {
Token::AttributeName(_) => (),
_ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_invalid_attribute_name() {
let element = create_element_with_attributes();
let token = AttributeName::new(Arc::clone(&element));
// Test with invalid attribute name
let next_tokens = token.append('x'); // 'x' doesn't start any valid attribute
assert!(next_tokens.is_empty(), "Should reject invalid attribute name");
}
#[test]
fn test_attribute_tracking() {
let element = create_element_with_attributes();
// Create AttributeName with pre-existing attributes
let attributes_set = vec!["other".to_string()];
let token = AttributeName::with_attributes(Arc::clone(&element), attributes_set.clone());
// Build attribute name and transition to AttributeValue
let mut current_token = token;
for c in "id".chars() {
let next_tokens = current_token.append(c);
match &next_tokens[0] {
Token::AttributeName(next) => current_token = next.clone(),
_ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
}
}
// Test equals sign
let next_tokens = current_token.append('=');
match &next_tokens[0] {
Token::AttributeValue(value) => {
// Verify that pre-existing attributes are preserved
assert!(value.attributes_set.contains(&"other".to_string()),
"Pre-existing attribute should be preserved");
},
_ => panic!("Expected AttributeValue, got {:?}", next_tokens[0]),
}
}
use crate::*;
use std::sync::Arc;
/// Represents an XML attribute name that is in the process of being parsed
#[derive(Clone, Debug)]
pub struct AttributeName {
buffer: String,
element: Arc<schema::XsElement>,
used_attributes: Vec<schema::XsAttribute>,
}
impl AttributeName {
pub fn new(first_char: char, element: Arc<schema::XsElement>) -> Self {
Self {
buffer: first_char.to_string(),
element,
used_attributes: Vec::new(),
}
}
pub fn new_with_used(first_char: char, element: Arc<schema::XsElement>, used_attributes: Vec<schema::XsAttribute>) -> Self {
Self {
buffer: first_char.to_string(),
element,
used_attributes,
}
}
pub fn append(mut self, c: char) -> Vec<Token> {
if c == '=' {
// Check if we've matched a valid attribute name
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
if let Some(attribute) = attributes
.iter()
.find(|attribute| attribute.name == self.buffer)
{
// Check if this attribute has already been used
if self.used_attributes.contains(&attribute) {
return vec![];
}
// Add this attribute to the used set
self.used_attributes.push(attribute.clone());
return vec![Token::AttributeEquals(AttributeEquals::new(
Arc::clone(&self.element),
self.used_attributes,
))];
}
}
return vec![];
} else if c.is_whitespace() {
// Check if we've matched a valid attribute name and allow whitespace after it
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
if attributes
.iter()
.any(|attribute| attribute.name == self.buffer)
{
// Check if this attribute has already been used
if self.used_attributes.iter().filter(|attribute| attribute.name == self.buffer).next().is_some() {
return vec![];
}
return vec![Token::AttributeName(self)];
}
}
return vec![];
} else {
// Add character to buffer and check if we're still building a valid attribute name
let new_buffer = self.buffer.clone() + &c.to_string();
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
// Check if any attribute name starts with our buffer
if attributes
.iter()
.any(|attribute| attribute.name.starts_with(&new_buffer))
{
return vec![Token::AttributeName(AttributeName {
buffer: new_buffer,
element: Arc::clone(&self.element),
used_attributes: self.used_attributes,
})];
}
}
// Character doesn't match what we expect for this attribute name
return vec![];
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_valid_attribute_name() {
let values = [
"<delete id=",
"<single timeout=",
"<single limit=",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeEquals(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_attribute_name_with_whitespace() {
let values = [
"<delete id =",
"<single timeout =",
"<single limit\t=",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeEquals(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_invalid_attribute_name() {
let values = [
"<delete invalid=",
"<single wrongattr=",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
#[test]
fn test_duplicate_attribute() {
let mut validator = crate::tests::example_validator();
// First attribute is valid
validator.append("<delete id=\"123\" ").expect("First attribute should be accepted");
// Try to add the same attribute again
let result = validator.append("id=");
assert!(result.is_err(), "Duplicate attribute should be rejected");
}
}

442
lib/xml_schema_validator/src/token/attribute_value.rs
View File

@@ -1,253 +1,191 @@
use std::sync::Arc;
use crate::*;
/// Represents an attribute value of an XML element
#[derive(Clone, Debug)]
pub struct AttributeValue {
/// The element that owns this attribute
element: Arc<schema::XsElement>,
/// The name of the attribute being processed
attribute_name: String,
/// Buffer containing the attribute value characters processed so far
buffer: String,
/// Whether we've encountered the opening quote
in_quotes: bool,
/// Whether we've seen the closing quote
closed: bool,
/// Track which attributes have already been set for this element
pub attributes_set: Vec<String>,
}
impl AttributeValue {
/// Create a new AttributeValue with a list of attributes already set
pub fn new(element: Arc<schema::XsElement>, attribute_name: String, attributes_set: Vec<String>) -> Self {
Self {
element,
attribute_name,
buffer: String::new(),
in_quotes: false,
closed: false,
attributes_set,
}
}
/// Append a character to the attribute value and return possible continuations
pub fn append(self, c: char) -> Vec<Token> {
if self.closed {
// We've already closed the attribute value with a quote
// Add this attribute to the list of attributes set
let mut new_attributes = self.attributes_set.clone();
if !new_attributes.contains(&self.attribute_name) {
new_attributes.push(self.attribute_name.clone());
}
if c.is_whitespace() {
// Check if there are more attributes to parse
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
// If there are other required attributes that haven't been set yet
for attr in attributes {
if attr.name != self.attribute_name && attr.xs_attribute_use == "required" && !new_attributes.contains(&attr.name) {
return vec![Token::AttributeName(AttributeName::with_attributes(
Arc::clone(&self.element),
new_attributes,
))];
}
}
}
// No more required attributes, whitespace after attribute value, continue to next state
return vec![
Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes.clone())),
Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))
];
} else if c == '>' {
// End of opening tag
return vec![Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes))];
} else if c == '/' {
// Beginning of self-closing tag
return vec![Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))];
} else {
// Unexpected character after attribute value
return vec![];
}
}
if !self.in_quotes {
if c.is_whitespace() {
// Skip whitespace before the quotes
return vec![Token::AttributeValue(Self {
element: Arc::clone(&self.element),
attribute_name: self.attribute_name,
buffer: self.buffer,
in_quotes: false,
closed: false,
attributes_set: self.attributes_set,
})];
} else if c == '"' || c == '\'' {
// Start of quotes
return vec![Token::AttributeValue(Self {
element: Arc::clone(&self.element),
attribute_name: self.attribute_name,
buffer: self.buffer,
in_quotes: true,
closed: false,
attributes_set: self.attributes_set,
})];
} else {
// Unexpected character before quotes
return vec![];
}
} else {
// We're inside quotes
if c == '"' || c == '\'' {
// End of quotes, validate the attribute value
// In a real implementation, we'd check if the value is valid for this attribute type
return vec![Token::AttributeValue(Self {
element: Arc::clone(&self.element),
attribute_name: self.attribute_name,
buffer: self.buffer,
in_quotes: true,
closed: true,
attributes_set: self.attributes_set,
})];
} else {
// Continue building the attribute value
let new_buffer = self.buffer + &c.to_string();
return vec![Token::AttributeValue(Self {
element: Arc::clone(&self.element),
attribute_name: self.attribute_name,
buffer: new_buffer,
in_quotes: true,
closed: false,
attributes_set: self.attributes_set,
})];
}
}
}
}
#[cfg(test)]
mod attribute_value_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_element_with_attributes() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "delete".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: Some(vec![
schema::XsAttribute {
name: "id".to_string(),
xs_attribute_type: "xs:string".to_string(),
xs_attribute_use: "required".to_string(),
}
]),
xs_sequence: None,
},
};
Arc::new(element)
}
#[test]
fn test_attribute_value_quotes() {
let element = create_element_with_attributes();
let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
// Test with opening quote
let next_tokens = token.append('"');
assert!(!next_tokens.is_empty(), "Should accept opening quote");
let token = next_tokens[0].clone();
match token.clone() {
Token::AttributeValue(value) => {
assert!(value.in_quotes, "Should mark as in quotes");
assert!(!value.closed, "Should not be closed yet");
},
_ => panic!("Expected AttributeValue, got {:?}", token),
}
// Test with content inside quotes
let next_tokens = token.append('a');
assert!(!next_tokens.is_empty(), "Should accept content inside quotes");
let token = next_tokens[0].clone();
// Test with closing quote
let next_tokens = token.append('"');
assert!(!next_tokens.is_empty(), "Should accept closing quote");
let token = next_tokens[0].clone();
match token {
Token::AttributeValue(value) => {
assert!(value.closed, "Should be closed");
},
_ => panic!("Expected AttributeValue, got {:?}", token),
}
}
#[test]
fn test_after_attribute_value_closed() {
let element = create_element_with_attributes();
let mut token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
token.in_quotes = true;
token.closed = true;
// Test with space after closed attribute value
let next_tokens = token.append(' ');
assert!(!next_tokens.is_empty(), "Should accept space after closed attribute");
// Test with > after closed attribute value
let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
let mut token = token;
token.in_quotes = true;
token.closed = true;
let next_tokens = token.append('>');
assert!(!next_tokens.is_empty(), "Should accept '>' after closed attribute");
match &next_tokens[0] {
Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
}
// Test with / after closed attribute value (for self-closing)
let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
let mut token = token;
token.in_quotes = true;
token.closed = true;
let next_tokens = token.append('/');
assert!(!next_tokens.is_empty(), "Should accept '/' after closed attribute");
match &next_tokens[0] {
Token::ElementSelfClose(_) => (),
_ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_attribute_tracking() {
let element = create_element_with_attributes();
let mut token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
token.in_quotes = true;
token.closed = true;
// After attribute is closed, it should be tracked
let next_tokens = token.append('>');
assert!(!next_tokens.is_empty(), "Should accept '>' after attribute");
match &next_tokens[0] {
Token::ElementOpenEnd(element_open) => {
// Verify that the id attribute is tracked
assert!(element_open.attributes_set.contains(&"id".to_string()),
"Attribute 'id' should be tracked");
},
_ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
}
}
use crate::*;
/// Represents an attribute value of an XML element
#[derive(Clone, Debug)]
pub struct AttributeValue {
/// The element that owns this attribute
element: Arc<schema::XsElement>,
/// Buffer containing the attribute value characters processed so far
buffer: String,
/// Track which attributes have already been set for this element
pub attributes_set: Vec<schema::XsAttribute>,
}
impl AttributeValue {
/// Create a new AttributeValue with a list of attributes already set
pub fn new(
element: Arc<schema::XsElement>,
attributes_set: Vec<schema::XsAttribute>,
) -> Self {
Self {
element,
buffer: "\"".to_string(),
attributes_set,
}
}
/// Append a character to the attribute value and return possible continuations
pub fn append(mut self, c: char) -> Vec<Token> {
// Handle escaped characters
if Self::in_escape_sequence(&self.buffer) {
self.buffer.push(c);
return vec![Token::AttributeValue(self)];
} else if self.value_closed() {
return self.next_tokens(c);
} else if '"' == c {
self.buffer.push(c);
return vec![Token::AttributeValue(self)];
} else {
self.buffer.push(c);
return vec![Token::AttributeValue(self)];
}
}
/// Checks if an XML character entity reference is in progress and correctly terminated
/// Returns true if the sequence is complete (ends with a semicolon)
pub fn in_escape_sequence(buffer: &str) -> bool {
if let Some(amp_pos) = buffer.rfind('&') {
if buffer[amp_pos..].find(';').is_some() {
return false;
} else {
return true;
}
}
false
}
fn value_closed(&self) -> bool {
if self.buffer.len() > 1 && self.buffer.trim().ends_with("\"") {
let trimmed = self.buffer.trim();
let trimmed = &trimmed[..trimmed.len() - 2];
if Self::in_escape_sequence(trimmed) {
false
} else {
true
}
} else {
false
}
}
fn next_tokens(mut self, c: char) -> Vec<Token> {
let all_required_attributes_set: bool = self
.element
.as_ref()
.xs_complex_type
.xs_attribute
.as_deref()
.unwrap()
.iter()
.all(|attr| {
if attr.xs_attribute_use == "required" {
self.attributes_set.contains(attr)
} else {
true
}
});
let all_attributes_set: bool = self
.element
.as_ref()
.xs_complex_type
.xs_attribute
.as_deref()
.unwrap()
.iter()
.all(|attr| self.attributes_set.contains(attr));
if c.is_whitespace() {
self.buffer.push(c);
vec![Token::AttributeValue(self)]
} else {
let mut tokens = vec![];
if all_required_attributes_set {
if '/' == c {
tokens.push(Token::ElementSelfClose(ElementSelfClose::new()));
} else if '>' == c {
tokens.push(Token::ElementOpenEnd(ElementOpenEnd::new(self.element.clone())));
}
}
if self.buffer.ends_with(char::is_whitespace) && !all_attributes_set {
tokens.push(Token::AttributeName(AttributeName::new_with_used(
c,
self.element,
self.attributes_set,
)));
}
tokens
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_unfinished_attribute_value() {
let values = [
"<delete id=\"12",
"<single timeout=\"1.",
"<single limit=\"50",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
// Should still be in AttributeValue state
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::AttributeValue(_))),
"{value}"
);
}
}
#[test]
fn test_type_validation() {
// Test integer validation
let mut validator = crate::tests::example_validator();
validator.append("<single limit=\"123\"").expect("Valid integer");
// Test float validation
validator = crate::tests::example_validator();
validator.append("<single timeout=\"1.5\"").expect("Valid float");
// Test string validation
validator = crate::tests::example_validator();
validator.append("<delete id=\"abc123\"").expect("Valid string");
}
#[test]
fn test_multiple_attributes() {
let values = [
"<single limit=\"100\" timeout=\"1.5\"",
"<single timeout=\"0.5\" limit=\"200\"",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
}
}
#[test]
fn test_attribute_value_escape_chars() {
// Test handling of escaped characters in attribute values
let values = [
"<delete id=\"escaped&quot;quote\"",
"<delete id=\"escaped&amp;amp\"",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
}
}
}

286
lib/xml_schema_validator/src/token/element_close_name.rs
View File

@@ -1,137 +1,149 @@
use std::sync::Arc;
use crate::*;
/// Represents a closing element tag that is in the process of being parsed
/// after the '</' and now parsing the element name
#[derive(Clone, Debug)]
pub struct ElementCloseName {
/// The element that was previously opened and is now being closed
element: Arc<schema::XsElement>,
/// Current buffer of characters processed for the element name
buffer: String,
}
impl ElementCloseName {
/// Create a new instance with the given buffer
pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Self {
Self {
element,
buffer,
}
}
pub fn append(self, c: char) -> Vec<Token> {
if self.element.name == self.buffer && c.is_whitespace() {
return vec![Token::ElementCloseName(self)];
}
if self.element.name == self.buffer && c == '>' {
return vec![Token::EndOfFile(EndOfFile::new())];
}
let new_buffer = self.buffer + &c.to_string();
if self.element.name.starts_with(&new_buffer) {
return vec![Token::ElementCloseName(ElementCloseName::new(
Arc::clone(&self.element),
new_buffer,
))];
} else {
vec![]
}
}
}
#[cfg(test)]
mod element_close_name_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_close_name_partial() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
// Test with next character in name
let next_tokens = token.append('e');
assert!(!next_tokens.is_empty(), "Should accept next character in name");
match &next_tokens[0] {
Token::ElementCloseName(name) => {
assert_eq!(name.buffer, "re", "Buffer should contain 're'");
},
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_name_complete() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "reasonin".to_string());
// Test with last character of name
let next_tokens = token.append('g');
assert!(!next_tokens.is_empty(), "Should accept last character of name");
match &next_tokens[0] {
Token::ElementCloseName(name) => {
assert_eq!(name.buffer, "reasoning", "Buffer should contain 'reasoning'");
},
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
// Test with > after complete name
let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
let next_tokens = token.append('>');
assert!(!next_tokens.is_empty(), "Should accept '>' after complete name");
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_name_whitespace() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
// Test with whitespace after complete name
let next_tokens = token.append(' ');
assert!(!next_tokens.is_empty(), "Should accept whitespace after complete name");
match &next_tokens[0] {
Token::ElementCloseName(_) => (),
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_name_invalid() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
// Test with invalid next character
let next_tokens = token.append('x');
assert!(next_tokens.is_empty(), "Should reject invalid character");
}
}
use crate::*;
use std::sync::Arc;
/// Represents a closing element tag that is in the process of being parsed
/// after the '</' and now parsing the element name
#[derive(Clone, Debug)]
pub struct ElementCloseName {
/// The element that was previously opened and is now being closed
element: Arc<schema::XsElement>,
/// Current buffer of characters processed for the element name
buffer: String,
}
impl ElementCloseName {
/// Create a new instance with the given buffer
pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Self {
Self { element, buffer }
}
pub fn append(self, c: char) -> Vec<Token> {
if self.element.name == self.buffer && c.is_whitespace() {
return vec![Token::ElementCloseName(self)];
}
if self.element.name == self.buffer && c == '>' {
return vec![Token::EndOfFile(EndOfFile::new())];
}
let new_buffer = self.buffer + &c.to_string();
if self.element.name.starts_with(&new_buffer) {
return vec![Token::ElementCloseName(ElementCloseName::new(
Arc::clone(&self.element),
new_buffer,
))];
} else {
vec![]
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_close_name_partial() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
// Test with next character in name
let next_tokens = token.append('e');
assert!(
!next_tokens.is_empty(),
"Should accept next character in name"
);
match &next_tokens[0] {
Token::ElementCloseName(name) => {
assert_eq!(name.buffer, "re", "Buffer should contain 're'");
}
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_name_complete() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "reasonin".to_string());
// Test with last character of name
let next_tokens = token.append('g');
assert!(
!next_tokens.is_empty(),
"Should accept last character of name"
);
match &next_tokens[0] {
Token::ElementCloseName(name) => {
assert_eq!(
name.buffer, "reasoning",
"Buffer should contain 'reasoning'"
);
}
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
// Test with > after complete name
let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
let next_tokens = token.append('>');
assert!(
!next_tokens.is_empty(),
"Should accept '>' after complete name"
);
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_name_whitespace() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
// Test with whitespace after complete name
let next_tokens = token.append(' ');
assert!(
!next_tokens.is_empty(),
"Should accept whitespace after complete name"
);
match &next_tokens[0] {
Token::ElementCloseName(_) => (),
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_name_invalid() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
// Test with invalid next character
let next_tokens = token.append('x');
assert!(next_tokens.is_empty(), "Should reject invalid character");
}
}

217
lib/xml_schema_validator/src/token/element_close_start.rs
View File

@@ -1,107 +1,110 @@
use std::sync::Arc;
use crate::*;
/// Represents the start of an XML element closing tag (the '</' sequence)
#[derive(Clone, Debug)]
pub struct ElementCloseStart {
/// The element that was previously opened and is now being closed
element: Arc<schema::XsElement>,
has_slash: bool,
}
impl ElementCloseStart {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
has_slash: false,
}
}
pub fn append(self, c: char) -> Vec<Token> {
if !self.has_slash && '/' == c {
vec![Token::ElementCloseStart(Self {
element: Arc::clone(&self.element),
has_slash: true,
})]
} else if self.has_slash && self.element.name.starts_with(c) {
vec![Token::ElementCloseName(ElementCloseName::new(
Arc::clone(&self.element),
c.to_string(),
))]
} else {
vec![]
}
}
}
#[cfg(test)]
mod element_close_start_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_close_start_slash() {
let element = create_test_element();
let token = ElementCloseStart::new(Arc::clone(&element));
// Test with slash
let next_tokens = token.append('/');
assert!(!next_tokens.is_empty(), "Should accept '/'");
match &next_tokens[0] {
Token::ElementCloseStart(start) => {
assert!(start.has_slash, "Should mark as having slash");
},
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_start_element_name() {
let element = create_test_element();
let mut token = ElementCloseStart::new(Arc::clone(&element));
token.has_slash = true;
// Test with first character of element name
let next_tokens = token.append('r');
assert!(!next_tokens.is_empty(), "Should accept first letter of element name");
match &next_tokens[0] {
Token::ElementCloseName(_) => (),
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_start_invalid_name() {
let element = create_test_element();
let mut token = ElementCloseStart::new(Arc::clone(&element));
token.has_slash = true;
// Test with invalid first character of element name
let next_tokens = token.append('x');
assert!(next_tokens.is_empty(), "Should reject invalid first letter");
}
}
use crate::*;
use std::sync::Arc;
/// Represents the start of an XML element closing tag (the '</' sequence)
#[derive(Clone, Debug)]
pub struct ElementCloseStart {
/// The element that was previously opened and is now being closed
element: Arc<schema::XsElement>,
has_slash: bool,
}
impl ElementCloseStart {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
has_slash: false,
}
}
pub fn append(self, c: char) -> Vec<Token> {
if !self.has_slash && '/' == c {
vec![Token::ElementCloseStart(Self {
element: Arc::clone(&self.element),
has_slash: true,
})]
} else if self.has_slash && self.element.name.starts_with(c) {
vec![Token::ElementCloseName(ElementCloseName::new(
Arc::clone(&self.element),
c.to_string(),
))]
} else {
vec![]
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_close_start_slash() {
let element = create_test_element();
let token = ElementCloseStart::new(Arc::clone(&element));
// Test with slash
let next_tokens = token.append('/');
assert!(!next_tokens.is_empty(), "Should accept '/'");
match &next_tokens[0] {
Token::ElementCloseStart(start) => {
assert!(start.has_slash, "Should mark as having slash");
}
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_start_element_name() {
let element = create_test_element();
let mut token = ElementCloseStart::new(Arc::clone(&element));
token.has_slash = true;
// Test with first character of element name
let next_tokens = token.append('r');
assert!(
!next_tokens.is_empty(),
"Should accept first letter of element name"
);
match &next_tokens[0] {
Token::ElementCloseName(_) => (),
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_start_invalid_name() {
let element = create_test_element();
let mut token = ElementCloseStart::new(Arc::clone(&element));
token.has_slash = true;
// Test with invalid first character of element name
let next_tokens = token.append('x');
assert!(next_tokens.is_empty(), "Should reject invalid first letter");
}
}

383
lib/xml_schema_validator/src/token/element_open_end.rs
View File

@@ -1,220 +1,163 @@
use std::sync::Arc;
use crate::*;
/// Represents a fully opened XML element, containing the schema element reference
#[derive(Clone, Debug)]
pub struct ElementOpenEnd {
element: Arc<schema::XsElement>,
// Track which attributes have been set for this element
pub attributes_set: Vec<String>,
}
impl ElementOpenEnd {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
attributes_set: Vec::new(),
}
}
pub fn with_attributes(element: Arc<schema::XsElement>, attributes_set: Vec<String>) -> Self {
Self {
element,
attributes_set,
}
}
pub fn append(self, c: char) -> Vec<Token> {
// Check if all required attributes are present before proceeding
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
for attr in attributes {
if attr.xs_attribute_use == "required" && !self.attributes_set.contains(&attr.name) {
// Missing a required attribute, this is an error
return vec![];
}
}
}
// Check if the element is a mixed content element that allows text
let allows_text = self.element.xs_complex_type.mixed
.as_ref()
.map(|mixed| mixed == "true")
.unwrap_or(false);
if allows_text {
if c.is_whitespace() {
// Handle whitespace in mixed content
vec![
Token::TextContent(TextContent::new(Arc::clone(&self.element))),
Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))
]
} else if c == '<' {
// Start of a closing tag or nested element
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
} else {
// Text content
vec![Token::TextContent(TextContent::new(Arc::clone(&self.element)))]
}
} else {
// Element doesn't allow text content, only expect closing tag
if c == '<' {
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
} else if c.is_whitespace() {
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
} else {
// Unexpected content in non-mixed element
vec![]
}
}
}
}
#[cfg(test)]
mod element_open_end_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
fn create_element_non_mixed() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "element".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None, // Not mixed content
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
fn create_element_with_required_attributes() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "delete".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: Some(vec![
schema::XsAttribute {
name: "id".to_string(),
xs_attribute_type: "xs:string".to_string(),
xs_attribute_use: "required".to_string(),
}
]),
xs_sequence: None,
},
};
Arc::new(element)
}
#[test]
fn test_append_text_in_mixed_content() {
let element = create_test_element();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test appending text in mixed content element
let next_tokens = token.append('a');
assert!(!next_tokens.is_empty(), "Should allow text content in mixed element");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_closing_tag_start() {
let element = create_test_element();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test starting a closing tag
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should allow closing tag start");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_whitespace_in_mixed() {
let element = create_test_element();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test whitespace in mixed content
let next_tokens = token.append(' ');
assert!(!next_tokens.is_empty(), "Should allow whitespace in mixed content");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_in_non_mixed() {
let element = create_element_non_mixed();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test appending text in non-mixed content (should reject)
let next_tokens = token.clone().append('a');
assert!(next_tokens.is_empty(), "Should reject text in non-mixed element");
// Test whitespace (should accept leading to closing tag)
let next_tokens = token.clone().append(' ');
assert!(!next_tokens.is_empty(), "Should allow whitespace in non-mixed");
// Test closing tag start
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should allow closing tag start in non-mixed");
}
#[test]
fn test_required_attributes() {
let element = create_element_with_required_attributes();
// No attributes set yet
let token = ElementOpenEnd::new(Arc::clone(&element));
let next_tokens = token.append('<');
assert!(next_tokens.is_empty(), "Should reject closing tag without required attributes");
// With required attribute set
let token = ElementOpenEnd::with_attributes(
Arc::clone(&element),
vec!["id".to_string()]
);
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should allow closing tag with required attributes set");
}
}
use crate::*;
use std::sync::Arc;
/// Represents a fully opened XML element, containing the schema element reference
#[derive(Clone, Debug)]
pub struct ElementOpenEnd {
element: Arc<schema::XsElement>,
}
impl ElementOpenEnd {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
}
}
pub fn append(self, c: char) -> Vec<Token> {
if '<' == c {
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
} else {
// Check if this element allows mixed content
let is_mixed = self.element.xs_complex_type.mixed
.as_ref()
.map(|val| val == "true")
.unwrap_or(false);
if is_mixed || c.is_whitespace() {
// Allow text content for mixed elements, or whitespace for any element
vec![Token::TextContent(TextContent::new(self.element))]
} else {
// Reject text content for non-mixed elements
vec![]
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
fn create_element_non_mixed() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "element".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None, // Not mixed content
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_append_text_in_mixed_content() {
let element = create_test_element();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test appending text in mixed content element
let next_tokens = token.append('a');
assert!(
!next_tokens.is_empty(),
"Should allow text content in mixed element"
);
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_closing_tag_start() {
let element = create_test_element();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test starting a closing tag
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should allow closing tag start");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_whitespace_in_mixed() {
let element = create_test_element();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test whitespace in mixed content
let next_tokens = token.append(' ');
assert!(
!next_tokens.is_empty(),
"Should allow whitespace in mixed content"
);
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_in_non_mixed() {
let element = create_element_non_mixed();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test appending text in non-mixed content (should reject)
let next_tokens = token.clone().append('a');
assert!(
next_tokens.is_empty(),
"Should reject text in non-mixed element"
);
// Test whitespace (should accept leading to closing tag)
let next_tokens = token.clone().append(' ');
assert!(
!next_tokens.is_empty(),
"Should allow whitespace in non-mixed"
);
// Test closing tag start
let next_tokens = token.append('<');
assert!(
!next_tokens.is_empty(),
"Should allow closing tag start in non-mixed"
);
}
}

458
lib/xml_schema_validator/src/token/element_open_name.rs
View File

@@ -1,272 +1,186 @@
use std::sync::Arc;
use crate::*;
/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
#[derive(Clone, Debug)]
pub struct ElementOpenName {
buffer: String,
element: Arc<schema::XsElement>,
}
impl ElementOpenName {
pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Result<Self> {
if element.name.starts_with(buffer.as_str()) {
Ok(Self {
buffer,
element,
})
} else {
Err(Error::InvalidXml(format!("Element name '{}' doesn't match '{}'", buffer, element.name)))
}
}
pub fn append(self, c: char) -> Vec<Token> {
if c.is_whitespace() {
// Check if we've matched the full element name
if self.element.name == self.buffer {
// Element name is complete, handle whitespace after name
// Check if the element has attributes
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
if !attributes.is_empty() {
// Transition to attribute name parsing
return vec![Token::AttributeName(AttributeName::new(Arc::clone(&self.element)))];
}
}
// No attributes
return vec![
Token::AttributeName(AttributeName::new(Arc::clone(&self.element))),
Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element))),
Token::ElementSelfClose(ElementSelfClose::new()),
Token::ElementOpenName(self),
];
} else {
// Whitespace in the middle of an element name is invalid
return vec![];
}
} else if c == '>' {
// Check if we've matched the full element name
if self.element.name == self.buffer {
// Element name is complete, end of opening tag
// Check if this element has required attributes
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
for attr in attributes {
if attr.xs_attribute_use == "required" {
// Found a required attribute but no attributes are set
// Return an empty vector to indicate error
return vec![];
}
}
}
return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element)))];
} else {
// '>' before full element name is invalid
return vec![];
}
} else if c == '/' {
// Check if we've matched the full element name
if self.element.name == self.buffer {
// Element name is complete, self-closing tag
return vec![Token::ElementSelfClose(ElementSelfClose::new())];
} else {
// '/' before full element name is invalid
return vec![];
}
} else {
// Add character to buffer and check if we're still building a valid element name
let new_buffer = self.buffer.clone() + &c.to_string();
if self.element.name.starts_with(&new_buffer) {
// Still building the element name
return vec![Token::ElementOpenName(ElementOpenName {
buffer: new_buffer,
element: Arc::clone(&self.element),
})];
} else {
// Character doesn't match what we expect for this element name
return vec![];
}
}
}
}
#[cfg(test)]
mod element_open_name_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
fn create_element_with_attributes() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "delete".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: Some(vec![
schema::XsAttribute {
name: "id".to_string(),
xs_attribute_type: "xs:string".to_string(),
xs_attribute_use: "required".to_string(),
}
]),
xs_sequence: None,
},
};
Arc::new(element)
}
#[test]
fn test_append_character_by_character() {
let element = create_test_element();
// Test each character in "reasoning"
let mut buffer = String::new();
for c in "reasoning".chars() {
buffer.push(c);
let result = ElementOpenName::new(Arc::clone(&element), buffer.clone());
assert!(result.is_ok(), "Failed to create ElementOpenName with buffer '{}'", buffer);
}
}
#[test]
fn test_append_full_name() {
let element = create_test_element();
// Test with the full element name
let result = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string());
assert!(result.is_ok(), "Failed to create ElementOpenName with full element name");
let element_name = result.unwrap();
// Test transition after full name with '>'
let next_tokens = element_name.append('>');
assert!(!next_tokens.is_empty(), "Should accept '>' after full element name");
match &next_tokens[0] {
Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_space_after_name() {
let element = create_test_element();
// Test with the full element name
let element_name = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()).unwrap();
// Test transition after full name with space
let next_tokens = element_name.append(' ');
assert!(!next_tokens.is_empty(), "Should accept space after full element name");
assert!(next_tokens.len() == 4);
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::ElementOpenName(_)))
.next()
.is_some());
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::AttributeName(_)))
.next()
.is_some());
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::ElementOpenEnd(_)))
.next()
.is_some());
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::ElementSelfClose(_)))
.next()
.is_some());
}
#[test]
fn test_append_with_attributes() {
let element = create_element_with_attributes();
// Test with the full element name
let element_name = ElementOpenName::new(Arc::clone(&element), "delete".to_string()).unwrap();
// Test transition after full name with space (expecting attribute name)
let next_tokens = element_name.append(' ');
assert!(!next_tokens.is_empty(), "Should accept space after element name with attributes");
// The space should lead to a whitespace token, which should then lead to attribute name
match &next_tokens[0] {
Token::AttributeName(_) => (),
_ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_slash_after_name() {
let element = create_test_element();
// Test with the full element name
let element_name = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()).unwrap();
// Test transition after full name with /
let next_tokens = element_name.append('/');
assert!(!next_tokens.is_empty(), "Should accept '/' after full element name");
match &next_tokens[0] {
Token::ElementSelfClose(_) => (),
_ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_invalid_continuation() {
let element = create_test_element();
// Test with a partial element name and an invalid continuation character
let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap();
let next_tokens = element_name.append('x');
assert!(next_tokens.is_empty(), "Should reject invalid character in element name");
// Test whitespace in middle of name - need to create a new instance since append consumes self
let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap();
let next_tokens = element_name.append(' ');
assert!(next_tokens.is_empty(), "Should reject whitespace in middle of element name");
// Test > in middle of name - need to create a new instance since append consumes self
let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap();
let next_tokens = element_name.append('>');
assert!(next_tokens.is_empty(), "Should reject '>' in middle of element name");
}
#[test]
fn test_required_attributes_validation() {
let element = create_element_with_attributes();
// Test with the full element name
let element_name = ElementOpenName::new(Arc::clone(&element), "delete".to_string()).unwrap();
// Try to close the tag with > directly, skipping the required attribute
let next_tokens = element_name.append('>');
assert!(next_tokens.is_empty(), "Should reject closing tag without required attributes");
}
}
use crate::*;
use std::sync::Arc;
/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
#[derive(Clone, Debug)]
pub struct ElementOpenName {
buffer: String,
element: Arc<schema::XsElement>,
}
impl ElementOpenName {
pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Self {
Self { buffer, element }
}
pub fn append(mut self, c: char) -> Vec<Token> {
if c == '>' {
// Check if we've matched the full element name
if !self.buffer.contains(&self.element.name) {
return vec![];
}
// Check if the element has attributes
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
// Check if they are required
if attributes
.iter()
.filter(|attribute| attribute.xs_attribute_use == "required")
.next()
.is_some()
{
return vec![];
} else {
return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(
&self.element,
)))];
}
} else {
return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(
&self.element,
)))];
}
} else if c == '/' {
// Check if we've matched the full element name
if !self.buffer.contains(&self.element.name) {
return vec![];
}
// Check if the element has attributes
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
// Check if they are required
if attributes
.iter()
.filter(|attribute| attribute.xs_attribute_use == "required")
.next()
.is_some()
{
return vec![];
} else {
return vec![Token::ElementSelfClose(ElementSelfClose::new())];
}
} else {
return vec![Token::ElementSelfClose(ElementSelfClose::new())];
}
} else if c.is_whitespace() {
// Check if we've matched the full element name
if self.buffer.contains(&self.element.name) {
if let Some(last) = self.buffer.chars().last() {
if last.is_whitespace() {
return vec![Token::ElementOpenName(self)];
}
}
self.buffer.push(c);
return vec![Token::ElementOpenName(self)];
} else {
return vec![];
}
} else if !self.buffer.is_empty() && self.buffer.chars().last().unwrap().is_whitespace() {
// Check for start of attribute name
if self.buffer.contains(&self.element.name) {
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
if attributes
.iter()
.filter(|attribute| attribute.name.starts_with(c))
.next()
.is_some()
{
return vec![Token::AttributeName(AttributeName::new(
c,
self.element.clone(),
))];
}
}
}
return vec![];
} else {
// Add character to buffer and check if we're still building a valid element name
let new_buffer = self.buffer.clone() + &c.to_string();
if self.element.name.starts_with(&new_buffer) {
// Still building the element name
return vec![Token::ElementOpenName(ElementOpenName {
buffer: new_buffer,
element: Arc::clone(&self.element),
})];
} else {
// Character doesn't match what we expect for this element name
return vec![];
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_attribute_name() {
let values = [
"<delete i", // id
"<single t", // timeout
"<single l", // limit
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeName(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_valid_self_closing() {
let values = ["<stop/", "<stop /"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementSelfClose(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_valid_closing() {
let values = ["<stop>", "<stop >", "<reasoning>", "<reasoning >"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementOpenEnd(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_invalid_closing() {
let values = ["<delete/", "<delete /", "<delete>", "<delete >"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
}

89
lib/xml_schema_validator/src/token/element_open_start.rs
View File

@@ -1,25 +1,64 @@
use std::sync::Arc;
use crate::*;
/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
#[derive(Clone, Debug)]
pub struct ElementOpenStart {
element: Arc<schema::XsElement>,
}
impl ElementOpenStart {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
}
}
pub fn append(self, c: char) -> Vec<Token> {
let element_name = ElementOpenName::new(Arc::clone(&self.element), c.to_string());
if let Ok(element_name) = element_name {
vec![Token::ElementOpenName(element_name)]
} else {
vec![]
}
}
}
use crate::*;
use std::sync::Arc;
/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
#[derive(Clone, Debug)]
pub struct ElementOpenStart {
element: Arc<schema::XsElement>,
}
impl ElementOpenStart {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self { element }
}
pub fn append(self, c: char) -> Vec<Token> {
if self.element.name.starts_with(c) {
vec![Token::ElementOpenName(ElementOpenName::new(
Arc::clone(&self.element),
c.to_string(),
))]
} else {
vec![]
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_accept_valid_element_names() {
let values = [
"<d", // delete
"<s", // stop / single
"<r", // repeat / reasoning / read_stdin
"<w", // write_stdout
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementOpenName(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_not_accept_other() {
let values = ["< ", "<a", "</"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
}

198
lib/xml_schema_validator/src/token/element_self_close.rs
View File

@@ -1,144 +1,54 @@
use std::sync::Arc;
use crate::*;
/// Represents a self-closing XML element
#[derive(Clone, Debug)]
pub struct ElementSelfClose {
/// Optional reference to the element schema for attribute validation
element: Option<Arc<schema::XsElement>>,
/// Track which attributes have been set
attributes_set: Vec<String>,
}
impl ElementSelfClose {
/// Create a new self-closing element
pub fn new() -> Self {
Self {
element: None,
attributes_set: Vec::new(),
}
}
/// Create a new self-closing element with element reference and attribute tracking
pub fn new_with_attributes(element: Arc<schema::XsElement>, attributes_set: Vec<String>) -> Self {
Self {
element: Some(element),
attributes_set,
}
}
/// Append a character to the self-closing tag and return possible continuations
pub fn append(self, c: char) -> Vec<Token> {
// Before we close, verify that all required attributes are present
if let Some(element) = &self.element {
if c == '>' {
if let Some(attributes) = &element.xs_complex_type.xs_attribute {
for attr in attributes {
if attr.xs_attribute_use == "required" && !self.attributes_set.contains(&attr.name) {
// Missing required attribute, this is an error
return vec![];
}
}
}
return vec![Token::EndOfFile(EndOfFile::new())];
}
} else if c == '>' {
// No element reference, can't validate attributes, just close
return vec![Token::EndOfFile(EndOfFile::new())];
}
if c.is_whitespace() {
vec![Token::ElementSelfClose(self)]
} else {
vec![]
}
}
}
#[cfg(test)]
mod element_self_close_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_element_with_required_attributes() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "delete".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: Some(vec![
schema::XsAttribute {
name: "id".to_string(),
xs_attribute_type: "xs:string".to_string(),
xs_attribute_use: "required".to_string(),
}
]),
xs_sequence: None,
},
};
Arc::new(element)
}
#[test]
fn test_self_close_with_greater_than() {
let token = ElementSelfClose::new();
// Test closing with >
let next_tokens = token.append('>');
assert!(!next_tokens.is_empty(), "Should accept '>' to close self-closing tag");
assert_eq!(next_tokens.len(), 1, "Should have exactly one token after '>'");
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_self_close_with_whitespace() {
let token = ElementSelfClose::new();
// Test with whitespace
let next_tokens = token.append(' ');
assert!(!next_tokens.is_empty(), "Should accept whitespace in self-closing tag");
match &next_tokens[0] {
Token::ElementSelfClose(_) => (),
_ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_self_close_with_invalid_char() {
let token = ElementSelfClose::new();
// Test with invalid character
let next_tokens = token.append('a');
assert!(next_tokens.is_empty(), "Should reject invalid character in self-closing tag");
}
#[test]
fn test_required_attributes_validation() {
let element = create_element_with_required_attributes();
// Without required attributes
let token = ElementSelfClose::new_with_attributes(Arc::clone(&element), vec![]);
let next_tokens = token.append('>');
assert!(next_tokens.is_empty(), "Should reject closing without required attributes");
// With required attributes
let token = ElementSelfClose::new_with_attributes(
Arc::clone(&element),
vec!["id".to_string()]
);
let next_tokens = token.append('>');
assert!(!next_tokens.is_empty(), "Should accept closing with required attributes");
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
}
}
}
use crate::*;
/// Represents a self-closing XML element
#[derive(Clone, Debug)]
pub struct ElementSelfClose {}
impl ElementSelfClose {
/// Create a new self-closing element
pub fn new() -> Self {
Self {}
}
pub fn append(self, c: char) -> Vec<Token> {
if c == '>' {
return vec![Token::EndOfFile(EndOfFile::new())];
} else {
vec![]
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_valid() {
let values = ["<stop/>"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::EndOfFile(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_invalid() {
let values = ["<stop/?", "<stop/a", "<stop/ "];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
}

59
lib/xml_schema_validator/src/token/end_of_file.rs
View File

@@ -1,30 +1,29 @@
use crate::*;
/// Represents the final state at the end of an XML file
#[derive(Clone, Debug)]
pub struct EndOfFile {
}
impl EndOfFile {
pub fn new() -> Self {
Self {}
}
pub fn append(self, _c: char) -> Vec<Token> {
vec![]
}
}
#[cfg(test)]
mod end_of_file_tests {
use super::*;
#[test]
fn test_end_of_file_any_char() {
let token = EndOfFile::new();
// Test with any character (should reject)
let next_tokens = token.append('a');
assert!(next_tokens.is_empty(), "Should reject any character at EOF");
}
}
use crate::*;
/// Represents the final state at the end of an XML file
#[derive(Clone, Debug)]
pub struct EndOfFile {}
impl EndOfFile {
pub fn new() -> Self {
Self {}
}
pub fn append(self, _c: char) -> Vec<Token> {
vec![]
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_end_of_file_any_char() {
let token = EndOfFile::new();
// Test with any character (should reject)
let next_tokens = token.append('a');
assert!(next_tokens.is_empty(), "Should reject any character at EOF");
}
}

577
lib/xml_schema_validator/src/token/mod.rs
View File

@@ -1,292 +1,285 @@
mod element_close_name;
mod element_close_start;
mod element_open_name;
mod element_open_end;
mod element_open_start;
mod end_of_file;
mod start_of_file;
mod text_content;
mod attribute_name;
mod attribute_value;
mod element_self_close;
pub use element_close_name::ElementCloseName;
pub use element_close_start::ElementCloseStart;
pub use element_open_name::ElementOpenName;
pub use element_open_end::ElementOpenEnd;
pub use element_open_start::ElementOpenStart;
pub use end_of_file::EndOfFile;
pub use start_of_file::StartOfFile;
pub use text_content::TextContent;
pub use attribute_name::AttributeName;
pub use attribute_value::AttributeValue;
pub use element_self_close::ElementSelfClose;
/// Represents the different states of XML token parsing
#[derive(Clone, Debug)]
pub enum Token {
ElementCloseName(ElementCloseName),
ElementCloseStart(ElementCloseStart),
ElementOpenName(ElementOpenName),
ElementOpenEnd(ElementOpenEnd),
ElementOpenStart(ElementOpenStart),
EndOfFile(EndOfFile),
StartOfFile(StartOfFile),
TextContent(TextContent),
AttributeName(AttributeName),
AttributeValue(AttributeValue),
ElementSelfClose(ElementSelfClose),
}
impl Token {
pub fn append(self, c: char) -> Vec<Token> {
match self {
Token::ElementCloseName(element_close_name) => element_close_name.append(c),
Token::ElementCloseStart(element_close_start) => element_close_start.append(c),
Token::ElementOpenName(element_name) => element_name.append(c),
Token::ElementOpenEnd(element_open) => element_open.append(c),
Token::ElementOpenStart(element_start) => element_start.append(c),
Token::EndOfFile(end_of_file) => end_of_file.append(c),
Token::StartOfFile(start_of_file) => start_of_file.append(c),
Token::TextContent(text_content) => text_content.append(c),
Token::AttributeName(attribute_name) => attribute_name.append(c),
Token::AttributeValue(attribute_value) => attribute_value.append(c),
Token::ElementSelfClose(element_self_close) => element_self_close.append(c),
}
}
pub fn is_eof(&self) -> bool {
match self {
Token::EndOfFile(_) => true,
_ => false,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_start_of_file_transition() {
let element = create_test_element();
let token = Token::StartOfFile(StartOfFile::new(Arc::clone(&element)));
// Should only accept '<'
let next_tokens = token.append('<');
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementOpenStart(_) => (),
_ => panic!("Expected ElementStart, got {:?}", next_tokens[0]),
}
// Should reject other characters
let token = Token::StartOfFile(StartOfFile::new(Arc::clone(&element)));
let next_tokens = token.append('a');
assert_eq!(next_tokens.len(), 0, "Should reject 'a'");
}
#[test]
fn test_element_start_transition() {
let element = create_test_element();
let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
// Should accept first char of element name
let next_tokens = token.append('r');
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementOpenName(_) => (),
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
}
// Should reject invalid char
let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
let next_tokens = token.append('x');
assert_eq!(next_tokens.len(), 0, "Should reject 'x'");
}
#[test]
fn test_element_name_transition() {
let element = create_test_element();
let token = Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()).unwrap());
// Continue building name
let next_tokens = token.append('e');
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementOpenName(_) => (),
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
}
// Build complete name and end with >
// Remove the unused buffer
let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)))];
for c in "reasoning>".chars() {
let mut new_tokens = Vec::new();
for token in current_tokens {
new_tokens.append(&mut token.append(c));
}
current_tokens = new_tokens;
assert!(!current_tokens.is_empty(),
"Should have valid transition after appending '{}'", c);
}
// Check final state
assert_eq!(current_tokens.len(), 1, "Should have one final transition");
match &current_tokens[0] {
Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", current_tokens[0]),
}
}
#[test]
fn test_element_open_end_transition() {
let element = create_test_element();
let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
// Should accept text content in mixed element
let next_tokens = token.append('t');
assert_eq!(next_tokens.len(), 1, "Should accept text");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
// Should accept closing tag start
let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
let next_tokens = token.append('<');
assert_eq!(next_tokens.len(), 1, "Should accept '<'");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_text_content_transition() {
let element = create_test_element();
let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
// Should accept more text
let next_tokens = token.append('a');
assert_eq!(next_tokens.len(), 1, "Should accept more text");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
// Should accept closing tag start
let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
let next_tokens = token.append('<');
assert_eq!(next_tokens.len(), 1, "Should accept '<'");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_element_close_start_transition() {
let element = create_test_element();
let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
// Should accept /
let next_tokens = token.append('/');
assert_eq!(next_tokens.len(), 1, "Should accept '/'");
let token = next_tokens[0].clone();
match token {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart with slash, got {:?}", token),
}
// After /, should accept first char of element name
let next_tokens = token.append('r');
assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
match &next_tokens[0] {
Token::ElementCloseName(_) => (),
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_element_close_name_transition() {
let element = create_test_element();
let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
// Add / first
let next_tokens = token.append('/');
assert_eq!(next_tokens.len(), 1, "Should accept '/'");
// Start building name - clone the token to avoid moving out of the vector
let next_tokens = next_tokens[0].clone().append('r');
assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
// Continue with each character
let mut current_tokens = next_tokens;
for c in "easoning".chars() {
let mut new_tokens = Vec::new();
for token in current_tokens {
new_tokens.append(&mut token.append(c));
}
current_tokens = new_tokens;
assert!(!current_tokens.is_empty(),
"Should have valid transition after appending '{}'", c);
}
// Final > to close - clone the token to avoid moving out of the vector
let next_tokens = current_tokens[0].clone().append('>');
assert_eq!(next_tokens.len(), 1, "Should accept '>'");
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_end_of_file_properties() {
let token = Token::EndOfFile(EndOfFile::new());
// Should be identified as EOF
assert!(token.is_eof(), "EndOfFile should be recognized as EOF");
// Should not accept any more input
let next_tokens = token.append('a');
assert_eq!(next_tokens.len(), 0, "Should not accept any more input");
}
}
mod attribute_equals;
mod attribute_name;
mod attribute_value;
mod element_close_name;
mod element_close_start;
mod element_open_end;
mod element_open_name;
mod element_open_start;
mod element_self_close;
mod end_of_file;
mod start_of_file;
mod text_content;
pub use attribute_equals::AttributeEquals;
pub use attribute_name::AttributeName;
pub use attribute_value::AttributeValue;
pub use element_close_name::ElementCloseName;
pub use element_close_start::ElementCloseStart;
pub use element_open_end::ElementOpenEnd;
pub use element_open_name::ElementOpenName;
pub use element_open_start::ElementOpenStart;
pub use element_self_close::ElementSelfClose;
pub use end_of_file::EndOfFile;
pub use start_of_file::StartOfFile;
pub use text_content::TextContent;
/// Represents the different states of XML token parsing
#[derive(Clone, Debug)]
pub enum Token {
AttributeEquals(AttributeEquals),
AttributeName(AttributeName),
AttributeValue(AttributeValue),
ElementCloseName(ElementCloseName),
ElementCloseStart(ElementCloseStart),
ElementOpenEnd(ElementOpenEnd),
ElementOpenName(ElementOpenName),
ElementOpenStart(ElementOpenStart),
ElementSelfClose(ElementSelfClose),
EndOfFile(EndOfFile),
StartOfFile(StartOfFile),
TextContent(TextContent),
}
impl Token {
pub fn append(self, c: char) -> Vec<Token> {
match self {
Token::AttributeEquals(attribute_equals) => attribute_equals.append(c),
Token::ElementCloseName(element_close_name) => element_close_name.append(c),
Token::ElementCloseStart(element_close_start) => element_close_start.append(c),
Token::ElementOpenName(element_name) => element_name.append(c),
Token::ElementOpenEnd(element_open) => element_open.append(c),
Token::ElementOpenStart(element_start) => element_start.append(c),
Token::EndOfFile(end_of_file) => end_of_file.append(c),
Token::StartOfFile(start_of_file) => start_of_file.append(c),
Token::TextContent(text_content) => text_content.append(c),
Token::AttributeName(attribute_name) => attribute_name.append(c),
Token::AttributeValue(attribute_value) => attribute_value.append(c),
Token::ElementSelfClose(element_self_close) => element_self_close.append(c),
}
}
pub fn is_eof(&self) -> bool {
match self {
Token::EndOfFile(_) => true,
_ => false,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::schema;
use std::sync::Arc;
pub fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_element_start_transition() {
let element = create_test_element();
let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
// Should accept first char of element name
let next_tokens = token.append('r');
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementOpenName(_) => (),
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
}
// Should reject invalid char
let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
let next_tokens = token.append('x');
assert_eq!(next_tokens.len(), 0, "Should reject 'x'");
}
#[test]
fn test_element_name_transition() {
let element = create_test_element();
let token =
Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()));
// Continue building name
let next_tokens = token.append('e');
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementOpenName(_) => (),
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
}
// Build complete name and end with >
// Remove the unused buffer
let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(
&element,
)))];
for c in "reasoning>".chars() {
let mut new_tokens = Vec::new();
for token in current_tokens {
new_tokens.append(&mut token.append(c));
}
current_tokens = new_tokens;
assert!(
!current_tokens.is_empty(),
"Should have valid transition after appending '{}'",
c
);
}
// Check final state
assert_eq!(current_tokens.len(), 1, "Should have one final transition");
match &current_tokens[0] {
Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", current_tokens[0]),
}
}
#[test]
fn test_element_open_end_transition() {
let element = create_test_element();
let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
// Should accept text content in mixed element
let next_tokens = token.append('t');
assert_eq!(next_tokens.len(), 1, "Should accept text");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
// Should accept closing tag start
let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
let next_tokens = token.append('<');
assert_eq!(next_tokens.len(), 1, "Should accept '<'");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_text_content_transition() {
let element = create_test_element();
let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
// Should accept more text
let next_tokens = token.append('a');
assert_eq!(next_tokens.len(), 1, "Should accept more text");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
// Should accept closing tag start
let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
let next_tokens = token.append('<');
assert_eq!(next_tokens.len(), 1, "Should accept '<'");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_element_close_start_transition() {
let element = create_test_element();
let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
// Should accept /
let next_tokens = token.append('/');
assert_eq!(next_tokens.len(), 1, "Should accept '/'");
let token = next_tokens[0].clone();
match token {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart with slash, got {:?}", token),
}
// After /, should accept first char of element name
let next_tokens = token.append('r');
assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
match &next_tokens[0] {
Token::ElementCloseName(_) => (),
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_element_close_name_transition() {
let element = create_test_element();
let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
// Add / first
let next_tokens = token.append('/');
assert_eq!(next_tokens.len(), 1, "Should accept '/'");
// Start building name - clone the token to avoid moving out of the vector
let next_tokens = next_tokens[0].clone().append('r');
assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
// Continue with each character
let mut current_tokens = next_tokens;
for c in "easoning".chars() {
let mut new_tokens = Vec::new();
for token in current_tokens {
new_tokens.append(&mut token.append(c));
}
current_tokens = new_tokens;
assert!(
!current_tokens.is_empty(),
"Should have valid transition after appending '{}'",
c
);
}
// Final > to close - clone the token to avoid moving out of the vector
let next_tokens = current_tokens[0].clone().append('>');
assert_eq!(next_tokens.len(), 1, "Should accept '>'");
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_end_of_file_properties() {
let token = Token::EndOfFile(EndOfFile::new());
// Should be identified as EOF
assert!(token.is_eof(), "EndOfFile should be recognized as EOF");
// Should not accept any more input
let next_tokens = token.append('a');
assert_eq!(next_tokens.len(), 0, "Should not accept any more input");
}
}

127
lib/xml_schema_validator/src/token/start_of_file.rs
View File

@@ -1,75 +1,52 @@
use std::sync::Arc;
use crate::*;
/// Represents the initial state at the start of an XML file, containing a reference to the root element schema
#[derive(Clone, Debug)]
pub struct StartOfFile {
element: Arc<schema::XsElement>,
}
impl StartOfFile {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self { element }
}
pub fn append(self, c: char) -> Vec<Token> {
if '<' == c {
vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&self.element)))]
} else {
vec![]
}
}
}
#[cfg(test)]
mod start_of_file_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_start_of_file_less_than() {
let element = create_test_element();
let token = StartOfFile::new(Arc::clone(&element));
// Test with <
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should accept '<'");
match &next_tokens[0] {
Token::ElementOpenStart(_) => (),
_ => panic!("Expected ElementStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_start_of_file_invalid() {
let element = create_test_element();
let token = StartOfFile::new(Arc::clone(&element));
// Test with invalid character
let next_tokens = token.append('a');
assert!(next_tokens.is_empty(), "Should reject invalid character");
}
}
use crate::*;
use std::sync::Arc;
/// Represents the initial state at the start of an XML file, containing a reference to the root element schema
#[derive(Clone, Debug)]
pub struct StartOfFile {
element: Arc<schema::XsElement>,
}
impl StartOfFile {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self { element }
}
pub fn append(self, c: char) -> Vec<Token> {
if '<' == c {
vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(
&self.element,
)))]
} else {
vec![]
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_accept_open_start_token() {
let mut validator = crate::tests::example_validator();
validator.append("<").unwrap();
assert!(!validator.current_tokens.is_empty());
assert!(validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementOpenStart(_)))
.next()
.is_some());
}
#[test]
fn test_not_accept_other() {
let values = ["a", " ", "\t", ">", "/"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
}

168
lib/xml_schema_validator/src/token/text_content.rs
View File

@@ -1,84 +1,84 @@
use std::sync::Arc;
use crate::*;
/// Represents the text content within an XML element
#[derive(Clone, Debug)]
pub struct TextContent {
element: Arc<schema::XsElement>,
}
impl TextContent {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
}
}
pub fn append(self, c: char) -> Vec<Token> {
if c == '<' {
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
} else {
vec![Token::TextContent(TextContent::new(
Arc::clone(&self.element),
))]
}
}
}
#[cfg(test)]
mod text_content_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_text_content_normal_char() {
let element = create_test_element();
let token = TextContent::new(Arc::clone(&element));
// Test with normal character
let next_tokens = token.append('a');
assert!(!next_tokens.is_empty(), "Should accept normal character");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_text_content_less_than() {
let element = create_test_element();
let token = TextContent::new(Arc::clone(&element));
// Test with < (start of tag)
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should accept '<'");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
}
use crate::*;
use std::sync::Arc;
/// Represents the text content within an XML element
#[derive(Clone, Debug)]
pub struct TextContent {
element: Arc<schema::XsElement>,
}
impl TextContent {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self { element }
}
pub fn append(self, c: char) -> Vec<Token> {
if c == '<' {
vec![Token::ElementCloseStart(ElementCloseStart::new(
Arc::clone(&self.element),
))]
} else {
vec![Token::TextContent(TextContent::new(Arc::clone(
&self.element,
)))]
}
}
}
#[cfg(test)]
mod text_content_tests {
use super::*;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_text_content_normal_char() {
let element = create_test_element();
let token = TextContent::new(Arc::clone(&element));
// Test with normal character
let next_tokens = token.append('a');
assert!(!next_tokens.is_empty(), "Should accept normal character");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_text_content_less_than() {
let element = create_test_element();
let token = TextContent::new(Arc::clone(&element));
// Test with < (start of tag)
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should accept '<'");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
}

64
lib/xml_schema_validator/tests/test_logits_processor.py
View File

@@ -1,13 +1,12 @@
from transformers import AutoTokenizer
from xml_schema_validator import XmlLogitsProcessor
import os
import time
import torch
model_path = "../../model"
api_token = os.environ["SIA_HF_API_KEY"]
xml_schema_actions = open("../../action_schema.xsd").read()
xml_schema_actions = open("example_schema.xsd").read()
xml_schema_only_root_node = """<?xml version="1.0" encoding="UTF-8"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
<xs:element name="root">
@@ -59,60 +58,27 @@ def test_token_masking():
tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token)
processor = XmlLogitsProcessor(tokenizer, xml_schema_actions)
# Process empty input to set prompt length
input_ids = torch.tensor([tokenizer.encode("")])
scores = torch.ones((1, len(tokenizer))) # All tokens have equal probability
processed_scores = processor(input_ids, scores)
# Create dummy input_ids and scores
# Process valid continuation
input_ids = torch.tensor([tokenizer.encode("<reasoning>")])
scores = torch.ones((1, len(tokenizer))) # All tokens have equal probability
# Process the scores
processed_scores = processor(input_ids, scores)
# Verify that valid continuations still have positive scores
assert processed_scores[0, tokenizer.encode(" ", add_special_tokens=False)[0]] > -float('inf')
space_token = tokenizer.encode(" ", add_special_tokens=False)[0]
assert processed_scores[0, space_token] > -float('inf')
# Verify that invalid continuations are masked
assert processed_scores[0, tokenizer.encode("<invalid>", add_special_tokens=False)[0]] == -float('inf')
def test_performance():
"""Test performance with larger XML documents"""
tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token)
processor = XmlLogitsProcessor(tokenizer, xml_schema_actions)
# Generate a larger XML document
large_xml = "<reasoning>" + "Test content. " * 100 + "</reasoning>"
# Measure time to process
start_time = time.time()
processor.core.append(large_xml)
processing_time = time.time() - start_time
print(f"Processing time for large XML: {processing_time:.4f} seconds")
# You might want to assert that processing time is below a threshold
def test_subword_tokens():
"""Test handling of subword tokens that might split XML tags"""
tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token)
schema_text = open("../../action_schema.xsd").read()
processor = XmlLogitsProcessor(tokenizer, schema_text)
# Find some XML tag that gets split into multiple tokens by your tokenizer
tag = "<reasoning>"
tokens = tokenizer.encode(tag, add_special_tokens=False)
if len(tokens) > 1:
print(f"Tag '{tag}' is split into {len(tokens)} tokens")
# Test that the processor can handle these split tokens
import torch
input_ids = torch.tensor([[tokens[0]]])
scores = torch.ones((1, len(tokenizer)))
processed_scores = processor(input_ids, scores)
# The next token in the sequence should have a high score
assert processed_scores[0, tokens[1]] > -float('inf')
input_ids = torch.tensor([tokenizer.encode("</invalid>")])
scores = torch.ones((1, len(tokenizer))) # All tokens have equal probability
processed_scores = processor(input_ids, scores)
assert processed_scores[0, tokenizer.eos_token_id] == 1
assert processed_scores[0, space_token] == -float('inf')
if __name__ == "__main__":
# Run individual tests for debugging
@@ -120,5 +86,3 @@ if __name__ == "__main__":
test_xml_schema_parsing()
test_basic_xml_validation()
test_token_masking()
test_performance()
test_subword_tokens()