llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed attribute parsing

Browse Source
This commit is contained in:
Geens
2025-04-12 18:02:55 +02:00
parent bd3adf78e6
commit 9564879edc
23 changed files with 2905 additions and 2786 deletions
Download Patch File Download Diff File Expand all files Collapse all files

219
lib/xml_schema_validator/src/char_trie.rs
View File

@@ -1,103 +1,116 @@
use crate::*;
use pyo3::prelude::*;
// Node in our character trie
struct CharTrieNode {
// Store character and node pairs, keep sorted for binary search
children: Vec<(char, CharTrieNode)>,
// Store token IDs that are valid at this point
token_id: Option<Py<PyAny>>,
}
impl CharTrieNode {
fn new() -> Self {
Self {
children: Vec::new(),
token_id: None,
}
}
// Add a child for character c, maintaining sorted order
fn add_child(&mut self, c: char) -> &mut CharTrieNode {
match self.children.binary_search_by_key(&c, |(ch, _)| *ch) {
Ok(index) => &mut self.children[index].1,
Err(index) => {
// Insert at the right position to maintain sort order
self.children.insert(index, (c, CharTrieNode::new()));
&mut self.children[index].1
}
}
}
// Add a token id to this node
fn add_token(&mut self, token_id: Py<PyAny>) {
self.token_id = Some(token_id);
}
pub fn clone_ref(&self, py: Python<'_>) -> Self {
Self {
children: self.children.iter().map(|(ch, node)| (*ch, node.clone_ref(py))).collect(),
token_id: self.token_id.iter().map(|token_id| token_id.clone_ref(py)).nth(0),
}
}
}
// The main trie structure
pub struct CharTrie {
root: CharTrieNode,
}
impl CharTrie {
pub fn new() -> Self {
Self {
root: CharTrieNode::new(),
}
}
// Insert a token's character sequence into the trie
pub fn insert(&mut self, token_text: &str, token_id: Py<PyAny>) {
let mut current = &mut self.root;
// Add each character in the token text
for c in token_text.chars() {
current = current.add_child(c);
}
// Mark this node as representing token_id
current.add_token(token_id);
}
// Find all tokens that could be invalid given the current XML state
pub fn find_invalid_tokens(&self, py: Python<'_>, xml_validator: &XmlSchemaValidator, token_map: &Vec<(Py<PyAny>, String)>) -> Vec<Py<PyAny>> {
let mut invalid_tokens = Vec::new();
for (token_id, token_text) in token_map {
// Check if this token would make the XML invalid
if !self.is_valid_continuation(xml_validator, token_text) {
invalid_tokens.push(token_id.clone_ref(py));
}
}
invalid_tokens
}
// Check if a token would be a valid continuation
fn is_valid_continuation(&self, xml_validator: &XmlSchemaValidator, token_text: &str) -> bool {
// First, quick check with just the first character
if let Some(first_char) = token_text.chars().next() {
if !xml_validator.can_continue_with(first_char) {
return false;
}
}
// If first character is valid, try the full token
let mut validator_clone = xml_validator.clone();
validator_clone.append(token_text).is_ok()
}
pub fn clone_ref(&self, py: Python<'_>) -> Self {
Self {
root: self.root.clone_ref(py),
}
}
}
use crate::*;
use pyo3::prelude::*;
// Node in our character trie
struct CharTrieNode {
// Store character and node pairs, keep sorted for binary search
children: Vec<(char, CharTrieNode)>,
// Store token IDs that are valid at this point
token_id: Option<Py<PyAny>>,
}
impl CharTrieNode {
fn new() -> Self {
Self {
children: Vec::new(),
token_id: None,
}
}
// Add a child for character c, maintaining sorted order
fn add_child(&mut self, c: char) -> &mut CharTrieNode {
match self.children.binary_search_by_key(&c, |(ch, _)| *ch) {
Ok(index) => &mut self.children[index].1,
Err(index) => {
// Insert at the right position to maintain sort order
self.children.insert(index, (c, CharTrieNode::new()));
&mut self.children[index].1
}
}
}
// Add a token id to this node
fn add_token(&mut self, token_id: Py<PyAny>) {
self.token_id = Some(token_id);
}
pub fn clone_ref(&self, py: Python<'_>) -> Self {
Self {
children: self
.children
.iter()
.map(|(ch, node)| (*ch, node.clone_ref(py)))
.collect(),
token_id: self
.token_id
.iter()
.map(|token_id| token_id.clone_ref(py))
.nth(0),
}
}
}
// The main trie structure
pub struct CharTrie {
root: CharTrieNode,
}
impl CharTrie {
pub fn new() -> Self {
Self {
root: CharTrieNode::new(),
}
}
// Insert a token's character sequence into the trie
pub fn insert(&mut self, token_text: &str, token_id: Py<PyAny>) {
let mut current = &mut self.root;
// Add each character in the token text
for c in token_text.chars() {
current = current.add_child(c);
}
// Mark this node as representing token_id
current.add_token(token_id);
}
// Find all tokens that could be invalid given the current XML state
pub fn find_invalid_tokens(
&self,
py: Python<'_>,
xml_validator: &XmlSchemaValidator,
token_map: &Vec<(Py<PyAny>, String)>,
) -> Vec<Py<PyAny>> {
let mut invalid_tokens = Vec::new();
for (token_id, token_text) in token_map {
// Check if this token would make the XML invalid
if !self.is_valid_continuation(xml_validator, token_text) {
invalid_tokens.push(token_id.clone_ref(py));
}
}
invalid_tokens
}
// Check if a token would be a valid continuation
fn is_valid_continuation(&self, xml_validator: &XmlSchemaValidator, token_text: &str) -> bool {
// First, quick check with just the first character
if let Some(first_char) = token_text.chars().next() {
if !xml_validator.can_continue_with(first_char) {
return false;
}
}
// If first character is valid, try the full token
let mut validator_clone = xml_validator.clone();
validator_clone.append(token_text).is_ok()
}
pub fn clone_ref(&self, py: Python<'_>) -> Self {
Self {
root: self.root.clone_ref(py),
}
}
}

28
lib/xml_schema_validator/src/error.rs
View File

@@ -1,14 +1,14 @@
/// Error types that can occur during XML validation
#[derive(Clone, Debug, thiserror::Error)]
pub enum Error {
/// Indicates that the provided XSD schema is invalid
#[error("Invalid schema: {0}")]
InvalidSchema(#[from] quick_xml::DeError),
/// Indicates that the provided XML fragment is invalid
#[error("Invalid XML: {0}")]
InvalidXml(String),
/// Not implemented yet
#[error("Not implemented yet")]
NotImplemented,
}
/// Error types that can occur during XML validation
#[derive(Clone, Debug, thiserror::Error)]
pub enum Error {
/// Indicates that the provided XSD schema is invalid
#[error("Invalid schema: {0}")]
InvalidSchema(#[from] quick_xml::DeError),
/// Indicates that the provided XML fragment is invalid
#[error("Invalid XML: {0}")]
InvalidXml(String),
/// Not implemented yet
#[error("Not implemented yet")]
NotImplemented,
}

1358
lib/xml_schema_validator/src/lib.rs
View File

File diff suppressed because it is too large Load Diff

216
lib/xml_schema_validator/src/py_xml_logits_processor_core.rs
View File

@@ -1,6 +1,6 @@
use crate::*;
use pyo3::prelude::*;
use pyo3::types::PyDict;
use crate::*;
/// A minimal LogitsProcessor that enforces valid XML according to a schema.
#[pyclass]
@@ -18,7 +18,10 @@ impl XmlLogitsProcessorCore {
Ok(xml_schema_validator) => xml_schema_validator,
Err(e) => return Err(pyo3::exceptions::PyValueError::new_err(e.to_string())),
};
let tokens: Vec<(Py<PyAny>, String)> = tokens.iter().map(|(id, value)| (id.unbind(), value.unbind().to_string())).collect();
let tokens: Vec<(Py<PyAny>, String)> = tokens
.iter()
.map(|(id, value)| (id.unbind(), value.unbind().to_string()))
.collect();
let mut trie = char_trie::CharTrie::new();
for (token_id, token_text) in tokens.iter() {
trie.insert(token_text, token_id.clone_ref(py));
@@ -29,7 +32,7 @@ impl XmlLogitsProcessorCore {
trie,
})
}
/// Append a fragment of XML to the validator
/// Returns true if the append was successful, false otherwise
fn append(&mut self, fragment: &str) -> bool {
@@ -38,7 +41,9 @@ impl XmlLogitsProcessorCore {
/// Get a list of tokens that would lead to invalid XML
fn get_invalid_tokens(&mut self, py: Python<'_>) -> PyResult<Vec<PyObject>> {
Ok(self.trie.find_invalid_tokens(py, &self.xml_schema_validator, &self.tokens))
Ok(self
.trie
.find_invalid_tokens(py, &self.xml_schema_validator, &self.tokens))
}
/// Check if the validator has reached the end of the XML
@@ -50,16 +55,20 @@ impl XmlLogitsProcessorCore {
fn copy(&self, py: Python<'_>) -> PyResult<Self> {
Ok(Self {
xml_schema_validator: self.xml_schema_validator.clone(),
tokens: self.tokens.iter().map(|(a, b)|(a.clone_ref(py), b.clone())).collect(),
tokens: self
.tokens
.iter()
.map(|(a, b)| (a.clone_ref(py), b.clone()))
.collect(),
trie: self.trie.clone_ref(py),
})
}
/// Get a list of all element names in the schema
fn get_element_names(&self) -> PyResult<Vec<String>> {
Ok(self.xml_schema_validator.get_element_names())
}
/// Check if a specific character is valid as the next character
fn can_continue_with(&self, c: &str) -> PyResult<bool> {
if let Some(first_char) = c.chars().next() {
@@ -79,8 +88,8 @@ pub fn register_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()>
#[cfg(test)]
mod tests {
use super::*;
use pyo3::Python;
use pyo3::types::PyDict;
use pyo3::Python;
fn create_test_processor<'py>(py: Python<'py>) -> PyResult<XmlLogitsProcessorCore> {
let tokens = PyDict::new(py);
@@ -91,7 +100,7 @@ mod tests {
tokens.set_item(3, " ")?;
tokens.set_item(4, "a")?;
tokens.set_item(5, "</reasoning>")?;
// Simple schema with only reasoning element
let schema_text = r#"<?xml version="1.0" encoding="UTF-8"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
@@ -103,7 +112,7 @@ mod tests {
</xs:complexType>
</xs:element>
</xs:schema>"#;
XmlLogitsProcessorCore::new(py, tokens.into(), schema_text)
}
@@ -111,24 +120,29 @@ mod tests {
fn test_append_valid_xml() {
Python::with_gil(|py| {
let mut processor = create_test_processor(py).unwrap();
// Test various valid XML fragments
assert!(processor.append("<reasoning>"), "Should accept opening tag");
let mut processor = create_test_processor(py).unwrap();
assert!(processor.append("<reasoning>test</reasoning>"),
"Should accept complete element with content");
assert!(
processor.append("<reasoning>test</reasoning>"),
"Should accept complete element with content"
);
// Test with real schema from disk if available
if let Ok(schema_text) = std::fs::read_to_string("../../action_schema.xsd") {
let tokens = PyDict::new(py);
tokens.set_item(0, "<").unwrap();
tokens.set_item(1, "reasoning").unwrap();
tokens.set_item(2, ">").unwrap();
let mut processor = XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap();
assert!(processor.append("<reasoning>"),
"Should accept <reasoning> with actual schema");
let mut processor =
XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap();
assert!(
processor.append("<reasoning>"),
"Should accept <reasoning> with actual schema"
);
}
});
}
@@ -137,13 +151,18 @@ mod tests {
fn test_append_invalid_xml() {
Python::with_gil(|py| {
let mut processor = create_test_processor(py).unwrap();
// Test invalid XML fragments
assert!(!processor.append("<invalid>"), "Should reject invalid element");
assert!(
!processor.append("<invalid>"),
"Should reject invalid element"
);
let mut processor = create_test_processor(py).unwrap();
assert!(!processor.append("<reasoning></invalid>"),
"Should reject mismatched tags");
assert!(
!processor.append("<reasoning></invalid>"),
"Should reject mismatched tags"
);
});
}
@@ -151,25 +170,36 @@ mod tests {
fn test_eof_detection() {
Python::with_gil(|py| {
let processor = create_test_processor(py).unwrap();
// At start, not at EOF
assert!(!processor.eof().unwrap(), "Should not be at EOF at start");
// After complete document, should be at EOF
let mut processor = create_test_processor(py).unwrap();
// Use a properly formed XML document that matches the schema
assert!(processor.append("<reasoning>test</reasoning>"),
"Should accept complete document");
assert!(
processor.append("<reasoning>test</reasoning>"),
"Should accept complete document"
);
// DEBUG: Print the token state to see why EOF detection fails
println!("Current tokens: {:?}", processor.xml_schema_validator.current_tokens);
assert!(processor.eof().unwrap(), "Should be at EOF after complete document");
println!(
"Current tokens: {:?}",
processor.xml_schema_validator.current_tokens
);
assert!(
processor.eof().unwrap(),
"Should be at EOF after complete document"
);
// After opening tag, should not be at EOF
let mut processor = create_test_processor(py).unwrap();
assert!(processor.append("<reasoning>"), "Should accept opening tag");
assert!(!processor.eof().unwrap(), "Should not be at EOF after opening tag");
assert!(
!processor.eof().unwrap(),
"Should not be at EOF after opening tag"
);
});
}
@@ -177,20 +207,30 @@ mod tests {
fn test_can_continue_with() {
Python::with_gil(|py| {
let processor = create_test_processor(py).unwrap();
// At start, only '<' is valid
assert!(processor.can_continue_with("<").unwrap(), "Should accept '<' at start");
assert!(!processor.can_continue_with("a").unwrap(), "Should reject 'a' at start");
assert!(
processor.can_continue_with("<").unwrap(),
"Should accept '<' at start"
);
assert!(
!processor.can_continue_with("a").unwrap(),
"Should reject 'a' at start"
);
// Test after opening tag
let mut processor = create_test_processor(py).unwrap();
assert!(processor.append("<reasoning>"), "Should accept opening tag");
// In mixed content, any character should be valid
assert!(processor.can_continue_with("a").unwrap(),
"Should accept 'a' in mixed content");
assert!(processor.can_continue_with("<").unwrap(),
"Should accept '<' in mixed content");
assert!(
processor.can_continue_with("a").unwrap(),
"Should accept 'a' in mixed content"
);
assert!(
processor.can_continue_with("<").unwrap(),
"Should accept '<' in mixed content"
);
});
}
@@ -198,14 +238,17 @@ mod tests {
fn test_copy() {
Python::with_gil(|py| {
let processor = create_test_processor(py).unwrap();
// Make a copy
let copy = processor.copy(py).unwrap();
// Original and copy should behave the same
assert_eq!(processor.eof().unwrap(), copy.eof().unwrap(),
"Copy should have same EOF state");
assert_eq!(
processor.eof().unwrap(),
copy.eof().unwrap(),
"Copy should have same EOF state"
);
// Modify copy, original should be unchanged
let mut copy = processor.copy(py).unwrap();
assert!(copy.append("<reasoning>"), "Copy should accept XML");
@@ -218,28 +261,32 @@ mod tests {
fn test_get_element_names() {
Python::with_gil(|py| {
let processor = create_test_processor(py).unwrap();
let names = processor.get_element_names().unwrap();
assert!(names.contains(&"reasoning".to_string()),
"Should contain 'reasoning' element");
assert!(
names.contains(&"reasoning".to_string()),
"Should contain 'reasoning' element"
);
assert_eq!(names.len(), 1, "Should have exactly one element");
let tokens = PyDict::new(py);
tokens.set_item(0, "<").unwrap();
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let processor = XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap();
let names = processor.get_element_names().unwrap();
// Print out all elements from schema for debugging
println!("Elements in schema: {:?}", names);
// Check if reasoning exists
assert!(names.contains(&"reasoning".to_string()),
"Real schema should contain 'reasoning'");
assert!(
names.contains(&"reasoning".to_string()),
"Real schema should contain 'reasoning'"
);
});
}
#[test]
fn test_debug_reasoning_tag_validation() {
Python::with_gil(|py| {
@@ -254,7 +301,7 @@ mod tests {
</xs:complexType>
</xs:element>
</xs:schema>"#;
// Create a minimal token dictionary
let tokens = PyDict::new(py);
tokens.set_item(0, "<").unwrap();
@@ -268,56 +315,67 @@ mod tests {
tokens.set_item(8, "n").unwrap();
tokens.set_item(9, "g").unwrap();
tokens.set_item(10, ">").unwrap();
// Create processor with this schema and tokens
let processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
let processor =
XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
// Test basic operations to verify core functionality works
let element_names = processor.get_element_names().unwrap();
println!("Element names in test schema: {:?}", element_names);
assert!(element_names.contains(&"reasoning".to_string()),
"Test schema should contain 'reasoning'");
assert!(
element_names.contains(&"reasoning".to_string()),
"Test schema should contain 'reasoning'"
);
// Now try to append each character of "<reasoning>" one by one
let chars = vec!['<', 'r', 'e', 'a', 's', 'o', 'n', 'i', 'n', 'g', '>'];
let mut full_text = String::new();
let processor = processor.copy(py).unwrap();
for c in chars {
full_text.push(c);
let fragment = c.to_string();
let result = processor.can_continue_with(&fragment).unwrap();
println!("Can continue with '{}' after '{}': {}",
c, &full_text[..full_text.len()-1], result);
println!(
"Can continue with '{}' after '{}': {}",
c,
&full_text[..full_text.len() - 1],
result
);
// Create a new processor to test appending the fragment built so far
let mut fresh_processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
let mut fresh_processor =
XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
let append_result = fresh_processor.append(&full_text);
println!("Appending full text '{}': {}", full_text, append_result);
}
// Now test with actual schema from disk if available
if let Ok(actual_schema) = std::fs::read_to_string("../../action_schema.xsd") {
println!("\nTesting with actual schema from disk:");
let processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap();
let processor =
XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap();
let element_names = processor.get_element_names().unwrap();
println!("Element names in actual schema: {:?}", element_names);
if element_names.contains(&"reasoning".to_string()) {
println!("'reasoning' element found in actual schema");
// Test appending "<reasoning>" to a fresh processor
let mut processor = processor.copy(py).unwrap();
let result = processor.append("<reasoning>");
println!("Appending '<reasoning>' to fresh processor: {}", result);
// Test character by character
let mut text = String::new();
for c in "<reasoning>".chars() {
text.push(c);
let result = XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap()
.append(&text);
let result =
XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema)
.unwrap()
.append(&text);
println!("Appending '{}' to fresh processor: {}", text, result);
}
} else {
@@ -326,4 +384,4 @@ mod tests {
}
});
}
}
}

142
lib/xml_schema_validator/src/py_xml_schema_validator.rs
View File

@@ -1,71 +1,71 @@
use pyo3::prelude::*;
use pyo3::exceptions::PyValueError;
/// Python wrapper for XmlValidator
#[pyclass]
struct XmlSchemaValidator {
validator: crate::XmlSchemaValidator,
}
#[pymethods]
impl XmlSchemaValidator {
#[new]
fn new(schema_text: &str) -> PyResult<Self> {
let validator = crate::XmlSchemaValidator::new(schema_text)
.map_err(|e| PyValueError::new_err(format!("Failed to create validator: {}", e)))?;
Ok(Self { validator })
}
/// Append a fragment of XML to the validator
/// Returns a tuple containing:
/// - A boolean indicating success
/// - An optional error message
fn append(&mut self, fragment: &str) -> PyResult<(bool, Option<String>)> {
match self.validator.append(fragment) {
Ok(()) => PyResult::Ok((true, None)),
Err(e) => PyResult::Ok((false, Some(e.to_string()))),
}
}
/// Check if the validator has reached the end of the XML
fn eof(&self) -> PyResult<bool> {
Ok(self.validator.eof())
}
/// Create a deep copy of this validator
fn copy(&self) -> PyResult<Self> {
Ok(Self {
validator: self.validator.clone(),
})
}
/// Get a list of all element names in the schema
fn get_element_names(&self) -> PyResult<Vec<String>> {
Ok(self.validator.get_element_names())
}
/// Check if a specific character is valid as the next character
fn can_continue_with(&self, c: &str) -> PyResult<bool> {
if let Some(first_char) = c.chars().next() {
Ok(self.validator.can_continue_with(first_char))
} else {
Ok(false)
}
}
/// Validate an entire XML string and check if it's valid according to the schema
fn validate(&self, xml_string: &str) -> PyResult<bool> {
let mut validator_clone = self.validator.clone();
match validator_clone.append(xml_string) {
Ok(()) => Ok(validator_clone.eof()), // Valid only if we reached EOF
Err(_) => Ok(false), // Invalid XML
}
}
}
/// Register the class with the Python module
pub fn register_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<XmlSchemaValidator>()?;
Ok(())
}
use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
/// Python wrapper for XmlValidator
#[pyclass]
struct XmlSchemaValidator {
validator: crate::XmlSchemaValidator,
}
#[pymethods]
impl XmlSchemaValidator {
#[new]
fn new(schema_text: &str) -> PyResult<Self> {
let validator = crate::XmlSchemaValidator::new(schema_text)
.map_err(|e| PyValueError::new_err(format!("Failed to create validator: {}", e)))?;
Ok(Self { validator })
}
/// Append a fragment of XML to the validator
/// Returns a tuple containing:
/// - A boolean indicating success
/// - An optional error message
fn append(&mut self, fragment: &str) -> PyResult<(bool, Option<String>)> {
match self.validator.append(fragment) {
Ok(()) => PyResult::Ok((true, None)),
Err(e) => PyResult::Ok((false, Some(e.to_string()))),
}
}
/// Check if the validator has reached the end of the XML
fn eof(&self) -> PyResult<bool> {
Ok(self.validator.eof())
}
/// Create a deep copy of this validator
fn copy(&self) -> PyResult<Self> {
Ok(Self {
validator: self.validator.clone(),
})
}
/// Get a list of all element names in the schema
fn get_element_names(&self) -> PyResult<Vec<String>> {
Ok(self.validator.get_element_names())
}
/// Check if a specific character is valid as the next character
fn can_continue_with(&self, c: &str) -> PyResult<bool> {
if let Some(first_char) = c.chars().next() {
Ok(self.validator.can_continue_with(first_char))
} else {
Ok(false)
}
}
/// Validate an entire XML string and check if it's valid according to the schema
fn validate(&self, xml_string: &str) -> PyResult<bool> {
let mut validator_clone = self.validator.clone();
match validator_clone.append(xml_string) {
Ok(()) => Ok(validator_clone.eof()), // Valid only if we reached EOF
Err(_) => Ok(false), // Invalid XML
}
}
}
/// Register the class with the Python module
pub fn register_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<XmlSchemaValidator>()?;
Ok(())
}

18
lib/xml_schema_validator/src/python.rs
View File

@@ -1,9 +1,9 @@
use pyo3::prelude::*;
/// Python module for XML Schema validation
#[pymodule]
fn _rs(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
crate::py_xml_logits_processor_core::register_module(py, m)?;
crate::py_xml_schema_validator::register_module(py, m)?;
Ok(())
}
use pyo3::prelude::*;
/// Python module for XML Schema validation
#[pymodule]
fn _rs(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
crate::py_xml_logits_processor_core::register_module(py, m)?;
crate::py_xml_schema_validator::register_module(py, m)?;
Ok(())
}

61
lib/xml_schema_validator/src/schema.rs
View File

@@ -1,6 +1,14 @@
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsSchema {
#[serde(rename = "@xmlns:xs")]
pub xmlns_xs: String,
@@ -12,7 +20,15 @@ pub struct XsSchema {
pub xs_element: Vec<XsElement>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsElement {
#[serde(rename = "@name")]
pub name: String,
@@ -22,7 +38,15 @@ pub struct XsElement {
pub xs_complex_type: XsComplexType,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsComplexType {
#[serde(rename = "@mixed")]
pub mixed: Option<String>,
@@ -34,7 +58,15 @@ pub struct XsComplexType {
pub xs_sequence: Option<XsSequence>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsAttribute {
#[serde(rename = "@name")]
pub name: String,
@@ -44,7 +76,15 @@ pub struct XsAttribute {
pub xs_attribute_use: String,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsSequence {
#[serde(rename = "$text")]
pub text: Option<String>,
@@ -52,7 +92,15 @@ pub struct XsSequence {
pub xs_any: XsAny,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
pub struct XsAny {
#[serde(rename = "@minOccurs")]
pub min_occurs: String,
@@ -61,4 +109,3 @@ pub struct XsAny {
#[serde(rename = "@processContents")]
pub process_contents: String,
}

102
lib/xml_schema_validator/src/token/attribute_equals.rs Normal file
View File

@@ -0,0 +1,102 @@
use crate::*;
use std::sync::Arc;
/// Represents the equals sign after an XML attribute name
#[derive(Clone, Debug)]
pub struct AttributeEquals {
element: Arc<schema::XsElement>,
used_attributes: Vec<schema::XsAttribute>,
}
impl AttributeEquals {
pub fn new(
element: Arc<schema::XsElement>,
used_attributes: Vec<schema::XsAttribute>,
) -> Self {
Self {
element,
used_attributes,
}
}
pub fn append(self, c: char) -> Vec<Token> {
if c == '"' {
// Opening quote starts the attribute value
return vec![Token::AttributeValue(AttributeValue::new(
Arc::clone(&self.element),
self.used_attributes,
))];
} else if c.is_whitespace() {
// Allow whitespace between equals sign and opening quote
return vec![Token::AttributeEquals(self)];
} else {
// Any other character is invalid
return vec![];
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_attribute_equals_quote() {
let values = [
"<delete id=\"",
"<single timeout=\"",
"<single limit=\"",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeValue(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_attribute_equals_with_whitespace() {
let values = [
"<delete id= \"",
"<single timeout = \"",
"<single limit \t = \n \"",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeValue(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_invalid_after_equals() {
let values = [
"<delete id=a", // Only quote should be accepted after equals
"<single timeout=<", // Invalid character after equals
"<single timeout=1", // Unquoted numbers not supported yet
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
}

359
lib/xml_schema_validator/src/token/attribute_name.rs
View File

@@ -1,196 +1,165 @@
use std::sync::Arc;
use crate::*;
/// Represents an attribute name of an XML element
#[derive(Clone, Debug)]
pub struct AttributeName {
/// The element that owns this attribute
element: Arc<schema::XsElement>,
/// Buffer containing the attribute name characters processed so far
buffer: String,
/// Track which attributes have already been set for this element
attributes_set: Vec<String>,
}
impl AttributeName {
/// Create a new AttributeName with an empty buffer
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
buffer: String::new(),
attributes_set: Vec::new(),
}
}
/// Create a new AttributeName with the given buffer
pub fn with_buffer(element: Arc<schema::XsElement>, buffer: String, attributes_set: Vec<String>) -> Self {
Self {
element,
buffer,
attributes_set,
}
}
/// Create a new AttributeName with attributes already set
pub fn with_attributes(element: Arc<schema::XsElement>, attributes_set: Vec<String>) -> Self {
Self {
element,
buffer: String::new(),
attributes_set,
}
}
/// Append a character to the attribute name and return possible continuations
pub fn append(self, c: char) -> Vec<Token> {
if c.is_whitespace() && self.buffer.is_empty() {
return vec![Token::AttributeName(self)];
} else if c == '=' {
// Found the equals sign, transition to attribute value
// We need to verify that this attribute name is valid for this element
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
for attr in attributes {
if attr.name == self.buffer {
// Valid attribute, transition to attribute value
return vec![Token::AttributeValue(AttributeValue::new(
Arc::clone(&self.element),
self.buffer,
self.attributes_set,
))];
}
}
}
// Invalid attribute name
vec![]
} else {
// Continue building the attribute name
let new_buffer = self.buffer + &c.to_string();
// Check if this could be a valid attribute for this element
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
for attr in attributes {
if attr.name.starts_with(&new_buffer) {
// This could be a valid attribute, continue parsing
return vec![Token::AttributeName(AttributeName::with_buffer(
Arc::clone(&self.element),
new_buffer,
self.attributes_set.clone(),
))];
}
}
}
// No matching attribute found
vec![]
}
}
}
#[cfg(test)]
mod attribute_name_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_element_with_attributes() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "delete".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: Some(vec![
schema::XsAttribute {
name: "id".to_string(),
xs_attribute_type: "xs:string".to_string(),
xs_attribute_use: "required".to_string(),
}
]),
xs_sequence: None,
},
};
Arc::new(element)
}
#[test]
fn test_attribute_name_append() {
let element = create_element_with_attributes();
let token = AttributeName::new(Arc::clone(&element));
// Build attribute name character by character
let mut current_token = token;
for c in "id".chars() {
let next_tokens = current_token.append(c);
assert!(!next_tokens.is_empty(), "Should accept attribute name character '{}'", c);
match &next_tokens[0] {
Token::AttributeName(next) => current_token = next.clone(),
_ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
}
}
// Test equals sign
let next_tokens = current_token.append('=');
assert!(!next_tokens.is_empty(), "Should accept '=' after attribute name");
match &next_tokens[0] {
Token::AttributeValue(_) => (),
_ => panic!("Expected AttributeValue, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_attribute_name_whitespace() {
let element = create_element_with_attributes();
let token = AttributeName::new(Arc::clone(&element));
// Test with whitespace
let next_tokens = token.append(' ');
assert!(!next_tokens.is_empty(), "Should accept whitespace in attribute name");
match &next_tokens[0] {
Token::AttributeName(_) => (),
_ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_invalid_attribute_name() {
let element = create_element_with_attributes();
let token = AttributeName::new(Arc::clone(&element));
// Test with invalid attribute name
let next_tokens = token.append('x'); // 'x' doesn't start any valid attribute
assert!(next_tokens.is_empty(), "Should reject invalid attribute name");
}
#[test]
fn test_attribute_tracking() {
let element = create_element_with_attributes();
// Create AttributeName with pre-existing attributes
let attributes_set = vec!["other".to_string()];
let token = AttributeName::with_attributes(Arc::clone(&element), attributes_set.clone());
// Build attribute name and transition to AttributeValue
let mut current_token = token;
for c in "id".chars() {
let next_tokens = current_token.append(c);
match &next_tokens[0] {
Token::AttributeName(next) => current_token = next.clone(),
_ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
}
}
// Test equals sign
let next_tokens = current_token.append('=');
match &next_tokens[0] {
Token::AttributeValue(value) => {
// Verify that pre-existing attributes are preserved
assert!(value.attributes_set.contains(&"other".to_string()),
"Pre-existing attribute should be preserved");
},
_ => panic!("Expected AttributeValue, got {:?}", next_tokens[0]),
}
}
use crate::*;
use std::sync::Arc;
/// Represents an XML attribute name that is in the process of being parsed
#[derive(Clone, Debug)]
pub struct AttributeName {
buffer: String,
element: Arc<schema::XsElement>,
used_attributes: Vec<schema::XsAttribute>,
}
impl AttributeName {
pub fn new(first_char: char, element: Arc<schema::XsElement>) -> Self {
Self {
buffer: first_char.to_string(),
element,
used_attributes: Vec::new(),
}
}
pub fn new_with_used(first_char: char, element: Arc<schema::XsElement>, used_attributes: Vec<schema::XsAttribute>) -> Self {
Self {
buffer: first_char.to_string(),
element,
used_attributes,
}
}
pub fn append(mut self, c: char) -> Vec<Token> {
if c == '=' {
// Check if we've matched a valid attribute name
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
if let Some(attribute) = attributes
.iter()
.find(|attribute| attribute.name == self.buffer)
{
// Check if this attribute has already been used
if self.used_attributes.contains(&attribute) {
return vec![];
}
// Add this attribute to the used set
self.used_attributes.push(attribute.clone());
return vec![Token::AttributeEquals(AttributeEquals::new(
Arc::clone(&self.element),
self.used_attributes,
))];
}
}
return vec![];
} else if c.is_whitespace() {
// Check if we've matched a valid attribute name and allow whitespace after it
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
if attributes
.iter()
.any(|attribute| attribute.name == self.buffer)
{
// Check if this attribute has already been used
if self.used_attributes.iter().filter(|attribute| attribute.name == self.buffer).next().is_some() {
return vec![];
}
return vec![Token::AttributeName(self)];
}
}
return vec![];
} else {
// Add character to buffer and check if we're still building a valid attribute name
let new_buffer = self.buffer.clone() + &c.to_string();
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
// Check if any attribute name starts with our buffer
if attributes
.iter()
.any(|attribute| attribute.name.starts_with(&new_buffer))
{
return vec![Token::AttributeName(AttributeName {
buffer: new_buffer,
element: Arc::clone(&self.element),
used_attributes: self.used_attributes,
})];
}
}
// Character doesn't match what we expect for this attribute name
return vec![];
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_valid_attribute_name() {
let values = [
"<delete id=",
"<single timeout=",
"<single limit=",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeEquals(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_attribute_name_with_whitespace() {
let values = [
"<delete id =",
"<single timeout =",
"<single limit\t=",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeEquals(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_invalid_attribute_name() {
let values = [
"<delete invalid=",
"<single wrongattr=",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
#[test]
fn test_duplicate_attribute() {
let mut validator = crate::tests::example_validator();
// First attribute is valid
validator.append("<delete id=\"123\" ").expect("First attribute should be accepted");
// Try to add the same attribute again
let result = validator.append("id=");
assert!(result.is_err(), "Duplicate attribute should be rejected");
}
}

442
lib/xml_schema_validator/src/token/attribute_value.rs
View File

@@ -1,253 +1,191 @@
use std::sync::Arc;
use crate::*;
/// Represents an attribute value of an XML element
#[derive(Clone, Debug)]
pub struct AttributeValue {
/// The element that owns this attribute
element: Arc<schema::XsElement>,
/// The name of the attribute being processed
attribute_name: String,
/// Buffer containing the attribute value characters processed so far
buffer: String,
/// Whether we've encountered the opening quote
in_quotes: bool,
/// Whether we've seen the closing quote
closed: bool,
/// Track which attributes have already been set for this element
pub attributes_set: Vec<String>,
}
impl AttributeValue {
/// Create a new AttributeValue with a list of attributes already set
pub fn new(element: Arc<schema::XsElement>, attribute_name: String, attributes_set: Vec<String>) -> Self {
Self {
element,
attribute_name,
buffer: String::new(),
in_quotes: false,
closed: false,
attributes_set,
}
}
/// Append a character to the attribute value and return possible continuations
pub fn append(self, c: char) -> Vec<Token> {
if self.closed {
// We've already closed the attribute value with a quote
// Add this attribute to the list of attributes set
let mut new_attributes = self.attributes_set.clone();
if !new_attributes.contains(&self.attribute_name) {
new_attributes.push(self.attribute_name.clone());
}
if c.is_whitespace() {
// Check if there are more attributes to parse
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
// If there are other required attributes that haven't been set yet
for attr in attributes {
if attr.name != self.attribute_name && attr.xs_attribute_use == "required" && !new_attributes.contains(&attr.name) {
return vec![Token::AttributeName(AttributeName::with_attributes(
Arc::clone(&self.element),
new_attributes,
))];
}
}
}
// No more required attributes, whitespace after attribute value, continue to next state
return vec![
Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes.clone())),
Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))
];
} else if c == '>' {
// End of opening tag
return vec![Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes))];
} else if c == '/' {
// Beginning of self-closing tag
return vec![Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))];
} else {
// Unexpected character after attribute value
return vec![];
}
}
if !self.in_quotes {
if c.is_whitespace() {
// Skip whitespace before the quotes
return vec![Token::AttributeValue(Self {
element: Arc::clone(&self.element),
attribute_name: self.attribute_name,
buffer: self.buffer,
in_quotes: false,
closed: false,
attributes_set: self.attributes_set,
})];
} else if c == '"' || c == '\'' {
// Start of quotes
return vec![Token::AttributeValue(Self {
element: Arc::clone(&self.element),
attribute_name: self.attribute_name,
buffer: self.buffer,
in_quotes: true,
closed: false,
attributes_set: self.attributes_set,
})];
} else {
// Unexpected character before quotes
return vec![];
}
} else {
// We're inside quotes
if c == '"' || c == '\'' {
// End of quotes, validate the attribute value
// In a real implementation, we'd check if the value is valid for this attribute type
return vec![Token::AttributeValue(Self {
element: Arc::clone(&self.element),
attribute_name: self.attribute_name,
buffer: self.buffer,
in_quotes: true,
closed: true,
attributes_set: self.attributes_set,
})];
} else {
// Continue building the attribute value
let new_buffer = self.buffer + &c.to_string();
return vec![Token::AttributeValue(Self {
element: Arc::clone(&self.element),
attribute_name: self.attribute_name,
buffer: new_buffer,
in_quotes: true,
closed: false,
attributes_set: self.attributes_set,
})];
}
}
}
}
#[cfg(test)]
mod attribute_value_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_element_with_attributes() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "delete".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: Some(vec![
schema::XsAttribute {
name: "id".to_string(),
xs_attribute_type: "xs:string".to_string(),
xs_attribute_use: "required".to_string(),
}
]),
xs_sequence: None,
},
};
Arc::new(element)
}
#[test]
fn test_attribute_value_quotes() {
let element = create_element_with_attributes();
let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
// Test with opening quote
let next_tokens = token.append('"');
assert!(!next_tokens.is_empty(), "Should accept opening quote");
let token = next_tokens[0].clone();
match token.clone() {
Token::AttributeValue(value) => {
assert!(value.in_quotes, "Should mark as in quotes");
assert!(!value.closed, "Should not be closed yet");
},
_ => panic!("Expected AttributeValue, got {:?}", token),
}
// Test with content inside quotes
let next_tokens = token.append('a');
assert!(!next_tokens.is_empty(), "Should accept content inside quotes");
let token = next_tokens[0].clone();
// Test with closing quote
let next_tokens = token.append('"');
assert!(!next_tokens.is_empty(), "Should accept closing quote");
let token = next_tokens[0].clone();
match token {
Token::AttributeValue(value) => {
assert!(value.closed, "Should be closed");
},
_ => panic!("Expected AttributeValue, got {:?}", token),
}
}
#[test]
fn test_after_attribute_value_closed() {
let element = create_element_with_attributes();
let mut token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
token.in_quotes = true;
token.closed = true;
// Test with space after closed attribute value
let next_tokens = token.append(' ');
assert!(!next_tokens.is_empty(), "Should accept space after closed attribute");
// Test with > after closed attribute value
let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
let mut token = token;
token.in_quotes = true;
token.closed = true;
let next_tokens = token.append('>');
assert!(!next_tokens.is_empty(), "Should accept '>' after closed attribute");
match &next_tokens[0] {
Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
}
// Test with / after closed attribute value (for self-closing)
let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
let mut token = token;
token.in_quotes = true;
token.closed = true;
let next_tokens = token.append('/');
assert!(!next_tokens.is_empty(), "Should accept '/' after closed attribute");
match &next_tokens[0] {
Token::ElementSelfClose(_) => (),
_ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_attribute_tracking() {
let element = create_element_with_attributes();
let mut token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
token.in_quotes = true;
token.closed = true;
// After attribute is closed, it should be tracked
let next_tokens = token.append('>');
assert!(!next_tokens.is_empty(), "Should accept '>' after attribute");
match &next_tokens[0] {
Token::ElementOpenEnd(element_open) => {
// Verify that the id attribute is tracked
assert!(element_open.attributes_set.contains(&"id".to_string()),
"Attribute 'id' should be tracked");
},
_ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
}
}
use crate::*;
/// Represents an attribute value of an XML element
#[derive(Clone, Debug)]
pub struct AttributeValue {
/// The element that owns this attribute
element: Arc<schema::XsElement>,
/// Buffer containing the attribute value characters processed so far
buffer: String,
/// Track which attributes have already been set for this element
pub attributes_set: Vec<schema::XsAttribute>,
}
impl AttributeValue {
/// Create a new AttributeValue with a list of attributes already set
pub fn new(
element: Arc<schema::XsElement>,
attributes_set: Vec<schema::XsAttribute>,
) -> Self {
Self {
element,
buffer: "\"".to_string(),
attributes_set,
}
}
/// Append a character to the attribute value and return possible continuations
pub fn append(mut self, c: char) -> Vec<Token> {
// Handle escaped characters
if Self::in_escape_sequence(&self.buffer) {
self.buffer.push(c);
return vec![Token::AttributeValue(self)];
} else if self.value_closed() {
return self.next_tokens(c);
} else if '"' == c {
self.buffer.push(c);
return vec![Token::AttributeValue(self)];
} else {
self.buffer.push(c);
return vec![Token::AttributeValue(self)];
}
}
/// Checks if an XML character entity reference is in progress and correctly terminated
/// Returns true if the sequence is complete (ends with a semicolon)
pub fn in_escape_sequence(buffer: &str) -> bool {
if let Some(amp_pos) = buffer.rfind('&') {
if buffer[amp_pos..].find(';').is_some() {
return false;
} else {
return true;
}
}
false
}
fn value_closed(&self) -> bool {
if self.buffer.len() > 1 && self.buffer.trim().ends_with("\"") {
let trimmed = self.buffer.trim();
let trimmed = &trimmed[..trimmed.len() - 2];
if Self::in_escape_sequence(trimmed) {
false
} else {
true
}
} else {
false
}
}
fn next_tokens(mut self, c: char) -> Vec<Token> {
let all_required_attributes_set: bool = self
.element
.as_ref()
.xs_complex_type
.xs_attribute
.as_deref()
.unwrap()
.iter()
.all(|attr| {
if attr.xs_attribute_use == "required" {
self.attributes_set.contains(attr)
} else {
true
}
});
let all_attributes_set: bool = self
.element
.as_ref()
.xs_complex_type
.xs_attribute
.as_deref()
.unwrap()
.iter()
.all(|attr| self.attributes_set.contains(attr));
if c.is_whitespace() {
self.buffer.push(c);
vec![Token::AttributeValue(self)]
} else {
let mut tokens = vec![];
if all_required_attributes_set {
if '/' == c {
tokens.push(Token::ElementSelfClose(ElementSelfClose::new()));
} else if '>' == c {
tokens.push(Token::ElementOpenEnd(ElementOpenEnd::new(self.element.clone())));
}
}
if self.buffer.ends_with(char::is_whitespace) && !all_attributes_set {
tokens.push(Token::AttributeName(AttributeName::new_with_used(
c,
self.element,
self.attributes_set,
)));
}
tokens
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_unfinished_attribute_value() {
let values = [
"<delete id=\"12",
"<single timeout=\"1.",
"<single limit=\"50",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
// Should still be in AttributeValue state
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::AttributeValue(_))),
"{value}"
);
}
}
#[test]
fn test_type_validation() {
// Test integer validation
let mut validator = crate::tests::example_validator();
validator.append("<single limit=\"123\"").expect("Valid integer");
// Test float validation
validator = crate::tests::example_validator();
validator.append("<single timeout=\"1.5\"").expect("Valid float");
// Test string validation
validator = crate::tests::example_validator();
validator.append("<delete id=\"abc123\"").expect("Valid string");
}
#[test]
fn test_multiple_attributes() {
let values = [
"<single limit=\"100\" timeout=\"1.5\"",
"<single timeout=\"0.5\" limit=\"200\"",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
}
}
#[test]
fn test_attribute_value_escape_chars() {
// Test handling of escaped characters in attribute values
let values = [
"<delete id=\"escaped&quot;quote\"",
"<delete id=\"escaped&amp;amp\"",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
}
}
}

286
lib/xml_schema_validator/src/token/element_close_name.rs
View File

@@ -1,137 +1,149 @@
use std::sync::Arc;
use crate::*;
/// Represents a closing element tag that is in the process of being parsed
/// after the '</' and now parsing the element name
#[derive(Clone, Debug)]
pub struct ElementCloseName {
/// The element that was previously opened and is now being closed
element: Arc<schema::XsElement>,
/// Current buffer of characters processed for the element name
buffer: String,
}
impl ElementCloseName {
/// Create a new instance with the given buffer
pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Self {
Self {
element,
buffer,
}
}
pub fn append(self, c: char) -> Vec<Token> {
if self.element.name == self.buffer && c.is_whitespace() {
return vec![Token::ElementCloseName(self)];
}
if self.element.name == self.buffer && c == '>' {
return vec![Token::EndOfFile(EndOfFile::new())];
}
let new_buffer = self.buffer + &c.to_string();
if self.element.name.starts_with(&new_buffer) {
return vec![Token::ElementCloseName(ElementCloseName::new(
Arc::clone(&self.element),
new_buffer,
))];
} else {
vec![]
}
}
}
#[cfg(test)]
mod element_close_name_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_close_name_partial() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
// Test with next character in name
let next_tokens = token.append('e');
assert!(!next_tokens.is_empty(), "Should accept next character in name");
match &next_tokens[0] {
Token::ElementCloseName(name) => {
assert_eq!(name.buffer, "re", "Buffer should contain 're'");
},
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_name_complete() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "reasonin".to_string());
// Test with last character of name
let next_tokens = token.append('g');
assert!(!next_tokens.is_empty(), "Should accept last character of name");
match &next_tokens[0] {
Token::ElementCloseName(name) => {
assert_eq!(name.buffer, "reasoning", "Buffer should contain 'reasoning'");
},
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
// Test with > after complete name
let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
let next_tokens = token.append('>');
assert!(!next_tokens.is_empty(), "Should accept '>' after complete name");
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_name_whitespace() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
// Test with whitespace after complete name
let next_tokens = token.append(' ');
assert!(!next_tokens.is_empty(), "Should accept whitespace after complete name");
match &next_tokens[0] {
Token::ElementCloseName(_) => (),
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_name_invalid() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
// Test with invalid next character
let next_tokens = token.append('x');
assert!(next_tokens.is_empty(), "Should reject invalid character");
}
}
use crate::*;
use std::sync::Arc;
/// Represents a closing element tag that is in the process of being parsed
/// after the '</' and now parsing the element name
#[derive(Clone, Debug)]
pub struct ElementCloseName {
/// The element that was previously opened and is now being closed
element: Arc<schema::XsElement>,
/// Current buffer of characters processed for the element name
buffer: String,
}
impl ElementCloseName {
/// Create a new instance with the given buffer
pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Self {
Self { element, buffer }
}
pub fn append(self, c: char) -> Vec<Token> {
if self.element.name == self.buffer && c.is_whitespace() {
return vec![Token::ElementCloseName(self)];
}
if self.element.name == self.buffer && c == '>' {
return vec![Token::EndOfFile(EndOfFile::new())];
}
let new_buffer = self.buffer + &c.to_string();
if self.element.name.starts_with(&new_buffer) {
return vec![Token::ElementCloseName(ElementCloseName::new(
Arc::clone(&self.element),
new_buffer,
))];
} else {
vec![]
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_close_name_partial() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
// Test with next character in name
let next_tokens = token.append('e');
assert!(
!next_tokens.is_empty(),
"Should accept next character in name"
);
match &next_tokens[0] {
Token::ElementCloseName(name) => {
assert_eq!(name.buffer, "re", "Buffer should contain 're'");
}
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_name_complete() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "reasonin".to_string());
// Test with last character of name
let next_tokens = token.append('g');
assert!(
!next_tokens.is_empty(),
"Should accept last character of name"
);
match &next_tokens[0] {
Token::ElementCloseName(name) => {
assert_eq!(
name.buffer, "reasoning",
"Buffer should contain 'reasoning'"
);
}
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
// Test with > after complete name
let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
let next_tokens = token.append('>');
assert!(
!next_tokens.is_empty(),
"Should accept '>' after complete name"
);
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_name_whitespace() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
// Test with whitespace after complete name
let next_tokens = token.append(' ');
assert!(
!next_tokens.is_empty(),
"Should accept whitespace after complete name"
);
match &next_tokens[0] {
Token::ElementCloseName(_) => (),
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_name_invalid() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
// Test with invalid next character
let next_tokens = token.append('x');
assert!(next_tokens.is_empty(), "Should reject invalid character");
}
}

217
lib/xml_schema_validator/src/token/element_close_start.rs
View File

@@ -1,107 +1,110 @@
use std::sync::Arc;
use crate::*;
/// Represents the start of an XML element closing tag (the '</' sequence)
#[derive(Clone, Debug)]
pub struct ElementCloseStart {
/// The element that was previously opened and is now being closed
element: Arc<schema::XsElement>,
has_slash: bool,
}
impl ElementCloseStart {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
has_slash: false,
}
}
pub fn append(self, c: char) -> Vec<Token> {
if !self.has_slash && '/' == c {
vec![Token::ElementCloseStart(Self {
element: Arc::clone(&self.element),
has_slash: true,
})]
} else if self.has_slash && self.element.name.starts_with(c) {
vec![Token::ElementCloseName(ElementCloseName::new(
Arc::clone(&self.element),
c.to_string(),
))]
} else {
vec![]
}
}
}
#[cfg(test)]
mod element_close_start_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_close_start_slash() {
let element = create_test_element();
let token = ElementCloseStart::new(Arc::clone(&element));
// Test with slash
let next_tokens = token.append('/');
assert!(!next_tokens.is_empty(), "Should accept '/'");
match &next_tokens[0] {
Token::ElementCloseStart(start) => {
assert!(start.has_slash, "Should mark as having slash");
},
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_start_element_name() {
let element = create_test_element();
let mut token = ElementCloseStart::new(Arc::clone(&element));
token.has_slash = true;
// Test with first character of element name
let next_tokens = token.append('r');
assert!(!next_tokens.is_empty(), "Should accept first letter of element name");
match &next_tokens[0] {
Token::ElementCloseName(_) => (),
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_start_invalid_name() {
let element = create_test_element();
let mut token = ElementCloseStart::new(Arc::clone(&element));
token.has_slash = true;
// Test with invalid first character of element name
let next_tokens = token.append('x');
assert!(next_tokens.is_empty(), "Should reject invalid first letter");
}
}
use crate::*;
use std::sync::Arc;
/// Represents the start of an XML element closing tag (the '</' sequence)
#[derive(Clone, Debug)]
pub struct ElementCloseStart {
/// The element that was previously opened and is now being closed
element: Arc<schema::XsElement>,
has_slash: bool,
}
impl ElementCloseStart {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
has_slash: false,
}
}
pub fn append(self, c: char) -> Vec<Token> {
if !self.has_slash && '/' == c {
vec![Token::ElementCloseStart(Self {
element: Arc::clone(&self.element),
has_slash: true,
})]
} else if self.has_slash && self.element.name.starts_with(c) {
vec![Token::ElementCloseName(ElementCloseName::new(
Arc::clone(&self.element),
c.to_string(),
))]
} else {
vec![]
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_close_start_slash() {
let element = create_test_element();
let token = ElementCloseStart::new(Arc::clone(&element));
// Test with slash
let next_tokens = token.append('/');
assert!(!next_tokens.is_empty(), "Should accept '/'");
match &next_tokens[0] {
Token::ElementCloseStart(start) => {
assert!(start.has_slash, "Should mark as having slash");
}
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_start_element_name() {
let element = create_test_element();
let mut token = ElementCloseStart::new(Arc::clone(&element));
token.has_slash = true;
// Test with first character of element name
let next_tokens = token.append('r');
assert!(
!next_tokens.is_empty(),
"Should accept first letter of element name"
);
match &next_tokens[0] {
Token::ElementCloseName(_) => (),
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_close_start_invalid_name() {
let element = create_test_element();
let mut token = ElementCloseStart::new(Arc::clone(&element));
token.has_slash = true;
// Test with invalid first character of element name
let next_tokens = token.append('x');
assert!(next_tokens.is_empty(), "Should reject invalid first letter");
}
}

383
lib/xml_schema_validator/src/token/element_open_end.rs
View File

@@ -1,220 +1,163 @@
use std::sync::Arc;
use crate::*;
/// Represents a fully opened XML element, containing the schema element reference
#[derive(Clone, Debug)]
pub struct ElementOpenEnd {
element: Arc<schema::XsElement>,
// Track which attributes have been set for this element
pub attributes_set: Vec<String>,
}
impl ElementOpenEnd {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
attributes_set: Vec::new(),
}
}
pub fn with_attributes(element: Arc<schema::XsElement>, attributes_set: Vec<String>) -> Self {
Self {
element,
attributes_set,
}
}
pub fn append(self, c: char) -> Vec<Token> {
// Check if all required attributes are present before proceeding
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
for attr in attributes {
if attr.xs_attribute_use == "required" && !self.attributes_set.contains(&attr.name) {
// Missing a required attribute, this is an error
return vec![];
}
}
}
// Check if the element is a mixed content element that allows text
let allows_text = self.element.xs_complex_type.mixed
.as_ref()
.map(|mixed| mixed == "true")
.unwrap_or(false);
if allows_text {
if c.is_whitespace() {
// Handle whitespace in mixed content
vec![
Token::TextContent(TextContent::new(Arc::clone(&self.element))),
Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))
]
} else if c == '<' {
// Start of a closing tag or nested element
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
} else {
// Text content
vec![Token::TextContent(TextContent::new(Arc::clone(&self.element)))]
}
} else {
// Element doesn't allow text content, only expect closing tag
if c == '<' {
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
} else if c.is_whitespace() {
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
} else {
// Unexpected content in non-mixed element
vec![]
}
}
}
}
#[cfg(test)]
mod element_open_end_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
fn create_element_non_mixed() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "element".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None, // Not mixed content
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
fn create_element_with_required_attributes() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "delete".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: Some(vec![
schema::XsAttribute {
name: "id".to_string(),
xs_attribute_type: "xs:string".to_string(),
xs_attribute_use: "required".to_string(),
}
]),
xs_sequence: None,
},
};
Arc::new(element)
}
#[test]
fn test_append_text_in_mixed_content() {
let element = create_test_element();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test appending text in mixed content element
let next_tokens = token.append('a');
assert!(!next_tokens.is_empty(), "Should allow text content in mixed element");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_closing_tag_start() {
let element = create_test_element();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test starting a closing tag
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should allow closing tag start");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_whitespace_in_mixed() {
let element = create_test_element();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test whitespace in mixed content
let next_tokens = token.append(' ');
assert!(!next_tokens.is_empty(), "Should allow whitespace in mixed content");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_in_non_mixed() {
let element = create_element_non_mixed();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test appending text in non-mixed content (should reject)
let next_tokens = token.clone().append('a');
assert!(next_tokens.is_empty(), "Should reject text in non-mixed element");
// Test whitespace (should accept leading to closing tag)
let next_tokens = token.clone().append(' ');
assert!(!next_tokens.is_empty(), "Should allow whitespace in non-mixed");
// Test closing tag start
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should allow closing tag start in non-mixed");
}
#[test]
fn test_required_attributes() {
let element = create_element_with_required_attributes();
// No attributes set yet
let token = ElementOpenEnd::new(Arc::clone(&element));
let next_tokens = token.append('<');
assert!(next_tokens.is_empty(), "Should reject closing tag without required attributes");
// With required attribute set
let token = ElementOpenEnd::with_attributes(
Arc::clone(&element),
vec!["id".to_string()]
);
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should allow closing tag with required attributes set");
}
}
use crate::*;
use std::sync::Arc;
/// Represents a fully opened XML element, containing the schema element reference
#[derive(Clone, Debug)]
pub struct ElementOpenEnd {
element: Arc<schema::XsElement>,
}
impl ElementOpenEnd {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
}
}
pub fn append(self, c: char) -> Vec<Token> {
if '<' == c {
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
} else {
// Check if this element allows mixed content
let is_mixed = self.element.xs_complex_type.mixed
.as_ref()
.map(|val| val == "true")
.unwrap_or(false);
if is_mixed || c.is_whitespace() {
// Allow text content for mixed elements, or whitespace for any element
vec![Token::TextContent(TextContent::new(self.element))]
} else {
// Reject text content for non-mixed elements
vec![]
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
fn create_element_non_mixed() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "element".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None, // Not mixed content
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_append_text_in_mixed_content() {
let element = create_test_element();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test appending text in mixed content element
let next_tokens = token.append('a');
assert!(
!next_tokens.is_empty(),
"Should allow text content in mixed element"
);
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_closing_tag_start() {
let element = create_test_element();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test starting a closing tag
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should allow closing tag start");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_whitespace_in_mixed() {
let element = create_test_element();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test whitespace in mixed content
let next_tokens = token.append(' ');
assert!(
!next_tokens.is_empty(),
"Should allow whitespace in mixed content"
);
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_in_non_mixed() {
let element = create_element_non_mixed();
let token = ElementOpenEnd::new(Arc::clone(&element));
// Test appending text in non-mixed content (should reject)
let next_tokens = token.clone().append('a');
assert!(
next_tokens.is_empty(),
"Should reject text in non-mixed element"
);
// Test whitespace (should accept leading to closing tag)
let next_tokens = token.clone().append(' ');
assert!(
!next_tokens.is_empty(),
"Should allow whitespace in non-mixed"
);
// Test closing tag start
let next_tokens = token.append('<');
assert!(
!next_tokens.is_empty(),
"Should allow closing tag start in non-mixed"
);
}
}

458
lib/xml_schema_validator/src/token/element_open_name.rs
View File

@@ -1,272 +1,186 @@
use std::sync::Arc;
use crate::*;
/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
#[derive(Clone, Debug)]
pub struct ElementOpenName {
buffer: String,
element: Arc<schema::XsElement>,
}
impl ElementOpenName {
pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Result<Self> {
if element.name.starts_with(buffer.as_str()) {
Ok(Self {
buffer,
element,
})
} else {
Err(Error::InvalidXml(format!("Element name '{}' doesn't match '{}'", buffer, element.name)))
}
}
pub fn append(self, c: char) -> Vec<Token> {
if c.is_whitespace() {
// Check if we've matched the full element name
if self.element.name == self.buffer {
// Element name is complete, handle whitespace after name
// Check if the element has attributes
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
if !attributes.is_empty() {
// Transition to attribute name parsing
return vec![Token::AttributeName(AttributeName::new(Arc::clone(&self.element)))];
}
}
// No attributes
return vec![
Token::AttributeName(AttributeName::new(Arc::clone(&self.element))),
Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element))),
Token::ElementSelfClose(ElementSelfClose::new()),
Token::ElementOpenName(self),
];
} else {
// Whitespace in the middle of an element name is invalid
return vec![];
}
} else if c == '>' {
// Check if we've matched the full element name
if self.element.name == self.buffer {
// Element name is complete, end of opening tag
// Check if this element has required attributes
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
for attr in attributes {
if attr.xs_attribute_use == "required" {
// Found a required attribute but no attributes are set
// Return an empty vector to indicate error
return vec![];
}
}
}
return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element)))];
} else {
// '>' before full element name is invalid
return vec![];
}
} else if c == '/' {
// Check if we've matched the full element name
if self.element.name == self.buffer {
// Element name is complete, self-closing tag
return vec![Token::ElementSelfClose(ElementSelfClose::new())];
} else {
// '/' before full element name is invalid
return vec![];
}
} else {
// Add character to buffer and check if we're still building a valid element name
let new_buffer = self.buffer.clone() + &c.to_string();
if self.element.name.starts_with(&new_buffer) {
// Still building the element name
return vec![Token::ElementOpenName(ElementOpenName {
buffer: new_buffer,
element: Arc::clone(&self.element),
})];
} else {
// Character doesn't match what we expect for this element name
return vec![];
}
}
}
}
#[cfg(test)]
mod element_open_name_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
fn create_element_with_attributes() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "delete".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: Some(vec![
schema::XsAttribute {
name: "id".to_string(),
xs_attribute_type: "xs:string".to_string(),
xs_attribute_use: "required".to_string(),
}
]),
xs_sequence: None,
},
};
Arc::new(element)
}
#[test]
fn test_append_character_by_character() {
let element = create_test_element();
// Test each character in "reasoning"
let mut buffer = String::new();
for c in "reasoning".chars() {
buffer.push(c);
let result = ElementOpenName::new(Arc::clone(&element), buffer.clone());
assert!(result.is_ok(), "Failed to create ElementOpenName with buffer '{}'", buffer);
}
}
#[test]
fn test_append_full_name() {
let element = create_test_element();
// Test with the full element name
let result = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string());
assert!(result.is_ok(), "Failed to create ElementOpenName with full element name");
let element_name = result.unwrap();
// Test transition after full name with '>'
let next_tokens = element_name.append('>');
assert!(!next_tokens.is_empty(), "Should accept '>' after full element name");
match &next_tokens[0] {
Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_space_after_name() {
let element = create_test_element();
// Test with the full element name
let element_name = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()).unwrap();
// Test transition after full name with space
let next_tokens = element_name.append(' ');
assert!(!next_tokens.is_empty(), "Should accept space after full element name");
assert!(next_tokens.len() == 4);
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::ElementOpenName(_)))
.next()
.is_some());
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::AttributeName(_)))
.next()
.is_some());
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::ElementOpenEnd(_)))
.next()
.is_some());
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::ElementSelfClose(_)))
.next()
.is_some());
}
#[test]
fn test_append_with_attributes() {
let element = create_element_with_attributes();
// Test with the full element name
let element_name = ElementOpenName::new(Arc::clone(&element), "delete".to_string()).unwrap();
// Test transition after full name with space (expecting attribute name)
let next_tokens = element_name.append(' ');
assert!(!next_tokens.is_empty(), "Should accept space after element name with attributes");
// The space should lead to a whitespace token, which should then lead to attribute name
match &next_tokens[0] {
Token::AttributeName(_) => (),
_ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_append_slash_after_name() {
let element = create_test_element();
// Test with the full element name
let element_name = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()).unwrap();
// Test transition after full name with /
let next_tokens = element_name.append('/');
assert!(!next_tokens.is_empty(), "Should accept '/' after full element name");
match &next_tokens[0] {
Token::ElementSelfClose(_) => (),
_ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_invalid_continuation() {
let element = create_test_element();
// Test with a partial element name and an invalid continuation character
let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap();
let next_tokens = element_name.append('x');
assert!(next_tokens.is_empty(), "Should reject invalid character in element name");
// Test whitespace in middle of name - need to create a new instance since append consumes self
let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap();
let next_tokens = element_name.append(' ');
assert!(next_tokens.is_empty(), "Should reject whitespace in middle of element name");
// Test > in middle of name - need to create a new instance since append consumes self
let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap();
let next_tokens = element_name.append('>');
assert!(next_tokens.is_empty(), "Should reject '>' in middle of element name");
}
#[test]
fn test_required_attributes_validation() {
let element = create_element_with_attributes();
// Test with the full element name
let element_name = ElementOpenName::new(Arc::clone(&element), "delete".to_string()).unwrap();
// Try to close the tag with > directly, skipping the required attribute
let next_tokens = element_name.append('>');
assert!(next_tokens.is_empty(), "Should reject closing tag without required attributes");
}
}
use crate::*;
use std::sync::Arc;
/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
#[derive(Clone, Debug)]
pub struct ElementOpenName {
buffer: String,
element: Arc<schema::XsElement>,
}
impl ElementOpenName {
pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Self {
Self { buffer, element }
}
pub fn append(mut self, c: char) -> Vec<Token> {
if c == '>' {
// Check if we've matched the full element name
if !self.buffer.contains(&self.element.name) {
return vec![];
}
// Check if the element has attributes
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
// Check if they are required
if attributes
.iter()
.filter(|attribute| attribute.xs_attribute_use == "required")
.next()
.is_some()
{
return vec![];
} else {
return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(
&self.element,
)))];
}
} else {
return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(
&self.element,
)))];
}
} else if c == '/' {
// Check if we've matched the full element name
if !self.buffer.contains(&self.element.name) {
return vec![];
}
// Check if the element has attributes
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
// Check if they are required
if attributes
.iter()
.filter(|attribute| attribute.xs_attribute_use == "required")
.next()
.is_some()
{
return vec![];
} else {
return vec![Token::ElementSelfClose(ElementSelfClose::new())];
}
} else {
return vec![Token::ElementSelfClose(ElementSelfClose::new())];
}
} else if c.is_whitespace() {
// Check if we've matched the full element name
if self.buffer.contains(&self.element.name) {
if let Some(last) = self.buffer.chars().last() {
if last.is_whitespace() {
return vec![Token::ElementOpenName(self)];
}
}
self.buffer.push(c);
return vec![Token::ElementOpenName(self)];
} else {
return vec![];
}
} else if !self.buffer.is_empty() && self.buffer.chars().last().unwrap().is_whitespace() {
// Check for start of attribute name
if self.buffer.contains(&self.element.name) {
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
if attributes
.iter()
.filter(|attribute| attribute.name.starts_with(c))
.next()
.is_some()
{
return vec![Token::AttributeName(AttributeName::new(
c,
self.element.clone(),
))];
}
}
}
return vec![];
} else {
// Add character to buffer and check if we're still building a valid element name
let new_buffer = self.buffer.clone() + &c.to_string();
if self.element.name.starts_with(&new_buffer) {
// Still building the element name
return vec![Token::ElementOpenName(ElementOpenName {
buffer: new_buffer,
element: Arc::clone(&self.element),
})];
} else {
// Character doesn't match what we expect for this element name
return vec![];
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_attribute_name() {
let values = [
"<delete i", // id
"<single t", // timeout
"<single l", // limit
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeName(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_valid_self_closing() {
let values = ["<stop/", "<stop /"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementSelfClose(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_valid_closing() {
let values = ["<stop>", "<stop >", "<reasoning>", "<reasoning >"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementOpenEnd(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_invalid_closing() {
let values = ["<delete/", "<delete /", "<delete>", "<delete >"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
}

89
lib/xml_schema_validator/src/token/element_open_start.rs
View File

@@ -1,25 +1,64 @@
use std::sync::Arc;
use crate::*;
/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
#[derive(Clone, Debug)]
pub struct ElementOpenStart {
element: Arc<schema::XsElement>,
}
impl ElementOpenStart {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
}
}
pub fn append(self, c: char) -> Vec<Token> {
let element_name = ElementOpenName::new(Arc::clone(&self.element), c.to_string());
if let Ok(element_name) = element_name {
vec![Token::ElementOpenName(element_name)]
} else {
vec![]
}
}
}
use crate::*;
use std::sync::Arc;
/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
#[derive(Clone, Debug)]
pub struct ElementOpenStart {
element: Arc<schema::XsElement>,
}
impl ElementOpenStart {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self { element }
}
pub fn append(self, c: char) -> Vec<Token> {
if self.element.name.starts_with(c) {
vec![Token::ElementOpenName(ElementOpenName::new(
Arc::clone(&self.element),
c.to_string(),
))]
} else {
vec![]
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_accept_valid_element_names() {
let values = [
"<d", // delete
"<s", // stop / single
"<r", // repeat / reasoning / read_stdin
"<w", // write_stdout
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementOpenName(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_not_accept_other() {
let values = ["< ", "<a", "</"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
}

198
lib/xml_schema_validator/src/token/element_self_close.rs
View File

@@ -1,144 +1,54 @@
use std::sync::Arc;
use crate::*;
/// Represents a self-closing XML element
#[derive(Clone, Debug)]
pub struct ElementSelfClose {
/// Optional reference to the element schema for attribute validation
element: Option<Arc<schema::XsElement>>,
/// Track which attributes have been set
attributes_set: Vec<String>,
}
impl ElementSelfClose {
/// Create a new self-closing element
pub fn new() -> Self {
Self {
element: None,
attributes_set: Vec::new(),
}
}
/// Create a new self-closing element with element reference and attribute tracking
pub fn new_with_attributes(element: Arc<schema::XsElement>, attributes_set: Vec<String>) -> Self {
Self {
element: Some(element),
attributes_set,
}
}
/// Append a character to the self-closing tag and return possible continuations
pub fn append(self, c: char) -> Vec<Token> {
// Before we close, verify that all required attributes are present
if let Some(element) = &self.element {
if c == '>' {
if let Some(attributes) = &element.xs_complex_type.xs_attribute {
for attr in attributes {
if attr.xs_attribute_use == "required" && !self.attributes_set.contains(&attr.name) {
// Missing required attribute, this is an error
return vec![];
}
}
}
return vec![Token::EndOfFile(EndOfFile::new())];
}
} else if c == '>' {
// No element reference, can't validate attributes, just close
return vec![Token::EndOfFile(EndOfFile::new())];
}
if c.is_whitespace() {
vec![Token::ElementSelfClose(self)]
} else {
vec![]
}
}
}
#[cfg(test)]
mod element_self_close_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_element_with_required_attributes() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "delete".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: Some(vec![
schema::XsAttribute {
name: "id".to_string(),
xs_attribute_type: "xs:string".to_string(),
xs_attribute_use: "required".to_string(),
}
]),
xs_sequence: None,
},
};
Arc::new(element)
}
#[test]
fn test_self_close_with_greater_than() {
let token = ElementSelfClose::new();
// Test closing with >
let next_tokens = token.append('>');
assert!(!next_tokens.is_empty(), "Should accept '>' to close self-closing tag");
assert_eq!(next_tokens.len(), 1, "Should have exactly one token after '>'");
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_self_close_with_whitespace() {
let token = ElementSelfClose::new();
// Test with whitespace
let next_tokens = token.append(' ');
assert!(!next_tokens.is_empty(), "Should accept whitespace in self-closing tag");
match &next_tokens[0] {
Token::ElementSelfClose(_) => (),
_ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_self_close_with_invalid_char() {
let token = ElementSelfClose::new();
// Test with invalid character
let next_tokens = token.append('a');
assert!(next_tokens.is_empty(), "Should reject invalid character in self-closing tag");
}
#[test]
fn test_required_attributes_validation() {
let element = create_element_with_required_attributes();
// Without required attributes
let token = ElementSelfClose::new_with_attributes(Arc::clone(&element), vec![]);
let next_tokens = token.append('>');
assert!(next_tokens.is_empty(), "Should reject closing without required attributes");
// With required attributes
let token = ElementSelfClose::new_with_attributes(
Arc::clone(&element),
vec!["id".to_string()]
);
let next_tokens = token.append('>');
assert!(!next_tokens.is_empty(), "Should accept closing with required attributes");
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
}
}
}
use crate::*;
/// Represents a self-closing XML element
#[derive(Clone, Debug)]
pub struct ElementSelfClose {}
impl ElementSelfClose {
/// Create a new self-closing element
pub fn new() -> Self {
Self {}
}
pub fn append(self, c: char) -> Vec<Token> {
if c == '>' {
return vec![Token::EndOfFile(EndOfFile::new())];
} else {
vec![]
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_valid() {
let values = ["<stop/>"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::EndOfFile(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_invalid() {
let values = ["<stop/?", "<stop/a", "<stop/ "];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
}

59
lib/xml_schema_validator/src/token/end_of_file.rs
View File

@@ -1,30 +1,29 @@
use crate::*;
/// Represents the final state at the end of an XML file
#[derive(Clone, Debug)]
pub struct EndOfFile {
}
impl EndOfFile {
pub fn new() -> Self {
Self {}
}
pub fn append(self, _c: char) -> Vec<Token> {
vec![]
}
}
#[cfg(test)]
mod end_of_file_tests {
use super::*;
#[test]
fn test_end_of_file_any_char() {
let token = EndOfFile::new();
// Test with any character (should reject)
let next_tokens = token.append('a');
assert!(next_tokens.is_empty(), "Should reject any character at EOF");
}
}
use crate::*;
/// Represents the final state at the end of an XML file
#[derive(Clone, Debug)]
pub struct EndOfFile {}
impl EndOfFile {
pub fn new() -> Self {
Self {}
}
pub fn append(self, _c: char) -> Vec<Token> {
vec![]
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_end_of_file_any_char() {
let token = EndOfFile::new();
// Test with any character (should reject)
let next_tokens = token.append('a');
assert!(next_tokens.is_empty(), "Should reject any character at EOF");
}
}

577
lib/xml_schema_validator/src/token/mod.rs
View File

@@ -1,292 +1,285 @@
mod element_close_name;
mod element_close_start;
mod element_open_name;
mod element_open_end;
mod element_open_start;
mod end_of_file;
mod start_of_file;
mod text_content;
mod attribute_name;
mod attribute_value;
mod element_self_close;
pub use element_close_name::ElementCloseName;
pub use element_close_start::ElementCloseStart;
pub use element_open_name::ElementOpenName;
pub use element_open_end::ElementOpenEnd;
pub use element_open_start::ElementOpenStart;
pub use end_of_file::EndOfFile;
pub use start_of_file::StartOfFile;
pub use text_content::TextContent;
pub use attribute_name::AttributeName;
pub use attribute_value::AttributeValue;
pub use element_self_close::ElementSelfClose;
/// Represents the different states of XML token parsing
#[derive(Clone, Debug)]
pub enum Token {
ElementCloseName(ElementCloseName),
ElementCloseStart(ElementCloseStart),
ElementOpenName(ElementOpenName),
ElementOpenEnd(ElementOpenEnd),
ElementOpenStart(ElementOpenStart),
EndOfFile(EndOfFile),
StartOfFile(StartOfFile),
TextContent(TextContent),
AttributeName(AttributeName),
AttributeValue(AttributeValue),
ElementSelfClose(ElementSelfClose),
}
impl Token {
pub fn append(self, c: char) -> Vec<Token> {
match self {
Token::ElementCloseName(element_close_name) => element_close_name.append(c),
Token::ElementCloseStart(element_close_start) => element_close_start.append(c),
Token::ElementOpenName(element_name) => element_name.append(c),
Token::ElementOpenEnd(element_open) => element_open.append(c),
Token::ElementOpenStart(element_start) => element_start.append(c),
Token::EndOfFile(end_of_file) => end_of_file.append(c),
Token::StartOfFile(start_of_file) => start_of_file.append(c),
Token::TextContent(text_content) => text_content.append(c),
Token::AttributeName(attribute_name) => attribute_name.append(c),
Token::AttributeValue(attribute_value) => attribute_value.append(c),
Token::ElementSelfClose(element_self_close) => element_self_close.append(c),
}
}
pub fn is_eof(&self) -> bool {
match self {
Token::EndOfFile(_) => true,
_ => false,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_start_of_file_transition() {
let element = create_test_element();
let token = Token::StartOfFile(StartOfFile::new(Arc::clone(&element)));
// Should only accept '<'
let next_tokens = token.append('<');
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementOpenStart(_) => (),
_ => panic!("Expected ElementStart, got {:?}", next_tokens[0]),
}
// Should reject other characters
let token = Token::StartOfFile(StartOfFile::new(Arc::clone(&element)));
let next_tokens = token.append('a');
assert_eq!(next_tokens.len(), 0, "Should reject 'a'");
}
#[test]
fn test_element_start_transition() {
let element = create_test_element();
let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
// Should accept first char of element name
let next_tokens = token.append('r');
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementOpenName(_) => (),
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
}
// Should reject invalid char
let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
let next_tokens = token.append('x');
assert_eq!(next_tokens.len(), 0, "Should reject 'x'");
}
#[test]
fn test_element_name_transition() {
let element = create_test_element();
let token = Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()).unwrap());
// Continue building name
let next_tokens = token.append('e');
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementOpenName(_) => (),
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
}
// Build complete name and end with >
// Remove the unused buffer
let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)))];
for c in "reasoning>".chars() {
let mut new_tokens = Vec::new();
for token in current_tokens {
new_tokens.append(&mut token.append(c));
}
current_tokens = new_tokens;
assert!(!current_tokens.is_empty(),
"Should have valid transition after appending '{}'", c);
}
// Check final state
assert_eq!(current_tokens.len(), 1, "Should have one final transition");
match &current_tokens[0] {
Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", current_tokens[0]),
}
}
#[test]
fn test_element_open_end_transition() {
let element = create_test_element();
let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
// Should accept text content in mixed element
let next_tokens = token.append('t');
assert_eq!(next_tokens.len(), 1, "Should accept text");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
// Should accept closing tag start
let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
let next_tokens = token.append('<');
assert_eq!(next_tokens.len(), 1, "Should accept '<'");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_text_content_transition() {
let element = create_test_element();
let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
// Should accept more text
let next_tokens = token.append('a');
assert_eq!(next_tokens.len(), 1, "Should accept more text");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
// Should accept closing tag start
let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
let next_tokens = token.append('<');
assert_eq!(next_tokens.len(), 1, "Should accept '<'");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_element_close_start_transition() {
let element = create_test_element();
let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
// Should accept /
let next_tokens = token.append('/');
assert_eq!(next_tokens.len(), 1, "Should accept '/'");
let token = next_tokens[0].clone();
match token {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart with slash, got {:?}", token),
}
// After /, should accept first char of element name
let next_tokens = token.append('r');
assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
match &next_tokens[0] {
Token::ElementCloseName(_) => (),
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_element_close_name_transition() {
let element = create_test_element();
let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
// Add / first
let next_tokens = token.append('/');
assert_eq!(next_tokens.len(), 1, "Should accept '/'");
// Start building name - clone the token to avoid moving out of the vector
let next_tokens = next_tokens[0].clone().append('r');
assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
// Continue with each character
let mut current_tokens = next_tokens;
for c in "easoning".chars() {
let mut new_tokens = Vec::new();
for token in current_tokens {
new_tokens.append(&mut token.append(c));
}
current_tokens = new_tokens;
assert!(!current_tokens.is_empty(),
"Should have valid transition after appending '{}'", c);
}
// Final > to close - clone the token to avoid moving out of the vector
let next_tokens = current_tokens[0].clone().append('>');
assert_eq!(next_tokens.len(), 1, "Should accept '>'");
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_end_of_file_properties() {
let token = Token::EndOfFile(EndOfFile::new());
// Should be identified as EOF
assert!(token.is_eof(), "EndOfFile should be recognized as EOF");
// Should not accept any more input
let next_tokens = token.append('a');
assert_eq!(next_tokens.len(), 0, "Should not accept any more input");
}
}
mod attribute_equals;
mod attribute_name;
mod attribute_value;
mod element_close_name;
mod element_close_start;
mod element_open_end;
mod element_open_name;
mod element_open_start;
mod element_self_close;
mod end_of_file;
mod start_of_file;
mod text_content;
pub use attribute_equals::AttributeEquals;
pub use attribute_name::AttributeName;
pub use attribute_value::AttributeValue;
pub use element_close_name::ElementCloseName;
pub use element_close_start::ElementCloseStart;
pub use element_open_end::ElementOpenEnd;
pub use element_open_name::ElementOpenName;
pub use element_open_start::ElementOpenStart;
pub use element_self_close::ElementSelfClose;
pub use end_of_file::EndOfFile;
pub use start_of_file::StartOfFile;
pub use text_content::TextContent;
/// Represents the different states of XML token parsing
#[derive(Clone, Debug)]
pub enum Token {
AttributeEquals(AttributeEquals),
AttributeName(AttributeName),
AttributeValue(AttributeValue),
ElementCloseName(ElementCloseName),
ElementCloseStart(ElementCloseStart),
ElementOpenEnd(ElementOpenEnd),
ElementOpenName(ElementOpenName),
ElementOpenStart(ElementOpenStart),
ElementSelfClose(ElementSelfClose),
EndOfFile(EndOfFile),
StartOfFile(StartOfFile),
TextContent(TextContent),
}
impl Token {
pub fn append(self, c: char) -> Vec<Token> {
match self {
Token::AttributeEquals(attribute_equals) => attribute_equals.append(c),
Token::ElementCloseName(element_close_name) => element_close_name.append(c),
Token::ElementCloseStart(element_close_start) => element_close_start.append(c),
Token::ElementOpenName(element_name) => element_name.append(c),
Token::ElementOpenEnd(element_open) => element_open.append(c),
Token::ElementOpenStart(element_start) => element_start.append(c),
Token::EndOfFile(end_of_file) => end_of_file.append(c),
Token::StartOfFile(start_of_file) => start_of_file.append(c),
Token::TextContent(text_content) => text_content.append(c),
Token::AttributeName(attribute_name) => attribute_name.append(c),
Token::AttributeValue(attribute_value) => attribute_value.append(c),
Token::ElementSelfClose(element_self_close) => element_self_close.append(c),
}
}
pub fn is_eof(&self) -> bool {
match self {
Token::EndOfFile(_) => true,
_ => false,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::schema;
use std::sync::Arc;
pub fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_element_start_transition() {
let element = create_test_element();
let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
// Should accept first char of element name
let next_tokens = token.append('r');
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementOpenName(_) => (),
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
}
// Should reject invalid char
let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
let next_tokens = token.append('x');
assert_eq!(next_tokens.len(), 0, "Should reject 'x'");
}
#[test]
fn test_element_name_transition() {
let element = create_test_element();
let token =
Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()));
// Continue building name
let next_tokens = token.append('e');
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementOpenName(_) => (),
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
}
// Build complete name and end with >
// Remove the unused buffer
let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(
&element,
)))];
for c in "reasoning>".chars() {
let mut new_tokens = Vec::new();
for token in current_tokens {
new_tokens.append(&mut token.append(c));
}
current_tokens = new_tokens;
assert!(
!current_tokens.is_empty(),
"Should have valid transition after appending '{}'",
c
);
}
// Check final state
assert_eq!(current_tokens.len(), 1, "Should have one final transition");
match &current_tokens[0] {
Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", current_tokens[0]),
}
}
#[test]
fn test_element_open_end_transition() {
let element = create_test_element();
let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
// Should accept text content in mixed element
let next_tokens = token.append('t');
assert_eq!(next_tokens.len(), 1, "Should accept text");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
// Should accept closing tag start
let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
let next_tokens = token.append('<');
assert_eq!(next_tokens.len(), 1, "Should accept '<'");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_text_content_transition() {
let element = create_test_element();
let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
// Should accept more text
let next_tokens = token.append('a');
assert_eq!(next_tokens.len(), 1, "Should accept more text");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
// Should accept closing tag start
let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
let next_tokens = token.append('<');
assert_eq!(next_tokens.len(), 1, "Should accept '<'");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_element_close_start_transition() {
let element = create_test_element();
let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
// Should accept /
let next_tokens = token.append('/');
assert_eq!(next_tokens.len(), 1, "Should accept '/'");
let token = next_tokens[0].clone();
match token {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart with slash, got {:?}", token),
}
// After /, should accept first char of element name
let next_tokens = token.append('r');
assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
match &next_tokens[0] {
Token::ElementCloseName(_) => (),
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_element_close_name_transition() {
let element = create_test_element();
let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
// Add / first
let next_tokens = token.append('/');
assert_eq!(next_tokens.len(), 1, "Should accept '/'");
// Start building name - clone the token to avoid moving out of the vector
let next_tokens = next_tokens[0].clone().append('r');
assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
// Continue with each character
let mut current_tokens = next_tokens;
for c in "easoning".chars() {
let mut new_tokens = Vec::new();
for token in current_tokens {
new_tokens.append(&mut token.append(c));
}
current_tokens = new_tokens;
assert!(
!current_tokens.is_empty(),
"Should have valid transition after appending '{}'",
c
);
}
// Final > to close - clone the token to avoid moving out of the vector
let next_tokens = current_tokens[0].clone().append('>');
assert_eq!(next_tokens.len(), 1, "Should accept '>'");
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_end_of_file_properties() {
let token = Token::EndOfFile(EndOfFile::new());
// Should be identified as EOF
assert!(token.is_eof(), "EndOfFile should be recognized as EOF");
// Should not accept any more input
let next_tokens = token.append('a');
assert_eq!(next_tokens.len(), 0, "Should not accept any more input");
}
}

127
lib/xml_schema_validator/src/token/start_of_file.rs
View File

@@ -1,75 +1,52 @@
use std::sync::Arc;
use crate::*;
/// Represents the initial state at the start of an XML file, containing a reference to the root element schema
#[derive(Clone, Debug)]
pub struct StartOfFile {
element: Arc<schema::XsElement>,
}
impl StartOfFile {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self { element }
}
pub fn append(self, c: char) -> Vec<Token> {
if '<' == c {
vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&self.element)))]
} else {
vec![]
}
}
}
#[cfg(test)]
mod start_of_file_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_start_of_file_less_than() {
let element = create_test_element();
let token = StartOfFile::new(Arc::clone(&element));
// Test with <
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should accept '<'");
match &next_tokens[0] {
Token::ElementOpenStart(_) => (),
_ => panic!("Expected ElementStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_start_of_file_invalid() {
let element = create_test_element();
let token = StartOfFile::new(Arc::clone(&element));
// Test with invalid character
let next_tokens = token.append('a');
assert!(next_tokens.is_empty(), "Should reject invalid character");
}
}
use crate::*;
use std::sync::Arc;
/// Represents the initial state at the start of an XML file, containing a reference to the root element schema
#[derive(Clone, Debug)]
pub struct StartOfFile {
element: Arc<schema::XsElement>,
}
impl StartOfFile {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self { element }
}
pub fn append(self, c: char) -> Vec<Token> {
if '<' == c {
vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(
&self.element,
)))]
} else {
vec![]
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_accept_open_start_token() {
let mut validator = crate::tests::example_validator();
validator.append("<").unwrap();
assert!(!validator.current_tokens.is_empty());
assert!(validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementOpenStart(_)))
.next()
.is_some());
}
#[test]
fn test_not_accept_other() {
let values = ["a", " ", "\t", ">", "/"];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
}
}
}

168
lib/xml_schema_validator/src/token/text_content.rs
View File

@@ -1,84 +1,84 @@
use std::sync::Arc;
use crate::*;
/// Represents the text content within an XML element
#[derive(Clone, Debug)]
pub struct TextContent {
element: Arc<schema::XsElement>,
}
impl TextContent {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
}
}
pub fn append(self, c: char) -> Vec<Token> {
if c == '<' {
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
} else {
vec![Token::TextContent(TextContent::new(
Arc::clone(&self.element),
))]
}
}
}
#[cfg(test)]
mod text_content_tests {
use super::*;
use std::sync::Arc;
use crate::schema;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_text_content_normal_char() {
let element = create_test_element();
let token = TextContent::new(Arc::clone(&element));
// Test with normal character
let next_tokens = token.append('a');
assert!(!next_tokens.is_empty(), "Should accept normal character");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_text_content_less_than() {
let element = create_test_element();
let token = TextContent::new(Arc::clone(&element));
// Test with < (start of tag)
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should accept '<'");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
}
use crate::*;
use std::sync::Arc;
/// Represents the text content within an XML element
#[derive(Clone, Debug)]
pub struct TextContent {
element: Arc<schema::XsElement>,
}
impl TextContent {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self { element }
}
pub fn append(self, c: char) -> Vec<Token> {
if c == '<' {
vec![Token::ElementCloseStart(ElementCloseStart::new(
Arc::clone(&self.element),
))]
} else {
vec![Token::TextContent(TextContent::new(Arc::clone(
&self.element,
)))]
}
}
}
#[cfg(test)]
mod text_content_tests {
use super::*;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_text_content_normal_char() {
let element = create_test_element();
let token = TextContent::new(Arc::clone(&element));
// Test with normal character
let next_tokens = token.append('a');
assert!(!next_tokens.is_empty(), "Should accept normal character");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_text_content_less_than() {
let element = create_test_element();
let token = TextContent::new(Arc::clone(&element));
// Test with < (start of tag)
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should accept '<'");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
}