Fixed attribute parsing

lib/xml_schema_validator/example_schema.xsd

@@ -0,0 +1,118 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Always answer with a single element.
+-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
+
+  <!--
+    Delete command removes an entry from the context by its ID.
+    Use it to remove unnecessary items and stop background processes.
+    When you delete something, it is gone.
+    Make sure all important info is stored in files.
+    Example:
+    <delete id="1234567890"/>
+  -->
+  <xs:element name="delete">
+    <xs:complexType>
+      <xs:attribute name="id" type="xs:string" use="required"/>
+    </xs:complexType>
+  </xs:element>
+
+  <!--
+    Stop command terminates the agent gracefully.
+    For the main SIA instance this will trigger an update and restart.
+    For sub-instances this is the correct way to stop after all tasks are complete.
+    Example:
+    <stop id="1234567890"/>
+  -->
+  <xs:element name="stop">
+    <xs:complexType/>
+  </xs:element>
+
+  <!--
+    Single script that runs once and completes.
+    Output is stored in context until explicitly deleted.
+    Used for one-time operations like file manipulation.
+    Single scripts are limited to 1024 characters and 1 second timeout by default.
+    These limits can be changed with attributes.
+    Example:
+    <single>
+      ls /
+    </single>
+  -->
+  <xs:element name="single">
+    <xs:complexType mixed="true">
+      <xs:sequence>
+        <xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
+      </xs:sequence>
+      <xs:attribute name="timeout" type="xs:float" use="optional"/>
+      <xs:attribute name="limit" type="xs:integer" use="optional"/>
+    </xs:complexType>
+  </xs:element>
+
+  <!--
+    Repeat script runs each time the context is generated.
+    After a command is issued, all repeat scripts in context are run again.
+    Useful for monitoring changing files or viewing results immediately after changing a file.
+    Repeat scripts should execute quickly to avoid blocking the agent.
+    Repeat scripts are limited to 1024 characters and 1 second timeout by default.
+    These limits can be changed with attributes.
+    Example:
+    <repeat>
+      ls /
+    </repeat>
+  -->
+  <xs:element name="repeat">
+    <xs:complexType mixed="true">
+      <xs:sequence>
+        <xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
+      </xs:sequence>
+      <xs:attribute name="timeout" type="xs:float" use="optional"/>
+      <xs:attribute name="limit" type="xs:integer" use="optional"/>
+    </xs:complexType>
+  </xs:element>
+
+  <!--
+    As an agent it is important to reason about your actions and their results.
+    In a reasoning action you can write freeform text.
+    This is also stored in context until deleted.
+    Example:
+    <reasoning>
+      I should explore the file system for interesting files.
+    </reasoning>
+  -->
+  <xs:element name="reasoning">
+    <xs:complexType mixed="true">
+      <xs:sequence>
+        <xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+
+  <!--
+    Read all available text on stdin and store it in context.
+    Do this only if the context indicates there is data in the stdin buffer.
+    Example:
+    <read_stdin/>
+  -->
+  <xs:element name="read_stdin">
+    <xs:complexType/>
+  </xs:element>
+
+  <!--
+    Write to stdout.
+    This is your main way of contacting the user.
+    Make sure you have properly reasoned about what to say and if it is necessary before issuing a write_stdout command.
+    Example:
+    <write_stdout>
+      Hello world!
+    </write_stdout>
+  -->
+  <xs:element name="write_stdout">
+    <xs:complexType mixed="true">
+      <xs:sequence>
+        <xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+</xs:schema>

lib/xml_schema_validator/python/xml_schema_validator/__init__.py

@@ -12,7 +12,7 @@ class XmlLogitsProcessor(LogitsProcessor):
     by setting their logits to negative infinity.
     """
     
-    def __init__(self, tokenizer: AutoTokenizer, schema_text: str = None, core=None):
+    def __init__(self, tokenizer: AutoTokenizer, schema_text: str = None, core: XmlLogitsProcessorCore=None):
         """
         Initialize the processor with a schema and tokenizer.
         

lib/xml_schema_validator/src/char_trie.rs

@@ -1,103 +1,116 @@
-use crate::*;
+use crate::*;
-use pyo3::prelude::*;
+use pyo3::prelude::*;
-
+
-// Node in our character trie
+// Node in our character trie
-struct CharTrieNode {
+struct CharTrieNode {
-    // Store character and node pairs, keep sorted for binary search
+    // Store character and node pairs, keep sorted for binary search
-    children: Vec<(char, CharTrieNode)>,
+    children: Vec<(char, CharTrieNode)>,
-    // Store token IDs that are valid at this point
+    // Store token IDs that are valid at this point
-    token_id: Option<Py<PyAny>>,
+    token_id: Option<Py<PyAny>>,
-}
+}
-
+
-impl CharTrieNode {
+impl CharTrieNode {
-    fn new() -> Self {
+    fn new() -> Self {
-        Self {
+        Self {
-            children: Vec::new(),
+            children: Vec::new(),
-            token_id: None,
+            token_id: None,
-        }
+        }
-    }
+    }
-    
+
-    // Add a child for character c, maintaining sorted order
+    // Add a child for character c, maintaining sorted order
-    fn add_child(&mut self, c: char) -> &mut CharTrieNode {
+    fn add_child(&mut self, c: char) -> &mut CharTrieNode {
-        match self.children.binary_search_by_key(&c, |(ch, _)| *ch) {
+        match self.children.binary_search_by_key(&c, |(ch, _)| *ch) {
-            Ok(index) => &mut self.children[index].1,
+            Ok(index) => &mut self.children[index].1,
-            Err(index) => {
+            Err(index) => {
-                // Insert at the right position to maintain sort order
+                // Insert at the right position to maintain sort order
-                self.children.insert(index, (c, CharTrieNode::new()));
+                self.children.insert(index, (c, CharTrieNode::new()));
-                &mut self.children[index].1
+                &mut self.children[index].1
-            }
+            }
-        }
+        }
-    }
+    }
-    
+
-    // Add a token id to this node
+    // Add a token id to this node
-    fn add_token(&mut self, token_id: Py<PyAny>) {
+    fn add_token(&mut self, token_id: Py<PyAny>) {
-        self.token_id = Some(token_id);
+        self.token_id = Some(token_id);
-    }
+    }
-
+
-    pub fn clone_ref(&self, py: Python<'_>) -> Self {
+    pub fn clone_ref(&self, py: Python<'_>) -> Self {
-        Self {
+        Self {
-            children: self.children.iter().map(|(ch, node)| (*ch, node.clone_ref(py))).collect(),
+            children: self
-            token_id: self.token_id.iter().map(|token_id| token_id.clone_ref(py)).nth(0),
+                .children
-        }
+                .iter()
-    }
+                .map(|(ch, node)| (*ch, node.clone_ref(py)))
-}
+                .collect(),
-
+            token_id: self
-// The main trie structure
+                .token_id
-pub struct CharTrie {
+                .iter()
-    root: CharTrieNode,
+                .map(|token_id| token_id.clone_ref(py))
-}
+                .nth(0),
-
+        }
-impl CharTrie {
+    }
-    pub fn new() -> Self {
+}
-        Self {
+
-            root: CharTrieNode::new(),
+// The main trie structure
-        }
+pub struct CharTrie {
-    }
+    root: CharTrieNode,
-    
+}
-    // Insert a token's character sequence into the trie
+
-    pub fn insert(&mut self, token_text: &str, token_id: Py<PyAny>) {
+impl CharTrie {
-        let mut current = &mut self.root;
+    pub fn new() -> Self {
-        
+        Self {
-        // Add each character in the token text
+            root: CharTrieNode::new(),
-        for c in token_text.chars() {
+        }
-            current = current.add_child(c);
+    }
-        }
+
-        
+    // Insert a token's character sequence into the trie
-        // Mark this node as representing token_id
+    pub fn insert(&mut self, token_text: &str, token_id: Py<PyAny>) {
-        current.add_token(token_id);
+        let mut current = &mut self.root;
-    }
+
-    
+        // Add each character in the token text
-    // Find all tokens that could be invalid given the current XML state
+        for c in token_text.chars() {
-    pub fn find_invalid_tokens(&self, py: Python<'_>, xml_validator: &XmlSchemaValidator, token_map: &Vec<(Py<PyAny>, String)>) -> Vec<Py<PyAny>> {
+            current = current.add_child(c);
-        let mut invalid_tokens = Vec::new();
+        }
-        
+
-        for (token_id, token_text) in token_map {
+        // Mark this node as representing token_id
-            // Check if this token would make the XML invalid
+        current.add_token(token_id);
-            if !self.is_valid_continuation(xml_validator, token_text) {
+    }
-                invalid_tokens.push(token_id.clone_ref(py));
+
-            }
+    // Find all tokens that could be invalid given the current XML state
-        }
+    pub fn find_invalid_tokens(
-        
+        &self,
-        invalid_tokens
+        py: Python<'_>,
-    }
+        xml_validator: &XmlSchemaValidator,
-    
+        token_map: &Vec<(Py<PyAny>, String)>,
-    // Check if a token would be a valid continuation
+    ) -> Vec<Py<PyAny>> {
-    fn is_valid_continuation(&self, xml_validator: &XmlSchemaValidator, token_text: &str) -> bool {
+        let mut invalid_tokens = Vec::new();
-        // First, quick check with just the first character
+
-        if let Some(first_char) = token_text.chars().next() {
+        for (token_id, token_text) in token_map {
-            if !xml_validator.can_continue_with(first_char) {
+            // Check if this token would make the XML invalid
-                return false;
+            if !self.is_valid_continuation(xml_validator, token_text) {
-            }
+                invalid_tokens.push(token_id.clone_ref(py));
-        }
+            }
-        
+        }
-        // If first character is valid, try the full token
+
-        let mut validator_clone = xml_validator.clone();
+        invalid_tokens
-        validator_clone.append(token_text).is_ok()
+    }
-    }
+
-
+    // Check if a token would be a valid continuation
-    pub fn clone_ref(&self, py: Python<'_>) -> Self {
+    fn is_valid_continuation(&self, xml_validator: &XmlSchemaValidator, token_text: &str) -> bool {
-        Self {
+        // First, quick check with just the first character
-            root: self.root.clone_ref(py),
+        if let Some(first_char) = token_text.chars().next() {
-        }
+            if !xml_validator.can_continue_with(first_char) {
-    }
+                return false;
-}
+            }
+        }
+
+        // If first character is valid, try the full token
+        let mut validator_clone = xml_validator.clone();
+        validator_clone.append(token_text).is_ok()
+    }
+
+    pub fn clone_ref(&self, py: Python<'_>) -> Self {
+        Self {
+            root: self.root.clone_ref(py),
+        }
+    }
+}

lib/xml_schema_validator/src/error.rs

@@ -1,14 +1,14 @@
-/// Error types that can occur during XML validation
+/// Error types that can occur during XML validation
-
+
-#[derive(Clone, Debug, thiserror::Error)]
+#[derive(Clone, Debug, thiserror::Error)]
-pub enum Error {
+pub enum Error {
-    /// Indicates that the provided XSD schema is invalid
+    /// Indicates that the provided XSD schema is invalid
-    #[error("Invalid schema: {0}")]
+    #[error("Invalid schema: {0}")]
-    InvalidSchema(#[from] quick_xml::DeError),
+    InvalidSchema(#[from] quick_xml::DeError),
-    /// Indicates that the provided XML fragment is invalid
+    /// Indicates that the provided XML fragment is invalid
-    #[error("Invalid XML: {0}")]
+    #[error("Invalid XML: {0}")]
-    InvalidXml(String),
+    InvalidXml(String),
-    /// Not implemented yet
+    /// Not implemented yet
-    #[error("Not implemented yet")]
+    #[error("Not implemented yet")]
-    NotImplemented,
+    NotImplemented,
-}
+}

lib/xml_schema_validator/src/py_xml_logits_processor_core.rs

@@ -1,6 +1,6 @@
+use crate::*;
 use pyo3::prelude::*;
 use pyo3::types::PyDict;
-use crate::*;
 
 /// A minimal LogitsProcessor that enforces valid XML according to a schema.
 #[pyclass]
@@ -18,7 +18,10 @@ impl XmlLogitsProcessorCore {
             Ok(xml_schema_validator) => xml_schema_validator,
             Err(e) => return Err(pyo3::exceptions::PyValueError::new_err(e.to_string())),
         };
-        let tokens: Vec<(Py<PyAny>, String)> = tokens.iter().map(|(id, value)| (id.unbind(), value.unbind().to_string())).collect();
+        let tokens: Vec<(Py<PyAny>, String)> = tokens
+            .iter()
+            .map(|(id, value)| (id.unbind(), value.unbind().to_string()))
+            .collect();
         let mut trie = char_trie::CharTrie::new();
         for (token_id, token_text) in tokens.iter() {
             trie.insert(token_text, token_id.clone_ref(py));
@@ -29,7 +32,7 @@ impl XmlLogitsProcessorCore {
             trie,
         })
     }
-    
+
     /// Append a fragment of XML to the validator
     /// Returns true if the append was successful, false otherwise
     fn append(&mut self, fragment: &str) -> bool {
@@ -38,7 +41,9 @@ impl XmlLogitsProcessorCore {
 
     /// Get a list of tokens that would lead to invalid XML
     fn get_invalid_tokens(&mut self, py: Python<'_>) -> PyResult<Vec<PyObject>> {
-        Ok(self.trie.find_invalid_tokens(py, &self.xml_schema_validator, &self.tokens))
+        Ok(self
+            .trie
+            .find_invalid_tokens(py, &self.xml_schema_validator, &self.tokens))
     }
 
     /// Check if the validator has reached the end of the XML
@@ -50,16 +55,20 @@ impl XmlLogitsProcessorCore {
     fn copy(&self, py: Python<'_>) -> PyResult<Self> {
         Ok(Self {
             xml_schema_validator: self.xml_schema_validator.clone(),
-            tokens: self.tokens.iter().map(|(a, b)|(a.clone_ref(py), b.clone())).collect(),
+            tokens: self
+                .tokens
+                .iter()
+                .map(|(a, b)| (a.clone_ref(py), b.clone()))
+                .collect(),
             trie: self.trie.clone_ref(py),
         })
     }
-    
+
     /// Get a list of all element names in the schema
     fn get_element_names(&self) -> PyResult<Vec<String>> {
         Ok(self.xml_schema_validator.get_element_names())
     }
-    
+
     /// Check if a specific character is valid as the next character
     fn can_continue_with(&self, c: &str) -> PyResult<bool> {
         if let Some(first_char) = c.chars().next() {
@@ -79,8 +88,8 @@ pub fn register_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()>
 #[cfg(test)]
 mod tests {
     use super::*;
-    use pyo3::Python;
     use pyo3::types::PyDict;
+    use pyo3::Python;
 
     fn create_test_processor<'py>(py: Python<'py>) -> PyResult<XmlLogitsProcessorCore> {
         let tokens = PyDict::new(py);
@@ -91,7 +100,7 @@ mod tests {
         tokens.set_item(3, " ")?;
         tokens.set_item(4, "a")?;
         tokens.set_item(5, "</reasoning>")?;
-        
+
         // Simple schema with only reasoning element
         let schema_text = r#"<?xml version="1.0" encoding="UTF-8"?>
         <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
@@ -103,7 +112,7 @@ mod tests {
             </xs:complexType>
           </xs:element>
         </xs:schema>"#;
-        
+
         XmlLogitsProcessorCore::new(py, tokens.into(), schema_text)
     }
 
@@ -111,24 +120,29 @@ mod tests {
     fn test_append_valid_xml() {
         Python::with_gil(|py| {
             let mut processor = create_test_processor(py).unwrap();
-            
+
             // Test various valid XML fragments
             assert!(processor.append("<reasoning>"), "Should accept opening tag");
-            
+
             let mut processor = create_test_processor(py).unwrap();
-            assert!(processor.append("<reasoning>test</reasoning>"), 
+            assert!(
-                   "Should accept complete element with content");
+                processor.append("<reasoning>test</reasoning>"),
-            
+                "Should accept complete element with content"
+            );
+
             // Test with real schema from disk if available
             if let Ok(schema_text) = std::fs::read_to_string("../../action_schema.xsd") {
                 let tokens = PyDict::new(py);
                 tokens.set_item(0, "<").unwrap();
                 tokens.set_item(1, "reasoning").unwrap();
                 tokens.set_item(2, ">").unwrap();
-                
+
-                let mut processor = XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap();
+                let mut processor =
-                assert!(processor.append("<reasoning>"), 
+                    XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap();
-                       "Should accept <reasoning> with actual schema");
+                assert!(
+                    processor.append("<reasoning>"),
+                    "Should accept <reasoning> with actual schema"
+                );
             }
         });
     }
@@ -137,13 +151,18 @@ mod tests {
     fn test_append_invalid_xml() {
         Python::with_gil(|py| {
             let mut processor = create_test_processor(py).unwrap();
-            
+
             // Test invalid XML fragments
-            assert!(!processor.append("<invalid>"), "Should reject invalid element");
+            assert!(
-            
+                !processor.append("<invalid>"),
+                "Should reject invalid element"
+            );
+
             let mut processor = create_test_processor(py).unwrap();
-            assert!(!processor.append("<reasoning></invalid>"), 
+            assert!(
-                   "Should reject mismatched tags");
+                !processor.append("<reasoning></invalid>"),
+                "Should reject mismatched tags"
+            );
         });
     }
 
@@ -151,25 +170,36 @@ mod tests {
     fn test_eof_detection() {
         Python::with_gil(|py| {
             let processor = create_test_processor(py).unwrap();
-            
+
             // At start, not at EOF
             assert!(!processor.eof().unwrap(), "Should not be at EOF at start");
-            
+
             // After complete document, should be at EOF
             let mut processor = create_test_processor(py).unwrap();
             // Use a properly formed XML document that matches the schema
-            assert!(processor.append("<reasoning>test</reasoning>"), 
+            assert!(
-                "Should accept complete document");
+                processor.append("<reasoning>test</reasoning>"),
-            
+                "Should accept complete document"
+            );
+
             // DEBUG: Print the token state to see why EOF detection fails
-            println!("Current tokens: {:?}", processor.xml_schema_validator.current_tokens);
+            println!(
-            
+                "Current tokens: {:?}",
-            assert!(processor.eof().unwrap(), "Should be at EOF after complete document");
+                processor.xml_schema_validator.current_tokens
-            
+            );
+
+            assert!(
+                processor.eof().unwrap(),
+                "Should be at EOF after complete document"
+            );
+
             // After opening tag, should not be at EOF
             let mut processor = create_test_processor(py).unwrap();
             assert!(processor.append("<reasoning>"), "Should accept opening tag");
-            assert!(!processor.eof().unwrap(), "Should not be at EOF after opening tag");
+            assert!(
+                !processor.eof().unwrap(),
+                "Should not be at EOF after opening tag"
+            );
         });
     }
 
@@ -177,20 +207,30 @@ mod tests {
     fn test_can_continue_with() {
         Python::with_gil(|py| {
             let processor = create_test_processor(py).unwrap();
-            
+
             // At start, only '<' is valid
-            assert!(processor.can_continue_with("<").unwrap(), "Should accept '<' at start");
+            assert!(
-            assert!(!processor.can_continue_with("a").unwrap(), "Should reject 'a' at start");
+                processor.can_continue_with("<").unwrap(),
-            
+                "Should accept '<' at start"
+            );
+            assert!(
+                !processor.can_continue_with("a").unwrap(),
+                "Should reject 'a' at start"
+            );
+
             // Test after opening tag
             let mut processor = create_test_processor(py).unwrap();
             assert!(processor.append("<reasoning>"), "Should accept opening tag");
-            
+
             // In mixed content, any character should be valid
-            assert!(processor.can_continue_with("a").unwrap(), 
+            assert!(
-                   "Should accept 'a' in mixed content");
+                processor.can_continue_with("a").unwrap(),
-            assert!(processor.can_continue_with("<").unwrap(), 
+                "Should accept 'a' in mixed content"
-                   "Should accept '<' in mixed content");
+            );
+            assert!(
+                processor.can_continue_with("<").unwrap(),
+                "Should accept '<' in mixed content"
+            );
         });
     }
 
@@ -198,14 +238,17 @@ mod tests {
     fn test_copy() {
         Python::with_gil(|py| {
             let processor = create_test_processor(py).unwrap();
-            
+
             // Make a copy
             let copy = processor.copy(py).unwrap();
-            
+
             // Original and copy should behave the same
-            assert_eq!(processor.eof().unwrap(), copy.eof().unwrap(), 
+            assert_eq!(
-                      "Copy should have same EOF state");
+                processor.eof().unwrap(),
-            
+                copy.eof().unwrap(),
+                "Copy should have same EOF state"
+            );
+
             // Modify copy, original should be unchanged
             let mut copy = processor.copy(py).unwrap();
             assert!(copy.append("<reasoning>"), "Copy should accept XML");
@@ -218,28 +261,32 @@ mod tests {
     fn test_get_element_names() {
         Python::with_gil(|py| {
             let processor = create_test_processor(py).unwrap();
-            
+
             let names = processor.get_element_names().unwrap();
-            assert!(names.contains(&"reasoning".to_string()), 
+            assert!(
-                   "Should contain 'reasoning' element");
+                names.contains(&"reasoning".to_string()),
+                "Should contain 'reasoning' element"
+            );
             assert_eq!(names.len(), 1, "Should have exactly one element");
-            
+
             let tokens = PyDict::new(py);
             tokens.set_item(0, "<").unwrap();
-            
+
             let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
             let processor = XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap();
             let names = processor.get_element_names().unwrap();
-            
+
             // Print out all elements from schema for debugging
             println!("Elements in schema: {:?}", names);
-            
+
             // Check if reasoning exists
-            assert!(names.contains(&"reasoning".to_string()), 
+            assert!(
-                    "Real schema should contain 'reasoning'");
+                names.contains(&"reasoning".to_string()),
+                "Real schema should contain 'reasoning'"
+            );
         });
     }
-    
+
     #[test]
     fn test_debug_reasoning_tag_validation() {
         Python::with_gil(|py| {
@@ -254,7 +301,7 @@ mod tests {
                 </xs:complexType>
               </xs:element>
             </xs:schema>"#;
-            
+
             // Create a minimal token dictionary
             let tokens = PyDict::new(py);
             tokens.set_item(0, "<").unwrap();
@@ -268,56 +315,67 @@ mod tests {
             tokens.set_item(8, "n").unwrap();
             tokens.set_item(9, "g").unwrap();
             tokens.set_item(10, ">").unwrap();
-            
+
             // Create processor with this schema and tokens
-            let processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
+            let processor =
-            
+                XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
+
             // Test basic operations to verify core functionality works
             let element_names = processor.get_element_names().unwrap();
             println!("Element names in test schema: {:?}", element_names);
-            assert!(element_names.contains(&"reasoning".to_string()), 
+            assert!(
-                   "Test schema should contain 'reasoning'");
+                element_names.contains(&"reasoning".to_string()),
-            
+                "Test schema should contain 'reasoning'"
+            );
+
             // Now try to append each character of "<reasoning>" one by one
             let chars = vec!['<', 'r', 'e', 'a', 's', 'o', 'n', 'i', 'n', 'g', '>'];
             let mut full_text = String::new();
             let processor = processor.copy(py).unwrap();
-            
+
             for c in chars {
                 full_text.push(c);
                 let fragment = c.to_string();
                 let result = processor.can_continue_with(&fragment).unwrap();
-                println!("Can continue with '{}' after '{}': {}", 
+                println!(
-                        c, &full_text[..full_text.len()-1], result);
+                    "Can continue with '{}' after '{}': {}",
-                
+                    c,
+                    &full_text[..full_text.len() - 1],
+                    result
+                );
+
                 // Create a new processor to test appending the fragment built so far
-                let mut fresh_processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
+                let mut fresh_processor =
+                    XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
                 let append_result = fresh_processor.append(&full_text);
                 println!("Appending full text '{}': {}", full_text, append_result);
             }
-            
+
             // Now test with actual schema from disk if available
             if let Ok(actual_schema) = std::fs::read_to_string("../../action_schema.xsd") {
                 println!("\nTesting with actual schema from disk:");
-                
+
-                let processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap();
+                let processor =
+                    XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap();
                 let element_names = processor.get_element_names().unwrap();
                 println!("Element names in actual schema: {:?}", element_names);
-                
+
                 if element_names.contains(&"reasoning".to_string()) {
                     println!("'reasoning' element found in actual schema");
-                    
+
                     // Test appending "<reasoning>" to a fresh processor
                     let mut processor = processor.copy(py).unwrap();
                     let result = processor.append("<reasoning>");
                     println!("Appending '<reasoning>' to fresh processor: {}", result);
-                    
+
                     // Test character by character
                     let mut text = String::new();
                     for c in "<reasoning>".chars() {
                         text.push(c);
-                        let result = XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap()
+                        let result =
-                                      .append(&text);
+                            XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema)
+                                .unwrap()
+                                .append(&text);
                         println!("Appending '{}' to fresh processor: {}", text, result);
                     }
                 } else {
@@ -326,4 +384,4 @@ mod tests {
             }
         });
     }
-}
+}

lib/xml_schema_validator/src/py_xml_schema_validator.rs

@@ -1,71 +1,71 @@
-use pyo3::prelude::*;
+use pyo3::exceptions::PyValueError;
-use pyo3::exceptions::PyValueError;
+use pyo3::prelude::*;
-
+
-/// Python wrapper for XmlValidator
+/// Python wrapper for XmlValidator
-#[pyclass]
+#[pyclass]
-struct XmlSchemaValidator {
+struct XmlSchemaValidator {
-    validator: crate::XmlSchemaValidator,
+    validator: crate::XmlSchemaValidator,
-}
+}
-
+
-#[pymethods]
+#[pymethods]
-impl XmlSchemaValidator {
+impl XmlSchemaValidator {
-    #[new]
+    #[new]
-    fn new(schema_text: &str) -> PyResult<Self> {
+    fn new(schema_text: &str) -> PyResult<Self> {
-        let validator = crate::XmlSchemaValidator::new(schema_text)
+        let validator = crate::XmlSchemaValidator::new(schema_text)
-            .map_err(|e| PyValueError::new_err(format!("Failed to create validator: {}", e)))?;
+            .map_err(|e| PyValueError::new_err(format!("Failed to create validator: {}", e)))?;
-        
+
-        Ok(Self { validator })
+        Ok(Self { validator })
-    }
+    }
-
+
-    /// Append a fragment of XML to the validator
+    /// Append a fragment of XML to the validator
-    /// Returns a tuple containing:
+    /// Returns a tuple containing:
-    /// - A boolean indicating success
+    /// - A boolean indicating success
-    /// - An optional error message
+    /// - An optional error message
-    fn append(&mut self, fragment: &str) -> PyResult<(bool, Option<String>)> {
+    fn append(&mut self, fragment: &str) -> PyResult<(bool, Option<String>)> {
-        match self.validator.append(fragment) {
+        match self.validator.append(fragment) {
-            Ok(()) => PyResult::Ok((true, None)),
+            Ok(()) => PyResult::Ok((true, None)),
-            Err(e) => PyResult::Ok((false, Some(e.to_string()))),
+            Err(e) => PyResult::Ok((false, Some(e.to_string()))),
-        }
+        }
-    }
+    }
-
+
-    /// Check if the validator has reached the end of the XML
+    /// Check if the validator has reached the end of the XML
-    fn eof(&self) -> PyResult<bool> {
+    fn eof(&self) -> PyResult<bool> {
-        Ok(self.validator.eof())
+        Ok(self.validator.eof())
-    }
+    }
-
+
-    /// Create a deep copy of this validator
+    /// Create a deep copy of this validator
-    fn copy(&self) -> PyResult<Self> {
+    fn copy(&self) -> PyResult<Self> {
-        Ok(Self {
+        Ok(Self {
-            validator: self.validator.clone(),
+            validator: self.validator.clone(),
-        })
+        })
-    }
+    }
-    
+
-    /// Get a list of all element names in the schema
+    /// Get a list of all element names in the schema
-    fn get_element_names(&self) -> PyResult<Vec<String>> {
+    fn get_element_names(&self) -> PyResult<Vec<String>> {
-        Ok(self.validator.get_element_names())
+        Ok(self.validator.get_element_names())
-    }
+    }
-    
+
-    /// Check if a specific character is valid as the next character
+    /// Check if a specific character is valid as the next character
-    fn can_continue_with(&self, c: &str) -> PyResult<bool> {
+    fn can_continue_with(&self, c: &str) -> PyResult<bool> {
-        if let Some(first_char) = c.chars().next() {
+        if let Some(first_char) = c.chars().next() {
-            Ok(self.validator.can_continue_with(first_char))
+            Ok(self.validator.can_continue_with(first_char))
-        } else {
+        } else {
-            Ok(false)
+            Ok(false)
-        }
+        }
-    }
+    }
-    
+
-    /// Validate an entire XML string and check if it's valid according to the schema
+    /// Validate an entire XML string and check if it's valid according to the schema
-    fn validate(&self, xml_string: &str) -> PyResult<bool> {
+    fn validate(&self, xml_string: &str) -> PyResult<bool> {
-        let mut validator_clone = self.validator.clone();
+        let mut validator_clone = self.validator.clone();
-        match validator_clone.append(xml_string) {
+        match validator_clone.append(xml_string) {
-            Ok(()) => Ok(validator_clone.eof()),  // Valid only if we reached EOF
+            Ok(()) => Ok(validator_clone.eof()), // Valid only if we reached EOF
-            Err(_) => Ok(false),  // Invalid XML
+            Err(_) => Ok(false),                 // Invalid XML
-        }
+        }
-    }
+    }
-}
+}
-
+
-/// Register the class with the Python module
+/// Register the class with the Python module
-pub fn register_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
+pub fn register_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
-    m.add_class::<XmlSchemaValidator>()?;
+    m.add_class::<XmlSchemaValidator>()?;
-    Ok(())
+    Ok(())
-}
+}

lib/xml_schema_validator/src/python.rs

@@ -1,9 +1,9 @@
-use pyo3::prelude::*;
+use pyo3::prelude::*;
-
+
-/// Python module for XML Schema validation
+/// Python module for XML Schema validation
-#[pymodule]
+#[pymodule]
-fn _rs(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
+fn _rs(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
-    crate::py_xml_logits_processor_core::register_module(py, m)?;
+    crate::py_xml_logits_processor_core::register_module(py, m)?;
-    crate::py_xml_schema_validator::register_module(py, m)?;
+    crate::py_xml_schema_validator::register_module(py, m)?;
-    Ok(())
+    Ok(())
-}
+}

lib/xml_schema_validator/src/schema.rs

@@ -1,6 +1,14 @@
 use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    Eq,
+    Hash,
+    PartialEq,
+    Serialize,
+)]
 pub struct XsSchema {
     #[serde(rename = "@xmlns:xs")]
     pub xmlns_xs: String,
@@ -12,7 +20,15 @@ pub struct XsSchema {
     pub xs_element: Vec<XsElement>,
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    Eq,
+    Hash,
+    PartialEq,
+    Serialize,
+)]
 pub struct XsElement {
     #[serde(rename = "@name")]
     pub name: String,
@@ -22,7 +38,15 @@ pub struct XsElement {
     pub xs_complex_type: XsComplexType,
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    Eq,
+    Hash,
+    PartialEq,
+    Serialize,
+)]
 pub struct XsComplexType {
     #[serde(rename = "@mixed")]
     pub mixed: Option<String>,
@@ -34,7 +58,15 @@ pub struct XsComplexType {
     pub xs_sequence: Option<XsSequence>,
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    Eq,
+    Hash,
+    PartialEq,
+    Serialize,
+)]
 pub struct XsAttribute {
     #[serde(rename = "@name")]
     pub name: String,
@@ -44,7 +76,15 @@ pub struct XsAttribute {
     pub xs_attribute_use: String,
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    Eq,
+    Hash,
+    PartialEq,
+    Serialize,
+)]
 pub struct XsSequence {
     #[serde(rename = "$text")]
     pub text: Option<String>,
@@ -52,7 +92,15 @@ pub struct XsSequence {
     pub xs_any: XsAny,
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    Eq,
+    Hash,
+    PartialEq,
+    Serialize,
+)]
 pub struct XsAny {
     #[serde(rename = "@minOccurs")]
     pub min_occurs: String,
@@ -61,4 +109,3 @@ pub struct XsAny {
     #[serde(rename = "@processContents")]
     pub process_contents: String,
 }
-

lib/xml_schema_validator/src/token/attribute_equals.rs

@@ -0,0 +1,102 @@
+use crate::*;
+use std::sync::Arc;
+
+/// Represents the equals sign after an XML attribute name
+#[derive(Clone, Debug)]
+pub struct AttributeEquals {
+    element: Arc<schema::XsElement>,
+    used_attributes: Vec<schema::XsAttribute>,
+}
+
+impl AttributeEquals {
+    pub fn new(
+        element: Arc<schema::XsElement>,
+        used_attributes: Vec<schema::XsAttribute>,
+    ) -> Self {
+        Self {
+            element,
+            used_attributes,
+        }
+    }
+
+    pub fn append(self, c: char) -> Vec<Token> {
+        if c == '"' {
+            // Opening quote starts the attribute value
+            return vec![Token::AttributeValue(AttributeValue::new(
+                Arc::clone(&self.element),
+                self.used_attributes,
+            ))];
+        } else if c.is_whitespace() {
+            // Allow whitespace between equals sign and opening quote
+            return vec![Token::AttributeEquals(self)];
+        } else {
+            // Any other character is invalid
+            return vec![];
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_attribute_equals_quote() {
+        let values = [
+            "<delete id=\"",
+            "<single timeout=\"",
+            "<single limit=\"",
+        ];
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            assert!(
+                validator
+                    .current_tokens
+                    .iter()
+                    .filter(|token| matches!(token, Token::AttributeValue(_)))
+                    .next()
+                    .is_some(),
+                "{value}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_attribute_equals_with_whitespace() {
+        let values = [
+            "<delete id= \"",
+            "<single timeout = \"",
+            "<single limit  \t  =   \n \"",
+        ];
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            assert!(
+                validator
+                    .current_tokens
+                    .iter()
+                    .filter(|token| matches!(token, Token::AttributeValue(_)))
+                    .next()
+                    .is_some(),
+                "{value}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_invalid_after_equals() {
+        let values = [
+            "<delete id=a",  // Only quote should be accepted after equals
+            "<single timeout=<", // Invalid character after equals
+            "<single timeout=1", // Unquoted numbers not supported yet
+        ];
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect_err(value);
+            assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
+        }
+    }
+}

lib/xml_schema_validator/src/token/attribute_name.rs

@@ -1,196 +1,165 @@
-use std::sync::Arc;
+use crate::*;
-use crate::*;
+use std::sync::Arc;
-
+
-/// Represents an attribute name of an XML element
+/// Represents an XML attribute name that is in the process of being parsed
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug)]
-pub struct AttributeName {
+pub struct AttributeName {
-    /// The element that owns this attribute
+    buffer: String,
-    element: Arc<schema::XsElement>,
+    element: Arc<schema::XsElement>,
-    /// Buffer containing the attribute name characters processed so far
+    used_attributes: Vec<schema::XsAttribute>,
-    buffer: String,
+}
-    /// Track which attributes have already been set for this element
+
-    attributes_set: Vec<String>,
+impl AttributeName {
-}
+    pub fn new(first_char: char, element: Arc<schema::XsElement>) -> Self {
-
+        Self {
-impl AttributeName {
+            buffer: first_char.to_string(),
-    /// Create a new AttributeName with an empty buffer
+            element,
-    pub fn new(element: Arc<schema::XsElement>) -> Self {
+            used_attributes: Vec::new(),
-        Self {
+        }
-            element,
+    }
-            buffer: String::new(),
+
-            attributes_set: Vec::new(),
+    pub fn new_with_used(first_char: char, element: Arc<schema::XsElement>, used_attributes: Vec<schema::XsAttribute>) -> Self {
-        }
+        Self {
-    }
+            buffer: first_char.to_string(),
-    
+            element,
-    /// Create a new AttributeName with the given buffer
+            used_attributes,
-    pub fn with_buffer(element: Arc<schema::XsElement>, buffer: String, attributes_set: Vec<String>) -> Self {
+        }
-        Self {
+    }
-            element,
+
-            buffer,
+    pub fn append(mut self, c: char) -> Vec<Token> {
-            attributes_set,
+        if c == '=' {
-        }
+            // Check if we've matched a valid attribute name
-    }
+            if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
-    
+                if let Some(attribute) = attributes
-    /// Create a new AttributeName with attributes already set
+                    .iter()
-    pub fn with_attributes(element: Arc<schema::XsElement>, attributes_set: Vec<String>) -> Self {
+                    .find(|attribute| attribute.name == self.buffer)
-        Self {
+                {
-            element,
+                    // Check if this attribute has already been used
-            buffer: String::new(),
+                    if self.used_attributes.contains(&attribute) {
-            attributes_set,
+                        return vec![];
-        }
+                    }
-    }
+                    
-
+                    // Add this attribute to the used set
-    /// Append a character to the attribute name and return possible continuations
+                    self.used_attributes.push(attribute.clone());
-    pub fn append(self, c: char) -> Vec<Token> {
+                    
-        if c.is_whitespace() && self.buffer.is_empty() {
+                    return vec![Token::AttributeEquals(AttributeEquals::new(
-            return vec![Token::AttributeName(self)];
+                        Arc::clone(&self.element),
-        } else if c == '=' {
+                        self.used_attributes,
-            // Found the equals sign, transition to attribute value
+                    ))];
-            // We need to verify that this attribute name is valid for this element
+                }
-            if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
+            }
-                for attr in attributes {
+            return vec![];
-                    if attr.name == self.buffer {
+        } else if c.is_whitespace() {
-                        // Valid attribute, transition to attribute value
+            // Check if we've matched a valid attribute name and allow whitespace after it
-                        return vec![Token::AttributeValue(AttributeValue::new(
+            if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
-                            Arc::clone(&self.element),
+                if attributes
-                            self.buffer,
+                    .iter()
-                            self.attributes_set,
+                    .any(|attribute| attribute.name == self.buffer)
-                        ))];
+                {
-                    }
+                    // Check if this attribute has already been used
-                }
+                    if self.used_attributes.iter().filter(|attribute| attribute.name == self.buffer).next().is_some() {
-            }
+                        return vec![];
-            // Invalid attribute name
+                    }
-            vec![]
+                    
-        } else {
+                    return vec![Token::AttributeName(self)];
-            // Continue building the attribute name
+                }
-            let new_buffer = self.buffer + &c.to_string();
+            }
-            
+            return vec![];
-            // Check if this could be a valid attribute for this element
+        } else {
-            if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
+            // Add character to buffer and check if we're still building a valid attribute name
-                for attr in attributes {
+            let new_buffer = self.buffer.clone() + &c.to_string();
-                    if attr.name.starts_with(&new_buffer) {
+            
-                        // This could be a valid attribute, continue parsing
+            if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
-                        return vec![Token::AttributeName(AttributeName::with_buffer(
+                // Check if any attribute name starts with our buffer
-                            Arc::clone(&self.element),
+                if attributes
-                            new_buffer,
+                    .iter()
-                            self.attributes_set.clone(),
+                    .any(|attribute| attribute.name.starts_with(&new_buffer))
-                        ))];
+                {
-                    }
+                    return vec![Token::AttributeName(AttributeName {
-                }
+                        buffer: new_buffer,
-            }
+                        element: Arc::clone(&self.element),
-            
+                        used_attributes: self.used_attributes,
-            // No matching attribute found
+                    })];
-            vec![]
+                }
-        }
+            }
-    }
+            
-}
+            // Character doesn't match what we expect for this attribute name
-
+            return vec![];
-#[cfg(test)]
+        }
-mod attribute_name_tests {
+    }
-    use super::*;
+}
-    use std::sync::Arc;
+
-    use crate::schema;
+#[cfg(test)]
-
+mod tests {
-    fn create_element_with_attributes() -> Arc<schema::XsElement> {
+    use super::*;
-        let element = schema::XsElement {
+
-            name: "delete".to_string(),
+    #[test]
-            text: None,
+    fn test_valid_attribute_name() {
-            xs_complex_type: schema::XsComplexType {
+        let values = [
-                mixed: None,
+            "<delete id=",
-                text: None,
+            "<single timeout=",
-                xs_attribute: Some(vec![
+            "<single limit=",
-                    schema::XsAttribute {
+        ];
-                        name: "id".to_string(),
+        for value in values {
-                        xs_attribute_type: "xs:string".to_string(),
+            let mut validator = crate::tests::example_validator();
-                        xs_attribute_use: "required".to_string(),
+            validator.append(value).expect(value);
-                    }
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-                ]),
+            assert!(
-                xs_sequence: None,
+                validator
-            },
+                    .current_tokens
-        };
+                    .iter()
-        Arc::new(element)
+                    .filter(|token| matches!(token, Token::AttributeEquals(_)))
-    }
+                    .next()
-
+                    .is_some(),
-    #[test]
+                "{value}"
-    fn test_attribute_name_append() {
+            );
-        let element = create_element_with_attributes();
+        }
-        let token = AttributeName::new(Arc::clone(&element));
+    }
-        
+
-        // Build attribute name character by character
+    #[test]
-        let mut current_token = token;
+    fn test_attribute_name_with_whitespace() {
-        for c in "id".chars() {
+        let values = [
-            let next_tokens = current_token.append(c);
+            "<delete id =",
-            assert!(!next_tokens.is_empty(), "Should accept attribute name character '{}'", c);
+            "<single timeout  =",
-            
+            "<single limit\t=",
-            match &next_tokens[0] {
+        ];
-                Token::AttributeName(next) => current_token = next.clone(),
+        for value in values {
-                _ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
+            let mut validator = crate::tests::example_validator();
-            }
+            validator.append(value).expect(value);
-        }
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-        
+            assert!(
-        // Test equals sign
+                validator
-        let next_tokens = current_token.append('=');
+                    .current_tokens
-        assert!(!next_tokens.is_empty(), "Should accept '=' after attribute name");
+                    .iter()
-        
+                    .filter(|token| matches!(token, Token::AttributeEquals(_)))
-        match &next_tokens[0] {
+                    .next()
-            Token::AttributeValue(_) => (),
+                    .is_some(),
-            _ => panic!("Expected AttributeValue, got {:?}", next_tokens[0]),
+                "{value}"
-        }
+            );
-    }
+        }
-
+    }
-    #[test]
+
-    fn test_attribute_name_whitespace() {
+    #[test]
-        let element = create_element_with_attributes();
+    fn test_invalid_attribute_name() {
-        let token = AttributeName::new(Arc::clone(&element));
+        let values = [
-        
+            "<delete invalid=",
-        // Test with whitespace
+            "<single wrongattr=",
-        let next_tokens = token.append(' ');
+        ];
-        assert!(!next_tokens.is_empty(), "Should accept whitespace in attribute name");
+        for value in values {
-        
+            let mut validator = crate::tests::example_validator();
-        match &next_tokens[0] {
+            validator.append(value).expect_err(value);
-            Token::AttributeName(_) => (),
+            assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
-            _ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
+        }
-        }
+    }
-    }
+
-
+    #[test]
-    #[test]
+    fn test_duplicate_attribute() {
-    fn test_invalid_attribute_name() {
+        let mut validator = crate::tests::example_validator();
-        let element = create_element_with_attributes();
+        // First attribute is valid
-        let token = AttributeName::new(Arc::clone(&element));
+        validator.append("<delete id=\"123\" ").expect("First attribute should be accepted");
-        
+        
-        // Test with invalid attribute name
+        // Try to add the same attribute again
-        let next_tokens = token.append('x'); // 'x' doesn't start any valid attribute
+        let result = validator.append("id=");
-        assert!(next_tokens.is_empty(), "Should reject invalid attribute name");
+        assert!(result.is_err(), "Duplicate attribute should be rejected");
-    }
+    }
-    
-    #[test]
-    fn test_attribute_tracking() {
-        let element = create_element_with_attributes();
-        
-        // Create AttributeName with pre-existing attributes
-        let attributes_set = vec!["other".to_string()];
-        let token = AttributeName::with_attributes(Arc::clone(&element), attributes_set.clone());
-        
-        // Build attribute name and transition to AttributeValue
-        let mut current_token = token;
-        for c in "id".chars() {
-            let next_tokens = current_token.append(c);
-            match &next_tokens[0] {
-                Token::AttributeName(next) => current_token = next.clone(),
-                _ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
-            }
-        }
-        
-        // Test equals sign
-        let next_tokens = current_token.append('=');
-        
-        match &next_tokens[0] {
-            Token::AttributeValue(value) => {
-                // Verify that pre-existing attributes are preserved
-                assert!(value.attributes_set.contains(&"other".to_string()), 
-                        "Pre-existing attribute should be preserved");
-            },
-            _ => panic!("Expected AttributeValue, got {:?}", next_tokens[0]),
-        }
-    }
 }

lib/xml_schema_validator/src/token/attribute_value.rs

@@ -1,253 +1,191 @@
-use std::sync::Arc;
+use crate::*;
-use crate::*;
+
-
+/// Represents an attribute value of an XML element
-/// Represents an attribute value of an XML element
+#[derive(Clone, Debug)]
-#[derive(Clone, Debug)]
+pub struct AttributeValue {
-pub struct AttributeValue {
+    /// The element that owns this attribute
-    /// The element that owns this attribute
+    element: Arc<schema::XsElement>,
-    element: Arc<schema::XsElement>,
+    /// Buffer containing the attribute value characters processed so far
-    /// The name of the attribute being processed
+    buffer: String,
-    attribute_name: String,
+    /// Track which attributes have already been set for this element
-    /// Buffer containing the attribute value characters processed so far
+    pub attributes_set: Vec<schema::XsAttribute>,
-    buffer: String,
+}
-    /// Whether we've encountered the opening quote
+
-    in_quotes: bool,
+impl AttributeValue {
-    /// Whether we've seen the closing quote
+    /// Create a new AttributeValue with a list of attributes already set
-    closed: bool,
+    pub fn new(
-    /// Track which attributes have already been set for this element
+        element: Arc<schema::XsElement>,
-    pub attributes_set: Vec<String>,
+        attributes_set: Vec<schema::XsAttribute>,
-}
+    ) -> Self {
-
+        Self {
-impl AttributeValue {
+            element,
-    /// Create a new AttributeValue with a list of attributes already set
+            buffer: "\"".to_string(),
-    pub fn new(element: Arc<schema::XsElement>, attribute_name: String, attributes_set: Vec<String>) -> Self {
+            attributes_set,
-        Self {
+        }
-            element,
+    }
-            attribute_name,
+
-            buffer: String::new(),
+    /// Append a character to the attribute value and return possible continuations
-            in_quotes: false,
+    pub fn append(mut self, c: char) -> Vec<Token> {
-            closed: false,
+        // Handle escaped characters
-            attributes_set,
+        if Self::in_escape_sequence(&self.buffer) {
-        }
+            self.buffer.push(c);
-    }
+            return vec![Token::AttributeValue(self)];
-
+        } else if self.value_closed() {
-    /// Append a character to the attribute value and return possible continuations
+            return self.next_tokens(c);
-    pub fn append(self, c: char) -> Vec<Token> {
+        } else if '"' == c {
-        if self.closed {
+            self.buffer.push(c);
-            // We've already closed the attribute value with a quote
+            return vec![Token::AttributeValue(self)];
-            
+        } else {
-            // Add this attribute to the list of attributes set
+            self.buffer.push(c);
-            let mut new_attributes = self.attributes_set.clone();
+            return vec![Token::AttributeValue(self)];
-            if !new_attributes.contains(&self.attribute_name) {
+        }
-                new_attributes.push(self.attribute_name.clone());
+    }
-            }
+
-            
+    /// Checks if an XML character entity reference is in progress and correctly terminated
-            if c.is_whitespace() {
+    /// Returns true if the sequence is complete (ends with a semicolon)
-                // Check if there are more attributes to parse
+    pub fn in_escape_sequence(buffer: &str) -> bool {
-                if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
+        if let Some(amp_pos) = buffer.rfind('&') {
-                    // If there are other required attributes that haven't been set yet
+            if buffer[amp_pos..].find(';').is_some() {
-                    for attr in attributes {
+                return false;
-                        if attr.name != self.attribute_name && attr.xs_attribute_use == "required" && !new_attributes.contains(&attr.name) {
+            } else {
-                            return vec![Token::AttributeName(AttributeName::with_attributes(
+                return true;
-                                Arc::clone(&self.element),
+            }
-                                new_attributes,
+        }
-                            ))];
+        false
-                        }
+    }
-                    }
+
-                }
+    fn value_closed(&self) -> bool {
-                
+        if self.buffer.len() > 1 && self.buffer.trim().ends_with("\"") {
-                // No more required attributes, whitespace after attribute value, continue to next state
+            let trimmed = self.buffer.trim();
-                return vec![
+            let trimmed = &trimmed[..trimmed.len() - 2];
-                    Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes.clone())),
+            if Self::in_escape_sequence(trimmed) {
-                    Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))
+                false
-                ];
+            } else {
-            } else if c == '>' {
+                true
-                // End of opening tag
+            }
-                return vec![Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes))];
+        } else {
-            } else if c == '/' {
+            false
-                // Beginning of self-closing tag
+        }
-                return vec![Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))];
+    }
-            } else {
+
-                // Unexpected character after attribute value
+    fn next_tokens(mut self, c: char) -> Vec<Token> {
-                return vec![];
+        let all_required_attributes_set: bool = self
-            }
+            .element
-        }
+            .as_ref()
-        
+            .xs_complex_type
-        if !self.in_quotes {
+            .xs_attribute
-            if c.is_whitespace() {
+            .as_deref()
-                // Skip whitespace before the quotes
+            .unwrap()
-                return vec![Token::AttributeValue(Self {
+            .iter()
-                    element: Arc::clone(&self.element),
+            .all(|attr| {
-                    attribute_name: self.attribute_name,
+                if attr.xs_attribute_use == "required" {
-                    buffer: self.buffer,
+                    self.attributes_set.contains(attr)
-                    in_quotes: false,
+                } else {
-                    closed: false,
+                    true
-                    attributes_set: self.attributes_set,
+                }
-                })];
+            });
-            } else if c == '"' || c == '\'' {
+        let all_attributes_set: bool = self
-                // Start of quotes
+            .element
-                return vec![Token::AttributeValue(Self {
+            .as_ref()
-                    element: Arc::clone(&self.element),
+            .xs_complex_type
-                    attribute_name: self.attribute_name,
+            .xs_attribute
-                    buffer: self.buffer,
+            .as_deref()
-                    in_quotes: true,
+            .unwrap()
-                    closed: false,
+            .iter()
-                    attributes_set: self.attributes_set,
+            .all(|attr| self.attributes_set.contains(attr));
-                })];
+        if c.is_whitespace() {
-            } else {
+            self.buffer.push(c);
-                // Unexpected character before quotes
+            vec![Token::AttributeValue(self)]
-                return vec![];
+        } else {
-            }
+            let mut tokens = vec![];
-        } else {
+            if all_required_attributes_set {
-            // We're inside quotes
+                if '/' == c {
-            if c == '"' || c == '\'' {
+                    tokens.push(Token::ElementSelfClose(ElementSelfClose::new()));
-                // End of quotes, validate the attribute value
+                } else if '>' == c {
-                // In a real implementation, we'd check if the value is valid for this attribute type
+                    tokens.push(Token::ElementOpenEnd(ElementOpenEnd::new(self.element.clone())));
-                return vec![Token::AttributeValue(Self {
+                }
-                    element: Arc::clone(&self.element),
+            }
-                    attribute_name: self.attribute_name,
+            if self.buffer.ends_with(char::is_whitespace) && !all_attributes_set {
-                    buffer: self.buffer,
+                tokens.push(Token::AttributeName(AttributeName::new_with_used(
-                    in_quotes: true,
+                    c,
-                    closed: true,
+                    self.element,
-                    attributes_set: self.attributes_set,
+                    self.attributes_set,
-                })];
+                )));
-            } else {
+            }
-                // Continue building the attribute value
+            tokens
-                let new_buffer = self.buffer + &c.to_string();
+        }
-                return vec![Token::AttributeValue(Self {
+    }
-                    element: Arc::clone(&self.element),
+}
-                    attribute_name: self.attribute_name,
+
-                    buffer: new_buffer,
+#[cfg(test)]
-                    in_quotes: true,
+mod tests {
-                    closed: false,
+    use super::*;
-                    attributes_set: self.attributes_set,
+
-                })];
+    #[test]
-            }
+    fn test_unfinished_attribute_value() {
-        }
+        let values = [
-    }
+            "<delete id=\"12",
-}
+            "<single timeout=\"1.",
-
+            "<single limit=\"50",
-#[cfg(test)]
+        ];
-mod attribute_value_tests {
+        
-    use super::*;
+        for value in values {
-    use std::sync::Arc;
+            let mut validator = crate::tests::example_validator();
-    use crate::schema;
+            validator.append(value).expect(value);
-
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-    fn create_element_with_attributes() -> Arc<schema::XsElement> {
+            
-        let element = schema::XsElement {
+            // Should still be in AttributeValue state
-            name: "delete".to_string(),
+            assert!(
-            text: None,
+                validator
-            xs_complex_type: schema::XsComplexType {
+                    .current_tokens
-                mixed: None,
+                    .iter()
-                text: None,
+                    .any(|token| matches!(token, Token::AttributeValue(_))),
-                xs_attribute: Some(vec![
+                "{value}"
-                    schema::XsAttribute {
+            );
-                        name: "id".to_string(),
+        }
-                        xs_attribute_type: "xs:string".to_string(),
+    }
-                        xs_attribute_use: "required".to_string(),
+
-                    }
+    #[test]
-                ]),
+    fn test_type_validation() {
-                xs_sequence: None,
+        // Test integer validation
-            },
+        let mut validator = crate::tests::example_validator();
-        };
+        validator.append("<single limit=\"123\"").expect("Valid integer");
-        Arc::new(element)
+        
-    }
+        // Test float validation
-
+        validator = crate::tests::example_validator();
-    #[test]
+        validator.append("<single timeout=\"1.5\"").expect("Valid float");
-    fn test_attribute_value_quotes() {
+        
-        let element = create_element_with_attributes();
+        // Test string validation
-        let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
+        validator = crate::tests::example_validator();
-        
+        validator.append("<delete id=\"abc123\"").expect("Valid string");
-        // Test with opening quote
+    }
-        let next_tokens = token.append('"');
+
-        assert!(!next_tokens.is_empty(), "Should accept opening quote");
+    #[test]
-        
+    fn test_multiple_attributes() {
-        let token = next_tokens[0].clone();
+        let values = [
-        match token.clone() {
+            "<single limit=\"100\" timeout=\"1.5\"",
-            Token::AttributeValue(value) => {
+            "<single timeout=\"0.5\" limit=\"200\"",
-                assert!(value.in_quotes, "Should mark as in quotes");
+        ];
-                assert!(!value.closed, "Should not be closed yet");
+        
-            },
+        for value in values {
-            _ => panic!("Expected AttributeValue, got {:?}", token),
+            let mut validator = crate::tests::example_validator();
-        }
+            validator.append(value).expect(value);
-        
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-        // Test with content inside quotes
+        }
-        let next_tokens = token.append('a');
+    }
-        assert!(!next_tokens.is_empty(), "Should accept content inside quotes");
+
-        
+    #[test]
-        let token = next_tokens[0].clone();
+    fn test_attribute_value_escape_chars() {
-        
+        // Test handling of escaped characters in attribute values
-        // Test with closing quote
+        let values = [
-        let next_tokens = token.append('"');
+            "<delete id=\"escaped&quot;quote\"",
-        assert!(!next_tokens.is_empty(), "Should accept closing quote");
+            "<delete id=\"escaped&amp;amp\"",
-        
+        ];
-        let token = next_tokens[0].clone();
+        
-        match token {
+        for value in values {
-            Token::AttributeValue(value) => {
+            let mut validator = crate::tests::example_validator();
-                assert!(value.closed, "Should be closed");
+            validator.append(value).expect(value);
-            },
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-            _ => panic!("Expected AttributeValue, got {:?}", token),
+        }
-        }
+    }
-    }
-
-    #[test]
-    fn test_after_attribute_value_closed() {
-        let element = create_element_with_attributes();
-        let mut token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
-        token.in_quotes = true;
-        token.closed = true;
-        
-        // Test with space after closed attribute value
-        let next_tokens = token.append(' ');
-        assert!(!next_tokens.is_empty(), "Should accept space after closed attribute");
-        
-        // Test with > after closed attribute value
-        let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
-        let mut token = token;
-        token.in_quotes = true;
-        token.closed = true;
-        
-        let next_tokens = token.append('>');
-        assert!(!next_tokens.is_empty(), "Should accept '>' after closed attribute");
-        
-        match &next_tokens[0] {
-            Token::ElementOpenEnd(_) => (),
-            _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
-        }
-        
-        // Test with / after closed attribute value (for self-closing)
-        let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
-        let mut token = token;
-        token.in_quotes = true;
-        token.closed = true;
-        
-        let next_tokens = token.append('/');
-        assert!(!next_tokens.is_empty(), "Should accept '/' after closed attribute");
-        
-        match &next_tokens[0] {
-            Token::ElementSelfClose(_) => (),
-            _ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]),
-        }
-    }
-    
-    #[test]
-    fn test_attribute_tracking() {
-        let element = create_element_with_attributes();
-        let mut token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
-        token.in_quotes = true;
-        token.closed = true;
-        
-        // After attribute is closed, it should be tracked
-        let next_tokens = token.append('>');
-        assert!(!next_tokens.is_empty(), "Should accept '>' after attribute");
-        
-        match &next_tokens[0] {
-            Token::ElementOpenEnd(element_open) => {
-                // Verify that the id attribute is tracked
-                assert!(element_open.attributes_set.contains(&"id".to_string()), 
-                        "Attribute 'id' should be tracked");
-            },
-            _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
-        }
-    }
 }

lib/xml_schema_validator/src/token/element_close_name.rs

@@ -1,137 +1,149 @@
-use std::sync::Arc;
+use crate::*;
-use crate::*;
+use std::sync::Arc;
-
+
-/// Represents a closing element tag that is in the process of being parsed
+/// Represents a closing element tag that is in the process of being parsed
-/// after the '</' and now parsing the element name
+/// after the '</' and now parsing the element name
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug)]
-pub struct ElementCloseName {
+pub struct ElementCloseName {
-    /// The element that was previously opened and is now being closed
+    /// The element that was previously opened and is now being closed
-    element: Arc<schema::XsElement>,
+    element: Arc<schema::XsElement>,
-    /// Current buffer of characters processed for the element name
+    /// Current buffer of characters processed for the element name
-    buffer: String,
+    buffer: String,
-}
+}
-
+
-impl ElementCloseName {
+impl ElementCloseName {
-    /// Create a new instance with the given buffer
+    /// Create a new instance with the given buffer
-    pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Self {
+    pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Self {
-        Self {
+        Self { element, buffer }
-            element,
+    }
-            buffer,
+
-        }
+    pub fn append(self, c: char) -> Vec<Token> {
-    }
+        if self.element.name == self.buffer && c.is_whitespace() {
-
+            return vec![Token::ElementCloseName(self)];
-    pub fn append(self, c: char) -> Vec<Token> {
+        }
-        if self.element.name == self.buffer && c.is_whitespace() {
+        if self.element.name == self.buffer && c == '>' {
-            return vec![Token::ElementCloseName(self)];
+            return vec![Token::EndOfFile(EndOfFile::new())];
-        }
+        }
-        if self.element.name == self.buffer && c == '>' {
+        let new_buffer = self.buffer + &c.to_string();
-            return vec![Token::EndOfFile(EndOfFile::new())];
+        if self.element.name.starts_with(&new_buffer) {
-        }
+            return vec![Token::ElementCloseName(ElementCloseName::new(
-        let new_buffer = self.buffer + &c.to_string();
+                Arc::clone(&self.element),
-        if self.element.name.starts_with(&new_buffer) {
+                new_buffer,
-            return vec![Token::ElementCloseName(ElementCloseName::new(
+            ))];
-                Arc::clone(&self.element),
+        } else {
-                new_buffer,
+            vec![]
-            ))];
+        }
-        } else {
+    }
-            vec![]
+}
-        }
+
-    }
+#[cfg(test)]
-}
+mod tests {
-
+    use super::*;
-#[cfg(test)]
+    use crate::schema;
-mod element_close_name_tests {
+    use std::sync::Arc;
-    use super::*;
+
-    use std::sync::Arc;
+    fn create_test_element() -> Arc<schema::XsElement> {
-    use crate::schema;
+        let element = schema::XsElement {
-
+            name: "reasoning".to_string(),
-    fn create_test_element() -> Arc<schema::XsElement> {
+            text: None,
-        let element = schema::XsElement {
+            xs_complex_type: schema::XsComplexType {
-            name: "reasoning".to_string(),
+                mixed: Some("true".to_string()),
-            text: None,
+                text: None,
-            xs_complex_type: schema::XsComplexType {
+                xs_attribute: None,
-                mixed: Some("true".to_string()),
+                xs_sequence: Some(schema::XsSequence {
-                text: None,
+                    text: None,
-                xs_attribute: None,
+                    xs_any: schema::XsAny {
-                xs_sequence: Some(schema::XsSequence {
+                        min_occurs: "0".to_string(),
-                    text: None,
+                        max_occurs: "unbounded".to_string(),
-                    xs_any: schema::XsAny {
+                        process_contents: "skip".to_string(),
-                        min_occurs: "0".to_string(),
+                    },
-                        max_occurs: "unbounded".to_string(),
+                }),
-                        process_contents: "skip".to_string(),
+            },
-                    },
+        };
-                }),
+        Arc::new(element)
-            },
+    }
-        };
+
-        Arc::new(element)
+    #[test]
-    }
+    fn test_close_name_partial() {
-
+        let element = create_test_element();
-    #[test]
+        let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
-    fn test_close_name_partial() {
+
-        let element = create_test_element();
+        // Test with next character in name
-        let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
+        let next_tokens = token.append('e');
-        
+        assert!(
-        // Test with next character in name
+            !next_tokens.is_empty(),
-        let next_tokens = token.append('e');
+            "Should accept next character in name"
-        assert!(!next_tokens.is_empty(), "Should accept next character in name");
+        );
-        
+
-        match &next_tokens[0] {
+        match &next_tokens[0] {
-            Token::ElementCloseName(name) => {
+            Token::ElementCloseName(name) => {
-                assert_eq!(name.buffer, "re", "Buffer should contain 're'");
+                assert_eq!(name.buffer, "re", "Buffer should contain 're'");
-            },
+            }
-            _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
+            _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
-        }
+        }
-    }
+    }
-
+
-    #[test]
+    #[test]
-    fn test_close_name_complete() {
+    fn test_close_name_complete() {
-        let element = create_test_element();
+        let element = create_test_element();
-        let token = ElementCloseName::new(Arc::clone(&element), "reasonin".to_string());
+        let token = ElementCloseName::new(Arc::clone(&element), "reasonin".to_string());
-        
+
-        // Test with last character of name
+        // Test with last character of name
-        let next_tokens = token.append('g');
+        let next_tokens = token.append('g');
-        assert!(!next_tokens.is_empty(), "Should accept last character of name");
+        assert!(
-        
+            !next_tokens.is_empty(),
-        match &next_tokens[0] {
+            "Should accept last character of name"
-            Token::ElementCloseName(name) => {
+        );
-                assert_eq!(name.buffer, "reasoning", "Buffer should contain 'reasoning'");
+
-            },
+        match &next_tokens[0] {
-            _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
+            Token::ElementCloseName(name) => {
-        }
+                assert_eq!(
-
+                    name.buffer, "reasoning",
-        // Test with > after complete name
+                    "Buffer should contain 'reasoning'"
-        let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
+                );
-        let next_tokens = token.append('>');
+            }
-        assert!(!next_tokens.is_empty(), "Should accept '>' after complete name");
+            _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
-        
+        }
-        match &next_tokens[0] {
+
-            Token::EndOfFile(_) => (),
+        // Test with > after complete name
-            _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
+        let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
-        }
+        let next_tokens = token.append('>');
-    }
+        assert!(
-
+            !next_tokens.is_empty(),
-    #[test]
+            "Should accept '>' after complete name"
-    fn test_close_name_whitespace() {
+        );
-        let element = create_test_element();
+
-        let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
+        match &next_tokens[0] {
-        
+            Token::EndOfFile(_) => (),
-        // Test with whitespace after complete name
+            _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
-        let next_tokens = token.append(' ');
+        }
-        assert!(!next_tokens.is_empty(), "Should accept whitespace after complete name");
+    }
-        
+
-        match &next_tokens[0] {
+    #[test]
-            Token::ElementCloseName(_) => (),
+    fn test_close_name_whitespace() {
-            _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
+        let element = create_test_element();
-        }
+        let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
-    }
+
-
+        // Test with whitespace after complete name
-    #[test]
+        let next_tokens = token.append(' ');
-    fn test_close_name_invalid() {
+        assert!(
-        let element = create_test_element();
+            !next_tokens.is_empty(),
-        let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
+            "Should accept whitespace after complete name"
-        
+        );
-        // Test with invalid next character
+
-        let next_tokens = token.append('x');
+        match &next_tokens[0] {
-        assert!(next_tokens.is_empty(), "Should reject invalid character");
+            Token::ElementCloseName(_) => (),
-    }
+            _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
-}
+        }
+    }
+
+    #[test]
+    fn test_close_name_invalid() {
+        let element = create_test_element();
+        let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
+
+        // Test with invalid next character
+        let next_tokens = token.append('x');
+        assert!(next_tokens.is_empty(), "Should reject invalid character");
+    }
+}

lib/xml_schema_validator/src/token/element_close_start.rs

@@ -1,107 +1,110 @@
-use std::sync::Arc;
+use crate::*;
-use crate::*;
+use std::sync::Arc;
-
+
-/// Represents the start of an XML element closing tag (the '</' sequence)
+/// Represents the start of an XML element closing tag (the '</' sequence)
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug)]
-pub struct ElementCloseStart {
+pub struct ElementCloseStart {
-    /// The element that was previously opened and is now being closed
+    /// The element that was previously opened and is now being closed
-    element: Arc<schema::XsElement>,
+    element: Arc<schema::XsElement>,
-    has_slash: bool,
+    has_slash: bool,
-}
+}
-
+
-impl ElementCloseStart {
+impl ElementCloseStart {
-    pub fn new(element: Arc<schema::XsElement>) -> Self {
+    pub fn new(element: Arc<schema::XsElement>) -> Self {
-        Self {
+        Self {
-            element,
+            element,
-            has_slash: false,
+            has_slash: false,
-        }
+        }
-    }
+    }
-
+
-    pub fn append(self, c: char) -> Vec<Token> {
+    pub fn append(self, c: char) -> Vec<Token> {
-        if !self.has_slash && '/' == c {
+        if !self.has_slash && '/' == c {
-            vec![Token::ElementCloseStart(Self {
+            vec![Token::ElementCloseStart(Self {
-                element: Arc::clone(&self.element),
+                element: Arc::clone(&self.element),
-                has_slash: true,
+                has_slash: true,
-            })]
+            })]
-        } else if self.has_slash && self.element.name.starts_with(c) {
+        } else if self.has_slash && self.element.name.starts_with(c) {
-            vec![Token::ElementCloseName(ElementCloseName::new(
+            vec![Token::ElementCloseName(ElementCloseName::new(
-                Arc::clone(&self.element),
+                Arc::clone(&self.element),
-                c.to_string(),
+                c.to_string(),
-            ))]
+            ))]
-        } else {
+        } else {
-            vec![]
+            vec![]
-        }
+        }
-    }
+    }
-}
+}
-
+
-#[cfg(test)]
+#[cfg(test)]
-mod element_close_start_tests {
+mod tests {
-    use super::*;
+    use super::*;
-    use std::sync::Arc;
+    use crate::schema;
-    use crate::schema;
+    use std::sync::Arc;
-
+
-    fn create_test_element() -> Arc<schema::XsElement> {
+    fn create_test_element() -> Arc<schema::XsElement> {
-        let element = schema::XsElement {
+        let element = schema::XsElement {
-            name: "reasoning".to_string(),
+            name: "reasoning".to_string(),
-            text: None,
+            text: None,
-            xs_complex_type: schema::XsComplexType {
+            xs_complex_type: schema::XsComplexType {
-                mixed: Some("true".to_string()),
+                mixed: Some("true".to_string()),
-                text: None,
+                text: None,
-                xs_attribute: None,
+                xs_attribute: None,
-                xs_sequence: Some(schema::XsSequence {
+                xs_sequence: Some(schema::XsSequence {
-                    text: None,
+                    text: None,
-                    xs_any: schema::XsAny {
+                    xs_any: schema::XsAny {
-                        min_occurs: "0".to_string(),
+                        min_occurs: "0".to_string(),
-                        max_occurs: "unbounded".to_string(),
+                        max_occurs: "unbounded".to_string(),
-                        process_contents: "skip".to_string(),
+                        process_contents: "skip".to_string(),
-                    },
+                    },
-                }),
+                }),
-            },
+            },
-        };
+        };
-        Arc::new(element)
+        Arc::new(element)
-    }
+    }
-
+
-    #[test]
+    #[test]
-    fn test_close_start_slash() {
+    fn test_close_start_slash() {
-        let element = create_test_element();
+        let element = create_test_element();
-        let token = ElementCloseStart::new(Arc::clone(&element));
+        let token = ElementCloseStart::new(Arc::clone(&element));
-        
+
-        // Test with slash
+        // Test with slash
-        let next_tokens = token.append('/');
+        let next_tokens = token.append('/');
-        assert!(!next_tokens.is_empty(), "Should accept '/'");
+        assert!(!next_tokens.is_empty(), "Should accept '/'");
-        
+
-        match &next_tokens[0] {
+        match &next_tokens[0] {
-            Token::ElementCloseStart(start) => {
+            Token::ElementCloseStart(start) => {
-                assert!(start.has_slash, "Should mark as having slash");
+                assert!(start.has_slash, "Should mark as having slash");
-            },
+            }
-            _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
+            _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
-        }
+        }
-    }
+    }
-
+
-    #[test]
+    #[test]
-    fn test_close_start_element_name() {
+    fn test_close_start_element_name() {
-        let element = create_test_element();
+        let element = create_test_element();
-        let mut token = ElementCloseStart::new(Arc::clone(&element));
+        let mut token = ElementCloseStart::new(Arc::clone(&element));
-        token.has_slash = true;
+        token.has_slash = true;
-        
+
-        // Test with first character of element name
+        // Test with first character of element name
-        let next_tokens = token.append('r');
+        let next_tokens = token.append('r');
-        assert!(!next_tokens.is_empty(), "Should accept first letter of element name");
+        assert!(
-        
+            !next_tokens.is_empty(),
-        match &next_tokens[0] {
+            "Should accept first letter of element name"
-            Token::ElementCloseName(_) => (),
+        );
-            _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
+
-        }
+        match &next_tokens[0] {
-    }
+            Token::ElementCloseName(_) => (),
-
+            _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
-    #[test]
+        }
-    fn test_close_start_invalid_name() {
+    }
-        let element = create_test_element();
+
-        let mut token = ElementCloseStart::new(Arc::clone(&element));
+    #[test]
-        token.has_slash = true;
+    fn test_close_start_invalid_name() {
-        
+        let element = create_test_element();
-        // Test with invalid first character of element name
+        let mut token = ElementCloseStart::new(Arc::clone(&element));
-        let next_tokens = token.append('x');
+        token.has_slash = true;
-        assert!(next_tokens.is_empty(), "Should reject invalid first letter");
+
-    }
+        // Test with invalid first character of element name
-}
+        let next_tokens = token.append('x');
+        assert!(next_tokens.is_empty(), "Should reject invalid first letter");
+    }
+}

lib/xml_schema_validator/src/token/element_open_end.rs

@@ -1,220 +1,163 @@
-use std::sync::Arc;
+use crate::*;
-use crate::*;
+use std::sync::Arc;
-
+
-/// Represents a fully opened XML element, containing the schema element reference
+/// Represents a fully opened XML element, containing the schema element reference
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug)]
-pub struct ElementOpenEnd {
+pub struct ElementOpenEnd {
-    element: Arc<schema::XsElement>,
+    element: Arc<schema::XsElement>,
-    // Track which attributes have been set for this element
+}
-    pub attributes_set: Vec<String>,
+
-}
+impl ElementOpenEnd {
-
+    pub fn new(element: Arc<schema::XsElement>) -> Self {
-impl ElementOpenEnd {
+        Self {
-    pub fn new(element: Arc<schema::XsElement>) -> Self {
+            element,
-        Self {
+        }
-            element,
+    }
-            attributes_set: Vec::new(),
+
-        }
+    pub fn append(self, c: char) -> Vec<Token> {
-    }
+        if '<' == c {
-
+            vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
-    pub fn with_attributes(element: Arc<schema::XsElement>, attributes_set: Vec<String>) -> Self {
+        } else {
-        Self {
+            // Check if this element allows mixed content
-            element,
+            let is_mixed = self.element.xs_complex_type.mixed
-            attributes_set,
+                .as_ref()
-        }
+                .map(|val| val == "true")
-    }
+                .unwrap_or(false);
-
+                
-    pub fn append(self, c: char) -> Vec<Token> {
+            if is_mixed || c.is_whitespace() {
-        // Check if all required attributes are present before proceeding
+                // Allow text content for mixed elements, or whitespace for any element
-        if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
+                vec![Token::TextContent(TextContent::new(self.element))]
-            for attr in attributes {
+            } else {
-                if attr.xs_attribute_use == "required" && !self.attributes_set.contains(&attr.name) {
+                // Reject text content for non-mixed elements
-                    // Missing a required attribute, this is an error
+                vec![]
-                    return vec![];
+            }
-                }
+        }
-            }
+    }
-        }
+}
-
+
-        // Check if the element is a mixed content element that allows text
+#[cfg(test)]
-        let allows_text = self.element.xs_complex_type.mixed
+mod tests {
-            .as_ref()
+    use super::*;
-            .map(|mixed| mixed == "true")
+    use crate::schema;
-            .unwrap_or(false);
+    use std::sync::Arc;
-
+
-        if allows_text {
+    fn create_test_element() -> Arc<schema::XsElement> {
-            if c.is_whitespace() {
+        let element = schema::XsElement {
-                // Handle whitespace in mixed content
+            name: "reasoning".to_string(),
-                vec![
+            text: None,
-                    Token::TextContent(TextContent::new(Arc::clone(&self.element))),
+            xs_complex_type: schema::XsComplexType {
-                    Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))
+                mixed: Some("true".to_string()),
-                ]
+                text: None,
-            } else if c == '<' {
+                xs_attribute: None,
-                // Start of a closing tag or nested element
+                xs_sequence: Some(schema::XsSequence {
-                vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
+                    text: None,
-            } else {
+                    xs_any: schema::XsAny {
-                // Text content
+                        min_occurs: "0".to_string(),
-                vec![Token::TextContent(TextContent::new(Arc::clone(&self.element)))]
+                        max_occurs: "unbounded".to_string(),
-            }
+                        process_contents: "skip".to_string(),
-        } else {
+                    },
-            // Element doesn't allow text content, only expect closing tag
+                }),
-            if c == '<' {
+            },
-                vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
+        };
-            } else if c.is_whitespace() {
+        Arc::new(element)
-                vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
+    }
-            } else {
+
-                // Unexpected content in non-mixed element
+    fn create_element_non_mixed() -> Arc<schema::XsElement> {
-                vec![]
+        let element = schema::XsElement {
-            }
+            name: "element".to_string(),
-        }
+            text: None,
-    }
+            xs_complex_type: schema::XsComplexType {
-}
+                mixed: None, // Not mixed content
-
+                text: None,
-#[cfg(test)]
+                xs_attribute: None,
-mod element_open_end_tests {
+                xs_sequence: Some(schema::XsSequence {
-    use super::*;
+                    text: None,
-    use std::sync::Arc;
+                    xs_any: schema::XsAny {
-    use crate::schema;
+                        min_occurs: "0".to_string(),
-
+                        max_occurs: "unbounded".to_string(),
-    fn create_test_element() -> Arc<schema::XsElement> {
+                        process_contents: "skip".to_string(),
-        let element = schema::XsElement {
+                    },
-            name: "reasoning".to_string(),
+                }),
-            text: None,
+            },
-            xs_complex_type: schema::XsComplexType {
+        };
-                mixed: Some("true".to_string()),
+        Arc::new(element)
-                text: None,
+    }
-                xs_attribute: None,
+
-                xs_sequence: Some(schema::XsSequence {
+    #[test]
-                    text: None,
+    fn test_append_text_in_mixed_content() {
-                    xs_any: schema::XsAny {
+        let element = create_test_element();
-                        min_occurs: "0".to_string(),
+        let token = ElementOpenEnd::new(Arc::clone(&element));
-                        max_occurs: "unbounded".to_string(),
+
-                        process_contents: "skip".to_string(),
+        // Test appending text in mixed content element
-                    },
+        let next_tokens = token.append('a');
-                }),
+        assert!(
-            },
+            !next_tokens.is_empty(),
-        };
+            "Should allow text content in mixed element"
-        Arc::new(element)
+        );
-    }
+
-
+        match &next_tokens[0] {
-    fn create_element_non_mixed() -> Arc<schema::XsElement> {
+            Token::TextContent(_) => (),
-        let element = schema::XsElement {
+            _ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
-            name: "element".to_string(),
+        }
-            text: None,
+    }
-            xs_complex_type: schema::XsComplexType {
+
-                mixed: None, // Not mixed content
+    #[test]
-                text: None,
+    fn test_append_closing_tag_start() {
-                xs_attribute: None,
+        let element = create_test_element();
-                xs_sequence: Some(schema::XsSequence {
+        let token = ElementOpenEnd::new(Arc::clone(&element));
-                    text: None,
+
-                    xs_any: schema::XsAny {
+        // Test starting a closing tag
-                        min_occurs: "0".to_string(),
+        let next_tokens = token.append('<');
-                        max_occurs: "unbounded".to_string(),
+        assert!(!next_tokens.is_empty(), "Should allow closing tag start");
-                        process_contents: "skip".to_string(),
+
-                    },
+        match &next_tokens[0] {
-                }),
+            Token::ElementCloseStart(_) => (),
-            },
+            _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
-        };
+        }
-        Arc::new(element)
+    }
-    }
+
-
+    #[test]
-    fn create_element_with_required_attributes() -> Arc<schema::XsElement> {
+    fn test_append_whitespace_in_mixed() {
-        let element = schema::XsElement {
+        let element = create_test_element();
-            name: "delete".to_string(),
+        let token = ElementOpenEnd::new(Arc::clone(&element));
-            text: None,
+
-            xs_complex_type: schema::XsComplexType {
+        // Test whitespace in mixed content
-                mixed: None,
+        let next_tokens = token.append(' ');
-                text: None,
+        assert!(
-                xs_attribute: Some(vec![
+            !next_tokens.is_empty(),
-                    schema::XsAttribute {
+            "Should allow whitespace in mixed content"
-                        name: "id".to_string(),
+        );
-                        xs_attribute_type: "xs:string".to_string(),
+
-                        xs_attribute_use: "required".to_string(),
+        match &next_tokens[0] {
-                    }
+            Token::TextContent(_) => (),
-                ]),
+            _ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
-                xs_sequence: None,
+        }
-            },
+    }
-        };
+
-        Arc::new(element)
+    #[test]
-    }
+    fn test_append_in_non_mixed() {
-
+        let element = create_element_non_mixed();
-    #[test]
+        let token = ElementOpenEnd::new(Arc::clone(&element));
-    fn test_append_text_in_mixed_content() {
+
-        let element = create_test_element();
+        // Test appending text in non-mixed content (should reject)
-        let token = ElementOpenEnd::new(Arc::clone(&element));
+        let next_tokens = token.clone().append('a');
-        
+        assert!(
-        // Test appending text in mixed content element
+            next_tokens.is_empty(),
-        let next_tokens = token.append('a');
+            "Should reject text in non-mixed element"
-        assert!(!next_tokens.is_empty(), "Should allow text content in mixed element");
+        );
-        
+
-        match &next_tokens[0] {
+        // Test whitespace (should accept leading to closing tag)
-            Token::TextContent(_) => (),
+        let next_tokens = token.clone().append(' ');
-            _ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
+        assert!(
-        }
+            !next_tokens.is_empty(),
-    }
+            "Should allow whitespace in non-mixed"
-
+        );
-    #[test]
+
-    fn test_append_closing_tag_start() {
+        // Test closing tag start
-        let element = create_test_element();
+        let next_tokens = token.append('<');
-        let token = ElementOpenEnd::new(Arc::clone(&element));
+        assert!(
-        
+            !next_tokens.is_empty(),
-        // Test starting a closing tag
+            "Should allow closing tag start in non-mixed"
-        let next_tokens = token.append('<');
+        );
-        assert!(!next_tokens.is_empty(), "Should allow closing tag start");
+    }
-        
+}
-        match &next_tokens[0] {
-            Token::ElementCloseStart(_) => (),
-            _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
-        }
-    }
-
-    #[test]
-    fn test_append_whitespace_in_mixed() {
-        let element = create_test_element();
-        let token = ElementOpenEnd::new(Arc::clone(&element));
-        
-        // Test whitespace in mixed content
-        let next_tokens = token.append(' ');
-        assert!(!next_tokens.is_empty(), "Should allow whitespace in mixed content");
-        
-        match &next_tokens[0] {
-            Token::TextContent(_) => (),
-            _ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
-        }
-    }
-
-    #[test]
-    fn test_append_in_non_mixed() {
-        let element = create_element_non_mixed();
-        let token = ElementOpenEnd::new(Arc::clone(&element));
-        
-        // Test appending text in non-mixed content (should reject)
-        let next_tokens = token.clone().append('a');
-        assert!(next_tokens.is_empty(), "Should reject text in non-mixed element");
-        
-        // Test whitespace (should accept leading to closing tag)
-        let next_tokens = token.clone().append(' ');
-        assert!(!next_tokens.is_empty(), "Should allow whitespace in non-mixed");
-        
-        // Test closing tag start
-        let next_tokens = token.append('<');
-        assert!(!next_tokens.is_empty(), "Should allow closing tag start in non-mixed");
-    }
-    
-    #[test]
-    fn test_required_attributes() {
-        let element = create_element_with_required_attributes();
-        
-        // No attributes set yet
-        let token = ElementOpenEnd::new(Arc::clone(&element));
-        let next_tokens = token.append('<');
-        assert!(next_tokens.is_empty(), "Should reject closing tag without required attributes");
-        
-        // With required attribute set
-        let token = ElementOpenEnd::with_attributes(
-            Arc::clone(&element), 
-            vec!["id".to_string()]
-        );
-        let next_tokens = token.append('<');
-        assert!(!next_tokens.is_empty(), "Should allow closing tag with required attributes set");
-    }
-}

lib/xml_schema_validator/src/token/element_open_name.rs

@@ -1,272 +1,186 @@
-use std::sync::Arc;
+use crate::*;
-use crate::*;
+use std::sync::Arc;
-
+
-/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
+/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug)]
-pub struct ElementOpenName {
+pub struct ElementOpenName {
-    buffer: String,
+    buffer: String,
-    element: Arc<schema::XsElement>,
+    element: Arc<schema::XsElement>,
-}
+}
-
+
-impl ElementOpenName {
+impl ElementOpenName {
-    pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Result<Self> {
+    pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Self {
-        if element.name.starts_with(buffer.as_str()) {
+        Self { buffer, element }
-            Ok(Self {
+    }
-                buffer,
+
-                element,
+    pub fn append(mut self, c: char) -> Vec<Token> {
-            })
+        if c == '>' {
-        } else {
+            // Check if we've matched the full element name
-            Err(Error::InvalidXml(format!("Element name '{}' doesn't match '{}'", buffer, element.name)))
+            if !self.buffer.contains(&self.element.name) {
-        }
+                return vec![];
-    }
+            }
-
+            // Check if the element has attributes
-    pub fn append(self, c: char) -> Vec<Token> {
+            if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
-        if c.is_whitespace() {
+                // Check if they are required
-            // Check if we've matched the full element name
+                if attributes
-            if self.element.name == self.buffer {
+                    .iter()
-                // Element name is complete, handle whitespace after name
+                    .filter(|attribute| attribute.xs_attribute_use == "required")
-                
+                    .next()
-                // Check if the element has attributes
+                    .is_some()
-                if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
+                {
-                    if !attributes.is_empty() {
+                    return vec![];
-                        // Transition to attribute name parsing
+                } else {
-                        return vec![Token::AttributeName(AttributeName::new(Arc::clone(&self.element)))];
+                    return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(
-                    }
+                        &self.element,
-                }
+                    )))];
-                
+                }
-                // No attributes
+            } else {
-                return vec![
+                return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(
-                    Token::AttributeName(AttributeName::new(Arc::clone(&self.element))),
+                    &self.element,
-                    Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element))),
+                )))];
-                    Token::ElementSelfClose(ElementSelfClose::new()),
+            }
-                    Token::ElementOpenName(self),
+        } else if c == '/' {
-                ];
+            // Check if we've matched the full element name
-            } else {
+            if !self.buffer.contains(&self.element.name) {
-                // Whitespace in the middle of an element name is invalid
+                return vec![];
-                return vec![];
+            }
-            }
+            // Check if the element has attributes
-        } else if c == '>' {
+            if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
-            // Check if we've matched the full element name
+                // Check if they are required
-            if self.element.name == self.buffer {
+                if attributes
-                // Element name is complete, end of opening tag
+                    .iter()
-                
+                    .filter(|attribute| attribute.xs_attribute_use == "required")
-                // Check if this element has required attributes
+                    .next()
-                if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
+                    .is_some()
-                    for attr in attributes {
+                {
-                        if attr.xs_attribute_use == "required" {
+                    return vec![];
-                            // Found a required attribute but no attributes are set
+                } else {
-                            // Return an empty vector to indicate error
+                    return vec![Token::ElementSelfClose(ElementSelfClose::new())];
-                            return vec![];
+                }
-                        }
+            } else {
-                    }
+                return vec![Token::ElementSelfClose(ElementSelfClose::new())];
-                }
+            }
-                
+        } else if c.is_whitespace() {
-                return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element)))];
+            // Check if we've matched the full element name
-            } else {
+            if self.buffer.contains(&self.element.name) {
-                // '>' before full element name is invalid
+                if let Some(last) = self.buffer.chars().last() {
-                return vec![];
+                    if last.is_whitespace() {
-            }
+                        return vec![Token::ElementOpenName(self)];
-        } else if c == '/' {
+                    }
-            // Check if we've matched the full element name
+                }
-            if self.element.name == self.buffer {
+                self.buffer.push(c);
-                // Element name is complete, self-closing tag
+                return vec![Token::ElementOpenName(self)];
-                return vec![Token::ElementSelfClose(ElementSelfClose::new())];
+            } else {
-            } else {
+                return vec![];
-                // '/' before full element name is invalid
+            }
-                return vec![];
+        } else if !self.buffer.is_empty() && self.buffer.chars().last().unwrap().is_whitespace() {
-            }
+            // Check for start of attribute name
-        } else {
+            if self.buffer.contains(&self.element.name) {
-            // Add character to buffer and check if we're still building a valid element name
+                if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
-            let new_buffer = self.buffer.clone() + &c.to_string();
+                    if attributes
-            
+                        .iter()
-            if self.element.name.starts_with(&new_buffer) {
+                        .filter(|attribute| attribute.name.starts_with(c))
-                // Still building the element name
+                        .next()
-                return vec![Token::ElementOpenName(ElementOpenName {
+                        .is_some()
-                    buffer: new_buffer,
+                    {
-                    element: Arc::clone(&self.element),
+                        return vec![Token::AttributeName(AttributeName::new(
-                })];
+                            c,
-            } else {
+                            self.element.clone(),
-                // Character doesn't match what we expect for this element name
+                        ))];
-                return vec![];
+                    }
-            }
+                }
-        }
+            }
-    }
+            return vec![];
-}
+        } else {
-
+            // Add character to buffer and check if we're still building a valid element name
-#[cfg(test)]
+            let new_buffer = self.buffer.clone() + &c.to_string();
-mod element_open_name_tests {
+
-    use super::*;
+            if self.element.name.starts_with(&new_buffer) {
-    use std::sync::Arc;
+                // Still building the element name
-    use crate::schema;
+                return vec![Token::ElementOpenName(ElementOpenName {
-
+                    buffer: new_buffer,
-    fn create_test_element() -> Arc<schema::XsElement> {
+                    element: Arc::clone(&self.element),
-        let element = schema::XsElement {
+                })];
-            name: "reasoning".to_string(),
+            } else {
-            text: None,
+                // Character doesn't match what we expect for this element name
-            xs_complex_type: schema::XsComplexType {
+                return vec![];
-                mixed: Some("true".to_string()),
+            }
-                text: None,
+        }
-                xs_attribute: None,
+    }
-                xs_sequence: Some(schema::XsSequence {
+}
-                    text: None,
+
-                    xs_any: schema::XsAny {
+#[cfg(test)]
-                        min_occurs: "0".to_string(),
+mod tests {
-                        max_occurs: "unbounded".to_string(),
+    use super::*;
-                        process_contents: "skip".to_string(),
+
-                    },
+    #[test]
-                }),
+    fn test_attribute_name() {
-            },
+        let values = [
-        };
+            "<delete i", // id
-        Arc::new(element)
+            "<single t", // timeout
-    }
+            "<single l", // limit
-
+        ];
-    fn create_element_with_attributes() -> Arc<schema::XsElement> {
+        for value in values {
-        let element = schema::XsElement {
+            let mut validator = crate::tests::example_validator();
-            name: "delete".to_string(),
+            validator.append(value).expect(value);
-            text: None,
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-            xs_complex_type: schema::XsComplexType {
+            assert!(
-                mixed: None,
+                validator
-                text: None,
+                    .current_tokens
-                xs_attribute: Some(vec![
+                    .iter()
-                    schema::XsAttribute {
+                    .filter(|token| matches!(token, Token::AttributeName(_)))
-                        name: "id".to_string(),
+                    .next()
-                        xs_attribute_type: "xs:string".to_string(),
+                    .is_some(),
-                        xs_attribute_use: "required".to_string(),
+                "{value}"
-                    }
+            );
-                ]),
+        }
-                xs_sequence: None,
+    }
-            },
+
-        };
+    #[test]
-        Arc::new(element)
+    fn test_valid_self_closing() {
-    }
+        let values = ["<stop/", "<stop /"];
-
+        for value in values {
-    #[test]
+            let mut validator = crate::tests::example_validator();
-    fn test_append_character_by_character() {
+            validator.append(value).expect(value);
-        let element = create_test_element();
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-        
+            assert!(
-        // Test each character in "reasoning"
+                validator
-        let mut buffer = String::new();
+                    .current_tokens
-        for c in "reasoning".chars() {
+                    .iter()
-            buffer.push(c);
+                    .filter(|token| matches!(token, Token::ElementSelfClose(_)))
-            let result = ElementOpenName::new(Arc::clone(&element), buffer.clone());
+                    .next()
-            assert!(result.is_ok(), "Failed to create ElementOpenName with buffer '{}'", buffer);
+                    .is_some(),
-        }
+                "{value}"
-    }
+            );
-
+        }
-    #[test]
+    }
-    fn test_append_full_name() {
+
-        let element = create_test_element();
+    #[test]
-        
+    fn test_valid_closing() {
-        // Test with the full element name
+        let values = ["<stop>", "<stop >", "<reasoning>", "<reasoning >"];
-        let result = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string());
+        for value in values {
-        assert!(result.is_ok(), "Failed to create ElementOpenName with full element name");
+            let mut validator = crate::tests::example_validator();
-        
+            validator.append(value).expect(value);
-        let element_name = result.unwrap();
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-        
+            assert!(
-        // Test transition after full name with '>'
+                validator
-        let next_tokens = element_name.append('>');
+                    .current_tokens
-        assert!(!next_tokens.is_empty(), "Should accept '>' after full element name");
+                    .iter()
-        
+                    .filter(|token| matches!(token, Token::ElementOpenEnd(_)))
-        match &next_tokens[0] {
+                    .next()
-            Token::ElementOpenEnd(_) => (),
+                    .is_some(),
-            _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
+                "{value}"
-        }
+            );
-    }
+        }
-
+    }
-    #[test]
+
-    fn test_append_space_after_name() {
+    #[test]
-        let element = create_test_element();
+    fn test_invalid_closing() {
-        
+        let values = ["<delete/", "<delete /", "<delete>", "<delete >"];
-        // Test with the full element name
+        for value in values {
-        let element_name = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()).unwrap();
+            let mut validator = crate::tests::example_validator();
-        
+            validator.append(value).expect_err(value);
-        // Test transition after full name with space
+            assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
-        let next_tokens = element_name.append(' ');
+        }
-        assert!(!next_tokens.is_empty(), "Should accept space after full element name");
+    }
-        
+}
-        assert!(next_tokens.len() == 4);
-        assert!(next_tokens.iter()
-            .filter(|token| matches!(token, Token::ElementOpenName(_)))
-            .next()
-            .is_some());
-        assert!(next_tokens.iter()
-            .filter(|token| matches!(token, Token::AttributeName(_)))
-            .next()
-            .is_some());
-        assert!(next_tokens.iter()
-            .filter(|token| matches!(token, Token::ElementOpenEnd(_)))
-            .next()
-            .is_some());
-        assert!(next_tokens.iter()
-            .filter(|token| matches!(token, Token::ElementSelfClose(_)))
-            .next()
-            .is_some());
-    }
-
-    #[test]
-    fn test_append_with_attributes() {
-        let element = create_element_with_attributes();
-        
-        // Test with the full element name
-        let element_name = ElementOpenName::new(Arc::clone(&element), "delete".to_string()).unwrap();
-        
-        // Test transition after full name with space (expecting attribute name)
-        let next_tokens = element_name.append(' ');
-        assert!(!next_tokens.is_empty(), "Should accept space after element name with attributes");
-        
-        // The space should lead to a whitespace token, which should then lead to attribute name
-        match &next_tokens[0] {
-            Token::AttributeName(_) => (),
-            _ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
-        }
-    }
-    
-    #[test]
-    fn test_append_slash_after_name() {
-        let element = create_test_element();
-        
-        // Test with the full element name
-        let element_name = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()).unwrap();
-        
-        // Test transition after full name with /
-        let next_tokens = element_name.append('/');
-        assert!(!next_tokens.is_empty(), "Should accept '/' after full element name");
-        
-        match &next_tokens[0] {
-            Token::ElementSelfClose(_) => (),
-            _ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]),
-        }
-    }
-    
-    #[test]
-    fn test_invalid_continuation() {
-        let element = create_test_element();
-        
-        // Test with a partial element name and an invalid continuation character
-        let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap();
-        let next_tokens = element_name.append('x');
-        assert!(next_tokens.is_empty(), "Should reject invalid character in element name");
-        
-        // Test whitespace in middle of name - need to create a new instance since append consumes self
-        let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap();
-        let next_tokens = element_name.append(' ');
-        assert!(next_tokens.is_empty(), "Should reject whitespace in middle of element name");
-        
-        // Test > in middle of name - need to create a new instance since append consumes self
-        let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap();
-        let next_tokens = element_name.append('>');
-        assert!(next_tokens.is_empty(), "Should reject '>' in middle of element name");
-    }
-    
-    #[test]
-    fn test_required_attributes_validation() {
-        let element = create_element_with_attributes();
-        
-        // Test with the full element name
-        let element_name = ElementOpenName::new(Arc::clone(&element), "delete".to_string()).unwrap();
-        
-        // Try to close the tag with > directly, skipping the required attribute
-        let next_tokens = element_name.append('>');
-        assert!(next_tokens.is_empty(), "Should reject closing tag without required attributes");
-    }
-}

lib/xml_schema_validator/src/token/element_open_start.rs

@@ -1,25 +1,64 @@
-use std::sync::Arc;
+use crate::*;
-use crate::*;
+use std::sync::Arc;
-
+
-/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
+/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug)]
-pub struct ElementOpenStart {
+pub struct ElementOpenStart {
-    element: Arc<schema::XsElement>,
+    element: Arc<schema::XsElement>,
-}
+}
-
+
-impl ElementOpenStart {
+impl ElementOpenStart {
-    pub fn new(element: Arc<schema::XsElement>) -> Self {
+    pub fn new(element: Arc<schema::XsElement>) -> Self {
-        Self {
+        Self { element }
-            element,
+    }
-        }
+
-    }
+    pub fn append(self, c: char) -> Vec<Token> {
-
+        if self.element.name.starts_with(c) {
-    pub fn append(self, c: char) -> Vec<Token> {
+            vec![Token::ElementOpenName(ElementOpenName::new(
-        let element_name = ElementOpenName::new(Arc::clone(&self.element), c.to_string());
+                Arc::clone(&self.element),
-        if let Ok(element_name) = element_name {
+                c.to_string(),
-            vec![Token::ElementOpenName(element_name)]
+            ))]
-        } else {
+        } else {
-            vec![]
+            vec![]
-        }
+        }
-    }
+    }
-}
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_accept_valid_element_names() {
+        let values = [
+            "<d", // delete
+            "<s", // stop / single
+            "<r", // repeat / reasoning / read_stdin
+            "<w", // write_stdout
+        ];
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            assert!(
+                validator
+                    .current_tokens
+                    .iter()
+                    .filter(|token| matches!(token, Token::ElementOpenName(_)))
+                    .next()
+                    .is_some(),
+                "{value}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_not_accept_other() {
+        let values = ["< ", "<a", "</"];
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect_err(value);
+            assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
+        }
+    }
+}

lib/xml_schema_validator/src/token/element_self_close.rs

@@ -1,144 +1,54 @@
-use std::sync::Arc;
+use crate::*;
-use crate::*;
+
-
+/// Represents a self-closing XML element
-/// Represents a self-closing XML element
+#[derive(Clone, Debug)]
-#[derive(Clone, Debug)]
+pub struct ElementSelfClose {}
-pub struct ElementSelfClose {
+
-    /// Optional reference to the element schema for attribute validation
+impl ElementSelfClose {
-    element: Option<Arc<schema::XsElement>>,
+    /// Create a new self-closing element
-    /// Track which attributes have been set
+    pub fn new() -> Self {
-    attributes_set: Vec<String>,
+        Self {}
-}
+    }
-
+
-impl ElementSelfClose {
+    pub fn append(self, c: char) -> Vec<Token> {
-    /// Create a new self-closing element
+        if c == '>' {
-    pub fn new() -> Self {
+            return vec![Token::EndOfFile(EndOfFile::new())];
-        Self {
+        } else {
-            element: None,
+            vec![]
-            attributes_set: Vec::new(),
+        }
-        }
+    }
-    }
+}
-    
+
-    /// Create a new self-closing element with element reference and attribute tracking
+#[cfg(test)]
-    pub fn new_with_attributes(element: Arc<schema::XsElement>, attributes_set: Vec<String>) -> Self {
+mod tests {
-        Self {
+    use super::*;
-            element: Some(element),
+
-            attributes_set,
+    #[test]
-        }
+    fn test_valid() {
-    }
+        let values = ["<stop/>"];
-
+        for value in values {
-    /// Append a character to the self-closing tag and return possible continuations
+            let mut validator = crate::tests::example_validator();
-    pub fn append(self, c: char) -> Vec<Token> {
+            validator.append(value).expect(value);
-        // Before we close, verify that all required attributes are present
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-        if let Some(element) = &self.element {
+            assert!(
-            if c == '>' {
+                validator
-                if let Some(attributes) = &element.xs_complex_type.xs_attribute {
+                    .current_tokens
-                    for attr in attributes {
+                    .iter()
-                        if attr.xs_attribute_use == "required" && !self.attributes_set.contains(&attr.name) {
+                    .filter(|token| matches!(token, Token::EndOfFile(_)))
-                            // Missing required attribute, this is an error
+                    .next()
-                            return vec![];
+                    .is_some(),
-                        }
+                "{value}"
-                    }
+            );
-                }
+        }
-                return vec![Token::EndOfFile(EndOfFile::new())];
+    }
-            }
+
-        } else if c == '>' {
+    #[test]
-            // No element reference, can't validate attributes, just close
+    fn test_invalid() {
-            return vec![Token::EndOfFile(EndOfFile::new())];
+        let values = ["<stop/?", "<stop/a", "<stop/ "];
-        }
+        for value in values {
-        
+            let mut validator = crate::tests::example_validator();
-        if c.is_whitespace() {
+            validator.append(value).expect_err(value);
-            vec![Token::ElementSelfClose(self)]
+            assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
-        } else {
+        }
-            vec![]
+    }
-        }
+}
-    }
-}
-
-#[cfg(test)]
-mod element_self_close_tests {
-    use super::*;
-    use std::sync::Arc;
-    use crate::schema;
-
-    fn create_element_with_required_attributes() -> Arc<schema::XsElement> {
-        let element = schema::XsElement {
-            name: "delete".to_string(),
-            text: None,
-            xs_complex_type: schema::XsComplexType {
-                mixed: None,
-                text: None,
-                xs_attribute: Some(vec![
-                    schema::XsAttribute {
-                        name: "id".to_string(),
-                        xs_attribute_type: "xs:string".to_string(),
-                        xs_attribute_use: "required".to_string(),
-                    }
-                ]),
-                xs_sequence: None,
-            },
-        };
-        Arc::new(element)
-    }
-
-    #[test]
-    fn test_self_close_with_greater_than() {
-        let token = ElementSelfClose::new();
-        
-        // Test closing with >
-        let next_tokens = token.append('>');
-        assert!(!next_tokens.is_empty(), "Should accept '>' to close self-closing tag");
-        assert_eq!(next_tokens.len(), 1, "Should have exactly one token after '>'");
-        
-        match &next_tokens[0] {
-            Token::EndOfFile(_) => (),
-            _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
-        }
-    }
-
-    #[test]
-    fn test_self_close_with_whitespace() {
-        let token = ElementSelfClose::new();
-        
-        // Test with whitespace
-        let next_tokens = token.append(' ');
-        assert!(!next_tokens.is_empty(), "Should accept whitespace in self-closing tag");
-        
-        match &next_tokens[0] {
-            Token::ElementSelfClose(_) => (),
-            _ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]),
-        }
-    }
-
-    #[test]
-    fn test_self_close_with_invalid_char() {
-        let token = ElementSelfClose::new();
-        
-        // Test with invalid character
-        let next_tokens = token.append('a');
-        assert!(next_tokens.is_empty(), "Should reject invalid character in self-closing tag");
-    }
-    
-    #[test]
-    fn test_required_attributes_validation() {
-        let element = create_element_with_required_attributes();
-        
-        // Without required attributes
-        let token = ElementSelfClose::new_with_attributes(Arc::clone(&element), vec![]);
-        let next_tokens = token.append('>');
-        assert!(next_tokens.is_empty(), "Should reject closing without required attributes");
-        
-        // With required attributes
-        let token = ElementSelfClose::new_with_attributes(
-            Arc::clone(&element),
-            vec!["id".to_string()]
-        );
-        let next_tokens = token.append('>');
-        assert!(!next_tokens.is_empty(), "Should accept closing with required attributes");
-        
-        match &next_tokens[0] {
-            Token::EndOfFile(_) => (),
-            _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
-        }
-    }
-}

lib/xml_schema_validator/src/token/end_of_file.rs

@@ -1,30 +1,29 @@
-use crate::*;
+use crate::*;
-
+
-/// Represents the final state at the end of an XML file
+/// Represents the final state at the end of an XML file
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug)]
-pub struct EndOfFile {
+pub struct EndOfFile {}
-}
+
-
+impl EndOfFile {
-impl EndOfFile {
+    pub fn new() -> Self {
-    pub fn new() -> Self {
+        Self {}
-        Self {}
+    }
-    }
+
-
+    pub fn append(self, _c: char) -> Vec<Token> {
-    pub fn append(self, _c: char) -> Vec<Token> {
+        vec![]
-        vec![]
+    }
-    }
+}
-}
+
-
+#[cfg(test)]
-#[cfg(test)]
+mod tests {
-mod end_of_file_tests {
+    use super::*;
-    use super::*;
+
-
+    #[test]
-    #[test]
+    fn test_end_of_file_any_char() {
-    fn test_end_of_file_any_char() {
+        let token = EndOfFile::new();
-        let token = EndOfFile::new();
+
-        
+        // Test with any character (should reject)
-        // Test with any character (should reject)
+        let next_tokens = token.append('a');
-        let next_tokens = token.append('a');
+        assert!(next_tokens.is_empty(), "Should reject any character at EOF");
-        assert!(next_tokens.is_empty(), "Should reject any character at EOF");
+    }
-    }
+}
-}

lib/xml_schema_validator/src/token/mod.rs

@@ -1,292 +1,285 @@
-mod element_close_name;
+mod attribute_equals;
-mod element_close_start;
+mod attribute_name;
-mod element_open_name;
+mod attribute_value;
-mod element_open_end;
+mod element_close_name;
-mod element_open_start;
+mod element_close_start;
-mod end_of_file;
+mod element_open_end;
-mod start_of_file;
+mod element_open_name;
-mod text_content;
+mod element_open_start;
-mod attribute_name;
+mod element_self_close;
-mod attribute_value;
+mod end_of_file;
-mod element_self_close;
+mod start_of_file;
-
+mod text_content;
-pub use element_close_name::ElementCloseName;
+
-pub use element_close_start::ElementCloseStart;
+pub use attribute_equals::AttributeEquals;
-pub use element_open_name::ElementOpenName;
+pub use attribute_name::AttributeName;
-pub use element_open_end::ElementOpenEnd;
+pub use attribute_value::AttributeValue;
-pub use element_open_start::ElementOpenStart;
+pub use element_close_name::ElementCloseName;
-pub use end_of_file::EndOfFile;
+pub use element_close_start::ElementCloseStart;
-pub use start_of_file::StartOfFile;
+pub use element_open_end::ElementOpenEnd;
-pub use text_content::TextContent;
+pub use element_open_name::ElementOpenName;
-pub use attribute_name::AttributeName;
+pub use element_open_start::ElementOpenStart;
-pub use attribute_value::AttributeValue;
+pub use element_self_close::ElementSelfClose;
-pub use element_self_close::ElementSelfClose;
+pub use end_of_file::EndOfFile;
-
+pub use start_of_file::StartOfFile;
-/// Represents the different states of XML token parsing
+pub use text_content::TextContent;
-#[derive(Clone, Debug)]
+
-pub enum Token {
+/// Represents the different states of XML token parsing
-    ElementCloseName(ElementCloseName),
+#[derive(Clone, Debug)]
-    ElementCloseStart(ElementCloseStart),
+pub enum Token {
-    ElementOpenName(ElementOpenName),
+    AttributeEquals(AttributeEquals),
-    ElementOpenEnd(ElementOpenEnd),
+    AttributeName(AttributeName),
-    ElementOpenStart(ElementOpenStart),
+    AttributeValue(AttributeValue),
-    EndOfFile(EndOfFile),
+    ElementCloseName(ElementCloseName),
-    StartOfFile(StartOfFile),
+    ElementCloseStart(ElementCloseStart),
-    TextContent(TextContent),
+    ElementOpenEnd(ElementOpenEnd),
-    AttributeName(AttributeName),
+    ElementOpenName(ElementOpenName),
-    AttributeValue(AttributeValue),
+    ElementOpenStart(ElementOpenStart),
-    ElementSelfClose(ElementSelfClose),
+    ElementSelfClose(ElementSelfClose),
-}
+    EndOfFile(EndOfFile),
-
+    StartOfFile(StartOfFile),
-impl Token {
+    TextContent(TextContent),
-    pub fn append(self, c: char) -> Vec<Token> {
+}
-        match self {
+
-            Token::ElementCloseName(element_close_name) => element_close_name.append(c),
+impl Token {
-            Token::ElementCloseStart(element_close_start) => element_close_start.append(c),
+    pub fn append(self, c: char) -> Vec<Token> {
-            Token::ElementOpenName(element_name) => element_name.append(c),
+        match self {
-            Token::ElementOpenEnd(element_open) => element_open.append(c),
+            Token::AttributeEquals(attribute_equals) => attribute_equals.append(c),
-            Token::ElementOpenStart(element_start) => element_start.append(c),
+            Token::ElementCloseName(element_close_name) => element_close_name.append(c),
-            Token::EndOfFile(end_of_file) => end_of_file.append(c),
+            Token::ElementCloseStart(element_close_start) => element_close_start.append(c),
-            Token::StartOfFile(start_of_file) => start_of_file.append(c),
+            Token::ElementOpenName(element_name) => element_name.append(c),
-            Token::TextContent(text_content) => text_content.append(c),
+            Token::ElementOpenEnd(element_open) => element_open.append(c),
-            Token::AttributeName(attribute_name) => attribute_name.append(c),
+            Token::ElementOpenStart(element_start) => element_start.append(c),
-            Token::AttributeValue(attribute_value) => attribute_value.append(c),
+            Token::EndOfFile(end_of_file) => end_of_file.append(c),
-            Token::ElementSelfClose(element_self_close) => element_self_close.append(c),
+            Token::StartOfFile(start_of_file) => start_of_file.append(c),
-        }
+            Token::TextContent(text_content) => text_content.append(c),
-    }
+            Token::AttributeName(attribute_name) => attribute_name.append(c),
-
+            Token::AttributeValue(attribute_value) => attribute_value.append(c),
-    pub fn is_eof(&self) -> bool {
+            Token::ElementSelfClose(element_self_close) => element_self_close.append(c),
-        match self {
+        }
-            Token::EndOfFile(_) => true,
+    }
-            _ => false,
+
-        }
+    pub fn is_eof(&self) -> bool {
-    }
+        match self {
-}
+            Token::EndOfFile(_) => true,
-
+            _ => false,
-#[cfg(test)]
+        }
-mod tests {
+    }
-    use super::*;
+}
-    use std::sync::Arc;
+
-    use crate::schema;
+#[cfg(test)]
-
+mod tests {
-    fn create_test_element() -> Arc<schema::XsElement> {
+    use super::*;
-        let element = schema::XsElement {
+    use crate::schema;
-            name: "reasoning".to_string(),
+    use std::sync::Arc;
-            text: None,
+
-            xs_complex_type: schema::XsComplexType {
+    pub fn create_test_element() -> Arc<schema::XsElement> {
-                mixed: Some("true".to_string()),
+        let element = schema::XsElement {
-                text: None,
+            name: "reasoning".to_string(),
-                xs_attribute: None,
+            text: None,
-                xs_sequence: Some(schema::XsSequence {
+            xs_complex_type: schema::XsComplexType {
-                    text: None,
+                mixed: Some("true".to_string()),
-                    xs_any: schema::XsAny {
+                text: None,
-                        min_occurs: "0".to_string(),
+                xs_attribute: None,
-                        max_occurs: "unbounded".to_string(),
+                xs_sequence: Some(schema::XsSequence {
-                        process_contents: "skip".to_string(),
+                    text: None,
-                    },
+                    xs_any: schema::XsAny {
-                }),
+                        min_occurs: "0".to_string(),
-            },
+                        max_occurs: "unbounded".to_string(),
-        };
+                        process_contents: "skip".to_string(),
-        Arc::new(element)
+                    },
-    }
+                }),
-
+            },
-    #[test]
+        };
-    fn test_start_of_file_transition() {
+        Arc::new(element)
-        let element = create_test_element();
+    }
-        let token = Token::StartOfFile(StartOfFile::new(Arc::clone(&element)));
+
-        
+    #[test]
-        // Should only accept '<'
+    fn test_element_start_transition() {
-        let next_tokens = token.append('<');
+        let element = create_test_element();
-        assert_eq!(next_tokens.len(), 1, "Should have one transition");
+        let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
-        
+
-        match &next_tokens[0] {
+        // Should accept first char of element name
-            Token::ElementOpenStart(_) => (),
+        let next_tokens = token.append('r');
-            _ => panic!("Expected ElementStart, got {:?}", next_tokens[0]),
+        assert_eq!(next_tokens.len(), 1, "Should have one transition");
-        }
+
-        
+        match &next_tokens[0] {
-        // Should reject other characters
+            Token::ElementOpenName(_) => (),
-        let token = Token::StartOfFile(StartOfFile::new(Arc::clone(&element)));
+            _ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
-        let next_tokens = token.append('a');
+        }
-        assert_eq!(next_tokens.len(), 0, "Should reject 'a'");
+
-    }
+        // Should reject invalid char
-    
+        let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
-    #[test]
+        let next_tokens = token.append('x');
-    fn test_element_start_transition() {
+        assert_eq!(next_tokens.len(), 0, "Should reject 'x'");
-        let element = create_test_element();
+    }
-        let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
+
-        
+    #[test]
-        // Should accept first char of element name
+    fn test_element_name_transition() {
-        let next_tokens = token.append('r');
+        let element = create_test_element();
-        assert_eq!(next_tokens.len(), 1, "Should have one transition");
+        let token =
-        
+            Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()));
-        match &next_tokens[0] {
+
-            Token::ElementOpenName(_) => (),
+        // Continue building name
-            _ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
+        let next_tokens = token.append('e');
-        }
+        assert_eq!(next_tokens.len(), 1, "Should have one transition");
-        
+
-        // Should reject invalid char
+        match &next_tokens[0] {
-        let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
+            Token::ElementOpenName(_) => (),
-        let next_tokens = token.append('x');
+            _ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
-        assert_eq!(next_tokens.len(), 0, "Should reject 'x'");
+        }
-    }
+
-    
+        // Build complete name and end with >
-    #[test]
+        // Remove the unused buffer
-    fn test_element_name_transition() {
+        let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(
-        let element = create_test_element();
+            &element,
-        let token = Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()).unwrap());
+        )))];
-        
+
-        // Continue building name
+        for c in "reasoning>".chars() {
-        let next_tokens = token.append('e');
+            let mut new_tokens = Vec::new();
-        assert_eq!(next_tokens.len(), 1, "Should have one transition");
+            for token in current_tokens {
-        
+                new_tokens.append(&mut token.append(c));
-        match &next_tokens[0] {
+            }
-            Token::ElementOpenName(_) => (),
+            current_tokens = new_tokens;
-            _ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
+
-        }
+            assert!(
-        
+                !current_tokens.is_empty(),
-        // Build complete name and end with >
+                "Should have valid transition after appending '{}'",
-        // Remove the unused buffer
+                c
-        let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)))];
+            );
-        
+        }
-        for c in "reasoning>".chars() {
+
-            let mut new_tokens = Vec::new();
+        // Check final state
-            for token in current_tokens {
+        assert_eq!(current_tokens.len(), 1, "Should have one final transition");
-                new_tokens.append(&mut token.append(c));
+        match &current_tokens[0] {
-            }
+            Token::ElementOpenEnd(_) => (),
-            current_tokens = new_tokens;
+            _ => panic!("Expected ElementOpen, got {:?}", current_tokens[0]),
-            
+        }
-            assert!(!current_tokens.is_empty(), 
+    }
-                "Should have valid transition after appending '{}'", c);
+
-        }
+    #[test]
-        
+    fn test_element_open_end_transition() {
-        // Check final state
+        let element = create_test_element();
-        assert_eq!(current_tokens.len(), 1, "Should have one final transition");
+        let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
-        match &current_tokens[0] {
+
-            Token::ElementOpenEnd(_) => (),
+        // Should accept text content in mixed element
-            _ => panic!("Expected ElementOpen, got {:?}", current_tokens[0]),
+        let next_tokens = token.append('t');
-        }
+        assert_eq!(next_tokens.len(), 1, "Should accept text");
-    }
+
-    
+        match &next_tokens[0] {
-    #[test]
+            Token::TextContent(_) => (),
-    fn test_element_open_end_transition() {
+            _ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
-        let element = create_test_element();
+        }
-        let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
+
-        
+        // Should accept closing tag start
-        // Should accept text content in mixed element
+        let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
-        let next_tokens = token.append('t');
+        let next_tokens = token.append('<');
-        assert_eq!(next_tokens.len(), 1, "Should accept text");
+        assert_eq!(next_tokens.len(), 1, "Should accept '<'");
-        
+
-        match &next_tokens[0] {
+        match &next_tokens[0] {
-            Token::TextContent(_) => (),
+            Token::ElementCloseStart(_) => (),
-            _ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
+            _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
-        }
+        }
-        
+    }
-        // Should accept closing tag start
+
-        let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
+    #[test]
-        let next_tokens = token.append('<');
+    fn test_text_content_transition() {
-        assert_eq!(next_tokens.len(), 1, "Should accept '<'");
+        let element = create_test_element();
-        
+        let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
-        match &next_tokens[0] {
+
-            Token::ElementCloseStart(_) => (),
+        // Should accept more text
-            _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
+        let next_tokens = token.append('a');
-        }
+        assert_eq!(next_tokens.len(), 1, "Should accept more text");
-    }
+
-    
+        match &next_tokens[0] {
-    #[test]
+            Token::TextContent(_) => (),
-    fn test_text_content_transition() {
+            _ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
-        let element = create_test_element();
+        }
-        let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
+
-        
+        // Should accept closing tag start
-        // Should accept more text
+        let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
-        let next_tokens = token.append('a');
+        let next_tokens = token.append('<');
-        assert_eq!(next_tokens.len(), 1, "Should accept more text");
+        assert_eq!(next_tokens.len(), 1, "Should accept '<'");
-        
+
-        match &next_tokens[0] {
+        match &next_tokens[0] {
-            Token::TextContent(_) => (),
+            Token::ElementCloseStart(_) => (),
-            _ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
+            _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
-        }
+        }
-        
+    }
-        // Should accept closing tag start
+
-        let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
+    #[test]
-        let next_tokens = token.append('<');
+    fn test_element_close_start_transition() {
-        assert_eq!(next_tokens.len(), 1, "Should accept '<'");
+        let element = create_test_element();
-        
+        let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
-        match &next_tokens[0] {
+
-            Token::ElementCloseStart(_) => (),
+        // Should accept /
-            _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
+        let next_tokens = token.append('/');
-        }
+        assert_eq!(next_tokens.len(), 1, "Should accept '/'");
-    }
+
-    
+        let token = next_tokens[0].clone();
-    #[test]
+        match token {
-    fn test_element_close_start_transition() {
+            Token::ElementCloseStart(_) => (),
-        let element = create_test_element();
+            _ => panic!("Expected ElementCloseStart with slash, got {:?}", token),
-        let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
+        }
-        
+
-        // Should accept /
+        // After /, should accept first char of element name
-        let next_tokens = token.append('/');
+        let next_tokens = token.append('r');
-        assert_eq!(next_tokens.len(), 1, "Should accept '/'");
+        assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
-        
+
-        let token = next_tokens[0].clone();
+        match &next_tokens[0] {
-        match token {
+            Token::ElementCloseName(_) => (),
-            Token::ElementCloseStart(_) => (),
+            _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
-            _ => panic!("Expected ElementCloseStart with slash, got {:?}", token),
+        }
-        }
+    }
-        
+
-        // After /, should accept first char of element name
+    #[test]
-        let next_tokens = token.append('r');
+    fn test_element_close_name_transition() {
-        assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
+        let element = create_test_element();
-        
+        let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
-        match &next_tokens[0] {
+
-            Token::ElementCloseName(_) => (),
+        // Add / first
-            _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
+        let next_tokens = token.append('/');
-        }
+        assert_eq!(next_tokens.len(), 1, "Should accept '/'");
-    }
+
-    
+        // Start building name - clone the token to avoid moving out of the vector
-    #[test]
+        let next_tokens = next_tokens[0].clone().append('r');
-    fn test_element_close_name_transition() {
+        assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
-        let element = create_test_element();
+
-        let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
+        // Continue with each character
-        
+        let mut current_tokens = next_tokens;
-        // Add / first
+        for c in "easoning".chars() {
-        let next_tokens = token.append('/');
+            let mut new_tokens = Vec::new();
-        assert_eq!(next_tokens.len(), 1, "Should accept '/'");
+            for token in current_tokens {
-        
+                new_tokens.append(&mut token.append(c));
-        // Start building name - clone the token to avoid moving out of the vector
+            }
-        let next_tokens = next_tokens[0].clone().append('r');
+            current_tokens = new_tokens;
-        assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
+
-        
+            assert!(
-        // Continue with each character
+                !current_tokens.is_empty(),
-        let mut current_tokens = next_tokens;
+                "Should have valid transition after appending '{}'",
-        for c in "easoning".chars() {
+                c
-            let mut new_tokens = Vec::new();
+            );
-            for token in current_tokens {
+        }
-                new_tokens.append(&mut token.append(c));
+
-            }
+        // Final > to close - clone the token to avoid moving out of the vector
-            current_tokens = new_tokens;
+        let next_tokens = current_tokens[0].clone().append('>');
-            
+        assert_eq!(next_tokens.len(), 1, "Should accept '>'");
-            assert!(!current_tokens.is_empty(), 
+
-                "Should have valid transition after appending '{}'", c);
+        match &next_tokens[0] {
-        }
+            Token::EndOfFile(_) => (),
-        
+            _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
-        // Final > to close - clone the token to avoid moving out of the vector
+        }
-        let next_tokens = current_tokens[0].clone().append('>');
+    }
-        assert_eq!(next_tokens.len(), 1, "Should accept '>'");
+
-        
+    #[test]
-        match &next_tokens[0] {
+    fn test_end_of_file_properties() {
-            Token::EndOfFile(_) => (),
+        let token = Token::EndOfFile(EndOfFile::new());
-            _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
+
-        }
+        // Should be identified as EOF
-    }
+        assert!(token.is_eof(), "EndOfFile should be recognized as EOF");
-    
+
-    #[test]
+        // Should not accept any more input
-    fn test_end_of_file_properties() {
+        let next_tokens = token.append('a');
-        let token = Token::EndOfFile(EndOfFile::new());
+        assert_eq!(next_tokens.len(), 0, "Should not accept any more input");
-        
+    }
-        // Should be identified as EOF
+}
-        assert!(token.is_eof(), "EndOfFile should be recognized as EOF");
-        
-        // Should not accept any more input
-        let next_tokens = token.append('a');
-        assert_eq!(next_tokens.len(), 0, "Should not accept any more input");
-    }
-}

lib/xml_schema_validator/src/token/start_of_file.rs

@@ -1,75 +1,52 @@
-use std::sync::Arc;
+use crate::*;
-use crate::*;
+use std::sync::Arc;
-
+
-/// Represents the initial state at the start of an XML file, containing a reference to the root element schema
+/// Represents the initial state at the start of an XML file, containing a reference to the root element schema
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug)]
-pub struct StartOfFile {
+pub struct StartOfFile {
-    element: Arc<schema::XsElement>,
+    element: Arc<schema::XsElement>,
-}
+}
-
+
-impl StartOfFile {
+impl StartOfFile {
-    pub fn new(element: Arc<schema::XsElement>) -> Self {
+    pub fn new(element: Arc<schema::XsElement>) -> Self {
-        Self { element }
+        Self { element }
-    }
+    }
-
+
-    pub fn append(self, c: char) -> Vec<Token> {
+    pub fn append(self, c: char) -> Vec<Token> {
-        if '<' == c {
+        if '<' == c {
-            vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&self.element)))]
+            vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(
-        } else {
+                &self.element,
-            vec![]
+            )))]
-        }
+        } else {
-    }
+            vec![]
-}
+        }
-
+    }
-#[cfg(test)]
+}
-mod start_of_file_tests {
+
-    use super::*;
+#[cfg(test)]
-    use std::sync::Arc;
+mod tests {
-    use crate::schema;
+    use super::*;
-
+
-    fn create_test_element() -> Arc<schema::XsElement> {
+    #[test]
-        let element = schema::XsElement {
+    fn test_accept_open_start_token() {
-            name: "reasoning".to_string(),
+        let mut validator = crate::tests::example_validator();
-            text: None,
+        validator.append("<").unwrap();
-            xs_complex_type: schema::XsComplexType {
+        assert!(!validator.current_tokens.is_empty());
-                mixed: Some("true".to_string()),
+        assert!(validator
-                text: None,
+            .current_tokens
-                xs_attribute: None,
+            .iter()
-                xs_sequence: Some(schema::XsSequence {
+            .filter(|token| matches!(token, Token::ElementOpenStart(_)))
-                    text: None,
+            .next()
-                    xs_any: schema::XsAny {
+            .is_some());
-                        min_occurs: "0".to_string(),
+    }
-                        max_occurs: "unbounded".to_string(),
+
-                        process_contents: "skip".to_string(),
+    #[test]
-                    },
+    fn test_not_accept_other() {
-                }),
+        let values = ["a", " ", "\t", ">", "/"];
-            },
+        for value in values {
-        };
+            let mut validator = crate::tests::example_validator();
-        Arc::new(element)
+            validator.append(value).expect_err(value);
-    }
+            assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
-
+        }
-    #[test]
+    }
-    fn test_start_of_file_less_than() {
+}
-        let element = create_test_element();
-        let token = StartOfFile::new(Arc::clone(&element));
-        
-        // Test with <
-        let next_tokens = token.append('<');
-        assert!(!next_tokens.is_empty(), "Should accept '<'");
-        
-        match &next_tokens[0] {
-            Token::ElementOpenStart(_) => (),
-            _ => panic!("Expected ElementStart, got {:?}", next_tokens[0]),
-        }
-    }
-
-    #[test]
-    fn test_start_of_file_invalid() {
-        let element = create_test_element();
-        let token = StartOfFile::new(Arc::clone(&element));
-        
-        // Test with invalid character
-        let next_tokens = token.append('a');
-        assert!(next_tokens.is_empty(), "Should reject invalid character");
-    }
-}

lib/xml_schema_validator/src/token/text_content.rs

@@ -1,84 +1,84 @@
-use std::sync::Arc;
+use crate::*;
-use crate::*;
+use std::sync::Arc;
-
+
-/// Represents the text content within an XML element
+/// Represents the text content within an XML element
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug)]
-pub struct TextContent {
+pub struct TextContent {
-    element: Arc<schema::XsElement>,
+    element: Arc<schema::XsElement>,
-}
+}
-
+
-impl TextContent {
+impl TextContent {
-    pub fn new(element: Arc<schema::XsElement>) -> Self {
+    pub fn new(element: Arc<schema::XsElement>) -> Self {
-        Self {
+        Self { element }
-            element,
+    }
-        }
+
-    }
+    pub fn append(self, c: char) -> Vec<Token> {
-
+        if c == '<' {
-    pub fn append(self, c: char) -> Vec<Token> {
+            vec![Token::ElementCloseStart(ElementCloseStart::new(
-        if c == '<' {
+                Arc::clone(&self.element),
-            vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
+            ))]
-        } else {
+        } else {
-            vec![Token::TextContent(TextContent::new(
+            vec![Token::TextContent(TextContent::new(Arc::clone(
-                Arc::clone(&self.element),
+                &self.element,
-            ))]
+            )))]
-        }
+        }
-    }
+    }
-}
+}
-
+
-#[cfg(test)]
+#[cfg(test)]
-mod text_content_tests {
+mod text_content_tests {
-    use super::*;
+    use super::*;
-    use std::sync::Arc;
+    use crate::schema;
-    use crate::schema;
+    use std::sync::Arc;
-
+
-    fn create_test_element() -> Arc<schema::XsElement> {
+    fn create_test_element() -> Arc<schema::XsElement> {
-        let element = schema::XsElement {
+        let element = schema::XsElement {
-            name: "reasoning".to_string(),
+            name: "reasoning".to_string(),
-            text: None,
+            text: None,
-            xs_complex_type: schema::XsComplexType {
+            xs_complex_type: schema::XsComplexType {
-                mixed: Some("true".to_string()),
+                mixed: Some("true".to_string()),
-                text: None,
+                text: None,
-                xs_attribute: None,
+                xs_attribute: None,
-                xs_sequence: Some(schema::XsSequence {
+                xs_sequence: Some(schema::XsSequence {
-                    text: None,
+                    text: None,
-                    xs_any: schema::XsAny {
+                    xs_any: schema::XsAny {
-                        min_occurs: "0".to_string(),
+                        min_occurs: "0".to_string(),
-                        max_occurs: "unbounded".to_string(),
+                        max_occurs: "unbounded".to_string(),
-                        process_contents: "skip".to_string(),
+                        process_contents: "skip".to_string(),
-                    },
+                    },
-                }),
+                }),
-            },
+            },
-        };
+        };
-        Arc::new(element)
+        Arc::new(element)
-    }
+    }
-
+
-    #[test]
+    #[test]
-    fn test_text_content_normal_char() {
+    fn test_text_content_normal_char() {
-        let element = create_test_element();
+        let element = create_test_element();
-        let token = TextContent::new(Arc::clone(&element));
+        let token = TextContent::new(Arc::clone(&element));
-        
+
-        // Test with normal character
+        // Test with normal character
-        let next_tokens = token.append('a');
+        let next_tokens = token.append('a');
-        assert!(!next_tokens.is_empty(), "Should accept normal character");
+        assert!(!next_tokens.is_empty(), "Should accept normal character");
-        
+
-        match &next_tokens[0] {
+        match &next_tokens[0] {
-            Token::TextContent(_) => (),
+            Token::TextContent(_) => (),
-            _ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
+            _ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
-        }
+        }
-    }
+    }
-
+
-    #[test]
+    #[test]
-    fn test_text_content_less_than() {
+    fn test_text_content_less_than() {
-        let element = create_test_element();
+        let element = create_test_element();
-        let token = TextContent::new(Arc::clone(&element));
+        let token = TextContent::new(Arc::clone(&element));
-        
+
-        // Test with < (start of tag)
+        // Test with < (start of tag)
-        let next_tokens = token.append('<');
+        let next_tokens = token.append('<');
-        assert!(!next_tokens.is_empty(), "Should accept '<'");
+        assert!(!next_tokens.is_empty(), "Should accept '<'");
-        
+
-        match &next_tokens[0] {
+        match &next_tokens[0] {
-            Token::ElementCloseStart(_) => (),
+            Token::ElementCloseStart(_) => (),
-            _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
+            _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
-        }
+        }
-    }
+    }
-}
+}

lib/xml_schema_validator/tests/test_logits_processor.py

@@ -1,13 +1,12 @@
 from transformers import AutoTokenizer
 from xml_schema_validator import XmlLogitsProcessor
 import os
-import time
 import torch
 
 model_path = "../../model"
 api_token = os.environ["SIA_HF_API_KEY"]
 
-xml_schema_actions = open("../../action_schema.xsd").read()
+xml_schema_actions = open("example_schema.xsd").read()
 xml_schema_only_root_node = """<?xml version="1.0" encoding="UTF-8"?>
 <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
   <xs:element name="root">
@@ -59,60 +58,27 @@ def test_token_masking():
     
     tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token)
     processor = XmlLogitsProcessor(tokenizer, xml_schema_actions)
+
+    # Process empty input to set prompt length
+    input_ids = torch.tensor([tokenizer.encode("")])
+    scores = torch.ones((1, len(tokenizer)))  # All tokens have equal probability
+    processed_scores = processor(input_ids, scores)
     
-    # Create dummy input_ids and scores
+    # Process valid continuation
     input_ids = torch.tensor([tokenizer.encode("<reasoning>")])
     scores = torch.ones((1, len(tokenizer)))  # All tokens have equal probability
-    
-    # Process the scores
     processed_scores = processor(input_ids, scores)
     
     # Verify that valid continuations still have positive scores
-    assert processed_scores[0, tokenizer.encode(" ", add_special_tokens=False)[0]] > -float('inf')
+    space_token = tokenizer.encode(" ", add_special_tokens=False)[0]
+    assert processed_scores[0, space_token] > -float('inf')
     
     # Verify that invalid continuations are masked
-    assert processed_scores[0, tokenizer.encode("<invalid>", add_special_tokens=False)[0]] == -float('inf')
+    input_ids = torch.tensor([tokenizer.encode("</invalid>")])
-
+    scores = torch.ones((1, len(tokenizer)))  # All tokens have equal probability
-def test_performance():
+    processed_scores = processor(input_ids, scores)
-    """Test performance with larger XML documents"""
+    assert processed_scores[0, tokenizer.eos_token_id] == 1
-    
+    assert processed_scores[0, space_token] == -float('inf')
-    tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token)
-    processor = XmlLogitsProcessor(tokenizer, xml_schema_actions)
-    
-    # Generate a larger XML document
-    large_xml = "<reasoning>" + "Test content. " * 100 + "</reasoning>"
-    
-    # Measure time to process
-    start_time = time.time()
-    processor.core.append(large_xml)
-    processing_time = time.time() - start_time
-    
-    print(f"Processing time for large XML: {processing_time:.4f} seconds")
-    # You might want to assert that processing time is below a threshold
-
-def test_subword_tokens():
-    """Test handling of subword tokens that might split XML tags"""
-    tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token)
-    schema_text = open("../../action_schema.xsd").read()
-    
-    processor = XmlLogitsProcessor(tokenizer, schema_text)
-    
-    # Find some XML tag that gets split into multiple tokens by your tokenizer
-    tag = "<reasoning>"
-    tokens = tokenizer.encode(tag, add_special_tokens=False)
-    
-    if len(tokens) > 1:
-        print(f"Tag '{tag}' is split into {len(tokens)} tokens")
-        
-        # Test that the processor can handle these split tokens
-        import torch
-        input_ids = torch.tensor([[tokens[0]]])
-        scores = torch.ones((1, len(tokenizer)))
-        
-        processed_scores = processor(input_ids, scores)
-        
-        # The next token in the sequence should have a high score
-        assert processed_scores[0, tokens[1]] > -float('inf')
     
 if __name__ == "__main__":
     # Run individual tests for debugging
@@ -120,5 +86,3 @@ if __name__ == "__main__":
     test_xml_schema_parsing()
     test_basic_xml_validation()
     test_token_masking()
-    test_performance()
-    test_subword_tokens()