Fixed attribute parsing

lib/xml_schema_validator/example_schema.xsd

@@ -0,0 +1,118 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Always answer with a single element.
+-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
+
+  <!--
+    Delete command removes an entry from the context by its ID.
+    Use it to remove unnecessary items and stop background processes.
+    When you delete something, it is gone.
+    Make sure all important info is stored in files.
+    Example:
+    <delete id="1234567890"/>
+  -->
+  <xs:element name="delete">
+    <xs:complexType>
+      <xs:attribute name="id" type="xs:string" use="required"/>
+    </xs:complexType>
+  </xs:element>
+
+  <!--
+    Stop command terminates the agent gracefully.
+    For the main SIA instance this will trigger an update and restart.
+    For sub-instances this is the correct way to stop after all tasks are complete.
+    Example:
+    <stop id="1234567890"/>
+  -->
+  <xs:element name="stop">
+    <xs:complexType/>
+  </xs:element>
+
+  <!--
+    Single script that runs once and completes.
+    Output is stored in context until explicitly deleted.
+    Used for one-time operations like file manipulation.
+    Single scripts are limited to 1024 characters and 1 second timeout by default.
+    These limits can be changed with attributes.
+    Example:
+    <single>
+      ls /
+    </single>
+  -->
+  <xs:element name="single">
+    <xs:complexType mixed="true">
+      <xs:sequence>
+        <xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
+      </xs:sequence>
+      <xs:attribute name="timeout" type="xs:float" use="optional"/>
+      <xs:attribute name="limit" type="xs:integer" use="optional"/>
+    </xs:complexType>
+  </xs:element>
+
+  <!--
+    Repeat script runs each time the context is generated.
+    After a command is issued, all repeat scripts in context are run again.
+    Useful for monitoring changing files or viewing results immediately after changing a file.
+    Repeat scripts should execute quickly to avoid blocking the agent.
+    Repeat scripts are limited to 1024 characters and 1 second timeout by default.
+    These limits can be changed with attributes.
+    Example:
+    <repeat>
+      ls /
+    </repeat>
+  -->
+  <xs:element name="repeat">
+    <xs:complexType mixed="true">
+      <xs:sequence>
+        <xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
+      </xs:sequence>
+      <xs:attribute name="timeout" type="xs:float" use="optional"/>
+      <xs:attribute name="limit" type="xs:integer" use="optional"/>
+    </xs:complexType>
+  </xs:element>
+
+  <!--
+    As an agent it is important to reason about your actions and their results.
+    In a reasoning action you can write freeform text.
+    This is also stored in context until deleted.
+    Example:
+    <reasoning>
+      I should explore the file system for interesting files.
+    </reasoning>
+  -->
+  <xs:element name="reasoning">
+    <xs:complexType mixed="true">
+      <xs:sequence>
+        <xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+
+  <!--
+    Read all available text on stdin and store it in context.
+    Do this only if the context indicates there is data in the stdin buffer.
+    Example:
+    <read_stdin/>
+  -->
+  <xs:element name="read_stdin">
+    <xs:complexType/>
+  </xs:element>
+
+  <!--
+    Write to stdout.
+    This is your main way of contacting the user.
+    Make sure you have properly reasoned about what to say and if it is necessary before issuing a write_stdout command.
+    Example:
+    <write_stdout>
+      Hello world!
+    </write_stdout>
+  -->
+  <xs:element name="write_stdout">
+    <xs:complexType mixed="true">
+      <xs:sequence>
+        <xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+</xs:schema>

lib/xml_schema_validator/python/xml_schema_validator/__init__.py

@@ -12,7 +12,7 @@ class XmlLogitsProcessor(LogitsProcessor):
     by setting their logits to negative infinity.
     """
     
-    def __init__(self, tokenizer: AutoTokenizer, schema_text: str = None, core=None):
+    def __init__(self, tokenizer: AutoTokenizer, schema_text: str = None, core: XmlLogitsProcessorCore=None):
         """
         Initialize the processor with a schema and tokenizer.
         

lib/xml_schema_validator/src/char_trie.rs

@@ -36,8 +36,16 @@ impl CharTrieNode {
 
     pub fn clone_ref(&self, py: Python<'_>) -> Self {
         Self {
-            children: self.children.iter().map(|(ch, node)| (*ch, node.clone_ref(py))).collect(),
+            children: self
-            token_id: self.token_id.iter().map(|token_id| token_id.clone_ref(py)).nth(0),
+                .children
+                .iter()
+                .map(|(ch, node)| (*ch, node.clone_ref(py)))
+                .collect(),
+            token_id: self
+                .token_id
+                .iter()
+                .map(|token_id| token_id.clone_ref(py))
+                .nth(0),
         }
     }
 }
@@ -68,7 +76,12 @@ impl CharTrie {
     }
 
     // Find all tokens that could be invalid given the current XML state
-    pub fn find_invalid_tokens(&self, py: Python<'_>, xml_validator: &XmlSchemaValidator, token_map: &Vec<(Py<PyAny>, String)>) -> Vec<Py<PyAny>> {
+    pub fn find_invalid_tokens(
+        &self,
+        py: Python<'_>,
+        xml_validator: &XmlSchemaValidator,
+        token_map: &Vec<(Py<PyAny>, String)>,
+    ) -> Vec<Py<PyAny>> {
         let mut invalid_tokens = Vec::new();
 
         for (token_id, token_text) in token_map {

lib/xml_schema_validator/src/lib.rs

@@ -11,8 +11,8 @@ mod python;
 mod schema;
 mod token;
 
-use std::sync::Arc;
 use error::Error;
+use std::sync::Arc;
 use token::*;
 
 /// An XML validator that checks XML fragments against an XSD schema
@@ -33,10 +33,14 @@ impl XmlSchemaValidator {
         let schema: schema::XsSchema = quick_xml::de::from_str(schema_text)?;
         let schema_arc = Arc::new(schema);
 
-        let current_tokens = schema_arc.xs_element.iter().map(|element| {
+        let current_tokens = schema_arc
+            .xs_element
+            .iter()
+            .map(|element| {
                 let element_arc = Arc::new(element.clone());
                 Token::StartOfFile(StartOfFile::new(element_arc))
-        }).collect();
+            })
+            .collect();
 
         Ok(Self {
             current_tokens,
@@ -64,7 +68,10 @@ impl XmlSchemaValidator {
             new_tokens.append(&mut token.append(c));
         }
         if new_tokens.is_empty() {
-            return Err(Error::InvalidXml(format!("No valid continuations for character: '{}'", c)));
+            return Err(Error::InvalidXml(format!(
+                "No valid continuations for character: '{}'",
+                c
+            )));
         }
         self.current_tokens = new_tokens;
         Ok(())
@@ -79,7 +86,9 @@ impl XmlSchemaValidator {
 
     /// Get a list of all element names in the schema
     pub fn get_element_names(&self) -> Vec<String> {
-        self.schema.xs_element.iter()
+        self.schema
+            .xs_element
+            .iter()
             .map(|element| element.name.clone())
             .collect()
     }
@@ -89,19 +98,22 @@ impl XmlSchemaValidator {
 mod tests {
     use super::*;
 
+    pub fn example_validator() -> XmlSchemaValidator {
+        let schema_text = std::fs::read_to_string("example_schema.xsd").unwrap();
+        XmlSchemaValidator::new(&schema_text).unwrap()
+    }
+
     #[test]
     fn test_element_open() {
-        let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
+        let mut validator = example_validator();
         let input = "<reasoning>";
-        let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
         validator.append(input).unwrap();
     }
 
     #[test]
     fn test_element_open_whitespace() {
-        let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
+        let mut validator = example_validator();
         let input = "<reasoning >";
-        let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
         validator.append(input).unwrap();
     }
 
@@ -126,7 +138,9 @@ mod tests {
         let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
         let input = "<reasoning>test</wrong>";
         let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
-        validator.append(input).expect_err("Should fail with mismatched closing tag");
+        validator
+            .append(input)
+            .expect_err("Should fail with mismatched closing tag");
     }
 
     #[test]
@@ -145,12 +159,15 @@ mod tests {
         let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
 
         // First, validate up to the opening tag
-        assert!(validator.append("<delete>").is_err(), "Should fail with missing required attribute");
+        assert!(
+            validator.append("<delete>").is_err(),
+            "Should fail with missing required attribute"
+        );
 
         // Create a new validator and try closing tag approach
         let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
         // Try to validate an incomplete delete tag that's missing the required id attribute
-        let input = "<delete></delete>";
+        let input = "<delete/>";
         let result = validator.append(input);
         assert!(result.is_err(), "Should fail without required id attribute");
 
@@ -158,8 +175,10 @@ mod tests {
         if let Err(err) = result {
             println!("Error message: {}", err);
             let error_str = err.to_string().to_lowercase();
-            assert!(error_str.contains("required") || error_str.contains("no valid continuations"), 
+            assert!(
-                    "Error should mention missing required attribute or invalid continuation");
+                error_str.contains("required") || error_str.contains("no valid continuations"),
+                "Error should mention missing required attribute or invalid continuation"
+            );
         }
     }
 
@@ -167,11 +186,26 @@ mod tests {
     fn test_invalid_nesting() {
         let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
         let validator = XmlSchemaValidator::new(&schema_text).unwrap();
-        validator.clone().append("<reasoning><reasoning>").expect_err("Should fail because of nested tag");
+        validator
-        validator.clone().append("<reasoning> <reasoning>").expect_err("Should fail because of nested tag, even with whitespace");
+            .clone()
-        validator.clone().append("<reasoning>foo<reasoning>").expect_err("Should fail because of nested tag, even with text");
+            .append("<reasoning><reasoning>")
-        validator.clone().append("<reasoning> foo <reasoning>").expect_err("Should fail because of nested tag, even with text and whitespace");
+            .expect_err("Should fail because of nested tag");
-        validator.clone().append("<reasoning> foo <reasoning/>").expect_err("Should fail because of nested tag, even with self closing tag");
+        validator
+            .clone()
+            .append("<reasoning> <reasoning>")
+            .expect_err("Should fail because of nested tag, even with whitespace");
+        validator
+            .clone()
+            .append("<reasoning>foo<reasoning>")
+            .expect_err("Should fail because of nested tag, even with text");
+        validator
+            .clone()
+            .append("<reasoning> foo <reasoning>")
+            .expect_err("Should fail because of nested tag, even with text and whitespace");
+        validator
+            .clone()
+            .append("<reasoning> foo <reasoning/>")
+            .expect_err("Should fail because of nested tag, even with self closing tag");
     }
 
     #[test]
@@ -204,7 +238,8 @@ mod tests {
     #[test]
     fn test_reasoning_with_content() {
         let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
-        let input = "<reasoning>I should explore the file system for interesting files.</reasoning>";
+        let input =
+            "<reasoning>I should explore the file system for interesting files.</reasoning>";
         let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
         validator.append(input).unwrap();
         assert!(validator.eof());
@@ -223,7 +258,11 @@ mod tests {
     fn test_create_validator() {
         let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
         let validator = XmlSchemaValidator::new(&schema_text);
-        assert!(validator.is_ok(), "Failed to create validator: {:?}", validator.err());
+        assert!(
+            validator.is_ok(),
+            "Failed to create validator: {:?}",
+            validator.err()
+        );
     }
 
     #[test]
@@ -231,8 +270,11 @@ mod tests {
         let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
         let validator = XmlSchemaValidator::new(&schema_text).unwrap();
         let names = validator.get_element_names();
-        assert!(names.contains(&"reasoning".to_string()), 
+        assert!(
-                "Element names should contain 'reasoning', got: {:?}", names);
+            names.contains(&"reasoning".to_string()),
+            "Element names should contain 'reasoning', got: {:?}",
+            names
+        );
     }
 
     // Test basic XML fragment validation - debugging the "<reasoning>" failure
@@ -255,7 +297,11 @@ mod tests {
         // Try the whole string at once
         let mut v4 = validator.clone();
         let result = v4.append("<reasoning>");
-        assert!(result.is_ok(), "Failed to append '<reasoning>': {:?}", result.err());
+        assert!(
+            result.is_ok(),
+            "Failed to append '<reasoning>': {:?}",
+            result.err()
+        );
     }
 
     // Test validation of multiple fragments
@@ -265,10 +311,19 @@ mod tests {
         let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
         // Test sequential appends
         assert!(validator.append("<").is_ok(), "Failed to append '<'");
-        assert!(validator.append("reasoning").is_ok(), "Failed to append 'reasoning'");
+        assert!(
+            validator.append("reasoning").is_ok(),
+            "Failed to append 'reasoning'"
+        );
         assert!(validator.append(">").is_ok(), "Failed to append '>'");
-        assert!(validator.append("test content").is_ok(), "Failed to append content");
+        assert!(
-        assert!(validator.append("</reasoning>").is_ok(), "Failed to append closing tag");
+            validator.append("test content").is_ok(),
+            "Failed to append content"
+        );
+        assert!(
+            validator.append("</reasoning>").is_ok(),
+            "Failed to append closing tag"
+        );
         // Should be at EOF now
         assert!(validator.eof(), "Should be at EOF after complete XML");
     }
@@ -279,15 +334,30 @@ mod tests {
         let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
         // Test at start
         let validator = XmlSchemaValidator::new(&schema_text).unwrap();
-        assert!(validator.can_continue_with('<'), "Should accept '<' at start");
+        assert!(
-        assert!(!validator.can_continue_with('a'), "Should reject 'a' at start");
+            validator.can_continue_with('<'),
+            "Should accept '<' at start"
+        );
+        assert!(
+            !validator.can_continue_with('a'),
+            "Should reject 'a' at start"
+        );
         // Test after opening tag
         let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
         validator.append("<reasoning>").unwrap();
         // Any character should be valid inside a mixed content element
-        assert!(validator.can_continue_with('a'), "Should accept 'a' inside element");
+        assert!(
-        assert!(validator.can_continue_with('1'), "Should accept '1' inside element");
+            validator.can_continue_with('a'),
-        assert!(validator.can_continue_with('<'), "Should accept '<' inside element");
+            "Should accept 'a' inside element"
+        );
+        assert!(
+            validator.can_continue_with('1'),
+            "Should accept '1' inside element"
+        );
+        assert!(
+            validator.can_continue_with('<'),
+            "Should accept '<' inside element"
+        );
     }
 }
 
@@ -307,7 +377,8 @@ mod diagnostic_tests {
               </xs:sequence>
             </xs:complexType>
           </xs:element>
-        </xs:schema>"#.to_string()
+        </xs:schema>"#
+            .to_string()
     }
 
     // Test every character in an opening tag one by one
@@ -361,16 +432,23 @@ mod diagnostic_tests {
         });
 
         // Test the transition from ElementOpenName to ElementOpenEnd
-        let element_name = token::ElementOpenName::new(Arc::clone(&test_element), "reasoning".to_string()).unwrap();
+        let element_name =
+            token::ElementOpenName::new(Arc::clone(&test_element), "reasoning".to_string());
         let tokens = element_name.append('>');
         assert!(!tokens.is_empty(), "Should produce valid tokens after '>'");
-        println!("Token after appending '>' to ElementOpenName: {:?}", tokens[0]);
+        println!(
+            "Token after appending '>' to ElementOpenName: {:?}",
+            tokens[0]
+        );
 
         // Check what type the token is
         match &tokens[0] {
             Token::ElementOpenEnd(element_open) => {
-                println!("Successfully transitioned to ElementOpen: {:?}", element_open);
+                println!(
-            },
+                    "Successfully transitioned to ElementOpen: {:?}",
+                    element_open
+                );
+            }
             other => {
                 panic!("Expected ElementOpen, got {:?}", other);
             }
@@ -392,7 +470,8 @@ mod diagnostic_tests {
         });
 
         // Test transition from ElementName to ElementSelfClose
-        let element_name = token::ElementOpenName::new(Arc::clone(&test_element), "stop".to_string()).unwrap();
+        let element_name =
+            token::ElementOpenName::new(Arc::clone(&test_element), "stop".to_string());
         let tokens = element_name.append('/');
         assert!(!tokens.is_empty(), "Should handle '/' after element name");
 
@@ -407,7 +486,7 @@ mod diagnostic_tests {
                     Token::EndOfFile(_) => println!("Successfully transitioned to EndOfFile"),
                     other => panic!("Expected EndOfFile, got {:?}", other),
                 }
-            },
+            }
             other => panic!("Expected ElementSelfClose, got {:?}", other),
         }
     }
@@ -427,8 +506,14 @@ mod diagnostic_tests {
         println!("State after '<reasoning ': {:?}", validator.current_tokens);
 
         // Test completion with >
-        assert!(validator.can_continue_with('>'), "Should accept '>' after whitespace");
+        assert!(
-        assert!(validator.append_char('>').is_ok(), "Failed to append '>' after whitespace");
+            validator.can_continue_with('>'),
+            "Should accept '>' after whitespace"
+        );
+        assert!(
+            validator.append_char('>').is_ok(),
+            "Failed to append '>' after whitespace"
+        );
     }
 
     // Test attribute handling
@@ -452,22 +537,49 @@ mod diagnostic_tests {
 
         // Test attribute name
         for c in "id".chars() {
-            assert!(validator.can_continue_with(c), "Should accept '{}' in attribute name", c);
+            assert!(
-            assert!(validator.append_char(c).is_ok(), "Failed to append '{}' in attribute name", c);
+                validator.can_continue_with(c),
+                "Should accept '{}' in attribute name",
+                c
+            );
+            assert!(
+                validator.append_char(c).is_ok(),
+                "Failed to append '{}' in attribute name",
+                c
+            );
         }
 
         // Test attribute equals and quotes
         for c in "=\"123\"".chars() {
             println!("Testing attribute character: '{}'", c);
-            assert!(validator.can_continue_with(c), "Should accept '{}' in attribute value", c);
+            assert!(
-            assert!(validator.append_char(c).is_ok(), "Failed to append '{}' in attribute value", c);
+                validator.can_continue_with(c),
+                "Should accept '{}' in attribute value",
+                c
+            );
+            assert!(
+                validator.append_char(c).is_ok(),
+                "Failed to append '{}' in attribute value",
+                c
+            );
         }
 
         // Test closing the self-closing tag
-        println!("Current state before closing: {:?}", validator.current_tokens);
+        println!(
+            "Current state before closing: {:?}",
+            validator.current_tokens
+        );
         for c in "/>".chars() {
-            assert!(validator.can_continue_with(c), "Should accept '{}' to close tag", c);
+            assert!(
-            assert!(validator.append_char(c).is_ok(), "Failed to append '{}' to close tag", c);
+                validator.can_continue_with(c),
+                "Should accept '{}' to close tag",
+                c
+            );
+            assert!(
+                validator.append_char(c).is_ok(),
+                "Failed to append '{}' to close tag",
+                c
+            );
         }
 
         // Check if we've reached EOF
@@ -477,7 +589,8 @@ mod diagnostic_tests {
     // Test for the specific error patterns seen in the test output
     #[test]
     fn test_specific_error_cases() {
-        let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap_or_else(|_| get_simple_schema());
+        let schema_text = std::fs::read_to_string("../../action_schema.xsd")
+            .unwrap_or_else(|_| get_simple_schema());
 
         // Test case 1: Simple opening tag (noticed in `test_element_open`)
         let input1 = "<reasoning>";
@@ -590,8 +703,12 @@ mod diagnostic_tests {
                 for c in open_tag.chars() {
                     progress.push(c);
                     let result = element_validator.can_continue_with(c);
-                    println!("  Can continue with '{}' after '{}': {}", 
+                    println!(
-                              c, &progress[..progress.len()-1], result);
+                        "  Can continue with '{}' after '{}': {}",
+                        c,
+                        &progress[..progress.len() - 1],
+                        result
+                    );
 
                     if !result {
                         // This is helpful to diagnose where the process fails
@@ -600,7 +717,10 @@ mod diagnostic_tests {
                     }
 
                     if let Err(e) = element_validator.append_char(c) {
-                        println!("  ❌ Failed to append '{}' for element '{}': {:?}", c, name, e);
+                        println!(
+                            "  ❌ Failed to append '{}' for element '{}': {:?}",
+                            c, name, e
+                        );
                         break;
                     }
                 }

lib/xml_schema_validator/src/py_xml_logits_processor_core.rs

@@ -1,6 +1,6 @@
+use crate::*;
 use pyo3::prelude::*;
 use pyo3::types::PyDict;
-use crate::*;
 
 /// A minimal LogitsProcessor that enforces valid XML according to a schema.
 #[pyclass]
@@ -18,7 +18,10 @@ impl XmlLogitsProcessorCore {
             Ok(xml_schema_validator) => xml_schema_validator,
             Err(e) => return Err(pyo3::exceptions::PyValueError::new_err(e.to_string())),
         };
-        let tokens: Vec<(Py<PyAny>, String)> = tokens.iter().map(|(id, value)| (id.unbind(), value.unbind().to_string())).collect();
+        let tokens: Vec<(Py<PyAny>, String)> = tokens
+            .iter()
+            .map(|(id, value)| (id.unbind(), value.unbind().to_string()))
+            .collect();
         let mut trie = char_trie::CharTrie::new();
         for (token_id, token_text) in tokens.iter() {
             trie.insert(token_text, token_id.clone_ref(py));
@@ -38,7 +41,9 @@ impl XmlLogitsProcessorCore {
 
     /// Get a list of tokens that would lead to invalid XML
     fn get_invalid_tokens(&mut self, py: Python<'_>) -> PyResult<Vec<PyObject>> {
-        Ok(self.trie.find_invalid_tokens(py, &self.xml_schema_validator, &self.tokens))
+        Ok(self
+            .trie
+            .find_invalid_tokens(py, &self.xml_schema_validator, &self.tokens))
     }
 
     /// Check if the validator has reached the end of the XML
@@ -50,7 +55,11 @@ impl XmlLogitsProcessorCore {
     fn copy(&self, py: Python<'_>) -> PyResult<Self> {
         Ok(Self {
             xml_schema_validator: self.xml_schema_validator.clone(),
-            tokens: self.tokens.iter().map(|(a, b)|(a.clone_ref(py), b.clone())).collect(),
+            tokens: self
+                .tokens
+                .iter()
+                .map(|(a, b)| (a.clone_ref(py), b.clone()))
+                .collect(),
             trie: self.trie.clone_ref(py),
         })
     }
@@ -79,8 +88,8 @@ pub fn register_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()>
 #[cfg(test)]
 mod tests {
     use super::*;
-    use pyo3::Python;
     use pyo3::types::PyDict;
+    use pyo3::Python;
 
     fn create_test_processor<'py>(py: Python<'py>) -> PyResult<XmlLogitsProcessorCore> {
         let tokens = PyDict::new(py);
@@ -116,8 +125,10 @@ mod tests {
             assert!(processor.append("<reasoning>"), "Should accept opening tag");
 
             let mut processor = create_test_processor(py).unwrap();
-            assert!(processor.append("<reasoning>test</reasoning>"), 
+            assert!(
-                   "Should accept complete element with content");
+                processor.append("<reasoning>test</reasoning>"),
+                "Should accept complete element with content"
+            );
 
             // Test with real schema from disk if available
             if let Ok(schema_text) = std::fs::read_to_string("../../action_schema.xsd") {
@@ -126,9 +137,12 @@ mod tests {
                 tokens.set_item(1, "reasoning").unwrap();
                 tokens.set_item(2, ">").unwrap();
 
-                let mut processor = XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap();
+                let mut processor =
-                assert!(processor.append("<reasoning>"), 
+                    XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap();
-                       "Should accept <reasoning> with actual schema");
+                assert!(
+                    processor.append("<reasoning>"),
+                    "Should accept <reasoning> with actual schema"
+                );
             }
         });
     }
@@ -139,11 +153,16 @@ mod tests {
             let mut processor = create_test_processor(py).unwrap();
 
             // Test invalid XML fragments
-            assert!(!processor.append("<invalid>"), "Should reject invalid element");
+            assert!(
+                !processor.append("<invalid>"),
+                "Should reject invalid element"
+            );
 
             let mut processor = create_test_processor(py).unwrap();
-            assert!(!processor.append("<reasoning></invalid>"), 
+            assert!(
-                   "Should reject mismatched tags");
+                !processor.append("<reasoning></invalid>"),
+                "Should reject mismatched tags"
+            );
         });
     }
 
@@ -158,18 +177,29 @@ mod tests {
             // After complete document, should be at EOF
             let mut processor = create_test_processor(py).unwrap();
             // Use a properly formed XML document that matches the schema
-            assert!(processor.append("<reasoning>test</reasoning>"), 
+            assert!(
-                "Should accept complete document");
+                processor.append("<reasoning>test</reasoning>"),
+                "Should accept complete document"
+            );
 
             // DEBUG: Print the token state to see why EOF detection fails
-            println!("Current tokens: {:?}", processor.xml_schema_validator.current_tokens);
+            println!(
+                "Current tokens: {:?}",
+                processor.xml_schema_validator.current_tokens
+            );
 
-            assert!(processor.eof().unwrap(), "Should be at EOF after complete document");
+            assert!(
+                processor.eof().unwrap(),
+                "Should be at EOF after complete document"
+            );
 
             // After opening tag, should not be at EOF
             let mut processor = create_test_processor(py).unwrap();
             assert!(processor.append("<reasoning>"), "Should accept opening tag");
-            assert!(!processor.eof().unwrap(), "Should not be at EOF after opening tag");
+            assert!(
+                !processor.eof().unwrap(),
+                "Should not be at EOF after opening tag"
+            );
         });
     }
 
@@ -179,18 +209,28 @@ mod tests {
             let processor = create_test_processor(py).unwrap();
 
             // At start, only '<' is valid
-            assert!(processor.can_continue_with("<").unwrap(), "Should accept '<' at start");
+            assert!(
-            assert!(!processor.can_continue_with("a").unwrap(), "Should reject 'a' at start");
+                processor.can_continue_with("<").unwrap(),
+                "Should accept '<' at start"
+            );
+            assert!(
+                !processor.can_continue_with("a").unwrap(),
+                "Should reject 'a' at start"
+            );
 
             // Test after opening tag
             let mut processor = create_test_processor(py).unwrap();
             assert!(processor.append("<reasoning>"), "Should accept opening tag");
 
             // In mixed content, any character should be valid
-            assert!(processor.can_continue_with("a").unwrap(), 
+            assert!(
-                   "Should accept 'a' in mixed content");
+                processor.can_continue_with("a").unwrap(),
-            assert!(processor.can_continue_with("<").unwrap(), 
+                "Should accept 'a' in mixed content"
-                   "Should accept '<' in mixed content");
+            );
+            assert!(
+                processor.can_continue_with("<").unwrap(),
+                "Should accept '<' in mixed content"
+            );
         });
     }
 
@@ -203,8 +243,11 @@ mod tests {
             let copy = processor.copy(py).unwrap();
 
             // Original and copy should behave the same
-            assert_eq!(processor.eof().unwrap(), copy.eof().unwrap(), 
+            assert_eq!(
-                      "Copy should have same EOF state");
+                processor.eof().unwrap(),
+                copy.eof().unwrap(),
+                "Copy should have same EOF state"
+            );
 
             // Modify copy, original should be unchanged
             let mut copy = processor.copy(py).unwrap();
@@ -220,8 +263,10 @@ mod tests {
             let processor = create_test_processor(py).unwrap();
 
             let names = processor.get_element_names().unwrap();
-            assert!(names.contains(&"reasoning".to_string()), 
+            assert!(
-                   "Should contain 'reasoning' element");
+                names.contains(&"reasoning".to_string()),
+                "Should contain 'reasoning' element"
+            );
             assert_eq!(names.len(), 1, "Should have exactly one element");
 
             let tokens = PyDict::new(py);
@@ -235,8 +280,10 @@ mod tests {
             println!("Elements in schema: {:?}", names);
 
             // Check if reasoning exists
-            assert!(names.contains(&"reasoning".to_string()), 
+            assert!(
-                    "Real schema should contain 'reasoning'");
+                names.contains(&"reasoning".to_string()),
+                "Real schema should contain 'reasoning'"
+            );
         });
     }
 
@@ -270,13 +317,16 @@ mod tests {
             tokens.set_item(10, ">").unwrap();
 
             // Create processor with this schema and tokens
-            let processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
+            let processor =
+                XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
 
             // Test basic operations to verify core functionality works
             let element_names = processor.get_element_names().unwrap();
             println!("Element names in test schema: {:?}", element_names);
-            assert!(element_names.contains(&"reasoning".to_string()), 
+            assert!(
-                   "Test schema should contain 'reasoning'");
+                element_names.contains(&"reasoning".to_string()),
+                "Test schema should contain 'reasoning'"
+            );
 
             // Now try to append each character of "<reasoning>" one by one
             let chars = vec!['<', 'r', 'e', 'a', 's', 'o', 'n', 'i', 'n', 'g', '>'];
@@ -287,11 +337,16 @@ mod tests {
                 full_text.push(c);
                 let fragment = c.to_string();
                 let result = processor.can_continue_with(&fragment).unwrap();
-                println!("Can continue with '{}' after '{}': {}", 
+                println!(
-                        c, &full_text[..full_text.len()-1], result);
+                    "Can continue with '{}' after '{}': {}",
+                    c,
+                    &full_text[..full_text.len() - 1],
+                    result
+                );
 
                 // Create a new processor to test appending the fragment built so far
-                let mut fresh_processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
+                let mut fresh_processor =
+                    XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap();
                 let append_result = fresh_processor.append(&full_text);
                 println!("Appending full text '{}': {}", full_text, append_result);
             }
@@ -300,7 +355,8 @@ mod tests {
             if let Ok(actual_schema) = std::fs::read_to_string("../../action_schema.xsd") {
                 println!("\nTesting with actual schema from disk:");
 
-                let processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap();
+                let processor =
+                    XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap();
                 let element_names = processor.get_element_names().unwrap();
                 println!("Element names in actual schema: {:?}", element_names);
 
@@ -316,7 +372,9 @@ mod tests {
                     let mut text = String::new();
                     for c in "<reasoning>".chars() {
                         text.push(c);
-                        let result = XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap()
+                        let result =
+                            XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema)
+                                .unwrap()
                                 .append(&text);
                         println!("Appending '{}' to fresh processor: {}", text, result);
                     }

lib/xml_schema_validator/src/py_xml_schema_validator.rs

@@ -1,5 +1,5 @@
-use pyo3::prelude::*;
 use pyo3::exceptions::PyValueError;
+use pyo3::prelude::*;
 
 /// Python wrapper for XmlValidator
 #[pyclass]

lib/xml_schema_validator/src/schema.rs

@@ -1,6 +1,14 @@
 use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    Eq,
+    Hash,
+    PartialEq,
+    Serialize,
+)]
 pub struct XsSchema {
     #[serde(rename = "@xmlns:xs")]
     pub xmlns_xs: String,
@@ -12,7 +20,15 @@ pub struct XsSchema {
     pub xs_element: Vec<XsElement>,
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    Eq,
+    Hash,
+    PartialEq,
+    Serialize,
+)]
 pub struct XsElement {
     #[serde(rename = "@name")]
     pub name: String,
@@ -22,7 +38,15 @@ pub struct XsElement {
     pub xs_complex_type: XsComplexType,
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    Eq,
+    Hash,
+    PartialEq,
+    Serialize,
+)]
 pub struct XsComplexType {
     #[serde(rename = "@mixed")]
     pub mixed: Option<String>,
@@ -34,7 +58,15 @@ pub struct XsComplexType {
     pub xs_sequence: Option<XsSequence>,
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    Eq,
+    Hash,
+    PartialEq,
+    Serialize,
+)]
 pub struct XsAttribute {
     #[serde(rename = "@name")]
     pub name: String,
@@ -44,7 +76,15 @@ pub struct XsAttribute {
     pub xs_attribute_use: String,
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    Eq,
+    Hash,
+    PartialEq,
+    Serialize,
+)]
 pub struct XsSequence {
     #[serde(rename = "$text")]
     pub text: Option<String>,
@@ -52,7 +92,15 @@ pub struct XsSequence {
     pub xs_any: XsAny,
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    Eq,
+    Hash,
+    PartialEq,
+    Serialize,
+)]
 pub struct XsAny {
     #[serde(rename = "@minOccurs")]
     pub min_occurs: String,
@@ -61,4 +109,3 @@ pub struct XsAny {
     #[serde(rename = "@processContents")]
     pub process_contents: String,
 }
-

lib/xml_schema_validator/src/token/attribute_equals.rs

@@ -0,0 +1,102 @@
+use crate::*;
+use std::sync::Arc;
+
+/// Represents the equals sign after an XML attribute name
+#[derive(Clone, Debug)]
+pub struct AttributeEquals {
+    element: Arc<schema::XsElement>,
+    used_attributes: Vec<schema::XsAttribute>,
+}
+
+impl AttributeEquals {
+    pub fn new(
+        element: Arc<schema::XsElement>,
+        used_attributes: Vec<schema::XsAttribute>,
+    ) -> Self {
+        Self {
+            element,
+            used_attributes,
+        }
+    }
+
+    pub fn append(self, c: char) -> Vec<Token> {
+        if c == '"' {
+            // Opening quote starts the attribute value
+            return vec![Token::AttributeValue(AttributeValue::new(
+                Arc::clone(&self.element),
+                self.used_attributes,
+            ))];
+        } else if c.is_whitespace() {
+            // Allow whitespace between equals sign and opening quote
+            return vec![Token::AttributeEquals(self)];
+        } else {
+            // Any other character is invalid
+            return vec![];
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_attribute_equals_quote() {
+        let values = [
+            "<delete id=\"",
+            "<single timeout=\"",
+            "<single limit=\"",
+        ];
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            assert!(
+                validator
+                    .current_tokens
+                    .iter()
+                    .filter(|token| matches!(token, Token::AttributeValue(_)))
+                    .next()
+                    .is_some(),
+                "{value}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_attribute_equals_with_whitespace() {
+        let values = [
+            "<delete id= \"",
+            "<single timeout = \"",
+            "<single limit  \t  =   \n \"",
+        ];
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            assert!(
+                validator
+                    .current_tokens
+                    .iter()
+                    .filter(|token| matches!(token, Token::AttributeValue(_)))
+                    .next()
+                    .is_some(),
+                "{value}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_invalid_after_equals() {
+        let values = [
+            "<delete id=a",  // Only quote should be accepted after equals
+            "<single timeout=<", // Invalid character after equals
+            "<single timeout=1", // Unquoted numbers not supported yet
+        ];
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect_err(value);
+            assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
+        }
+    }
+}

lib/xml_schema_validator/src/token/attribute_name.rs

@@ -1,196 +1,165 @@
-use std::sync::Arc;
 use crate::*;
+use std::sync::Arc;
 
-/// Represents an attribute name of an XML element
+/// Represents an XML attribute name that is in the process of being parsed
 #[derive(Clone, Debug)]
 pub struct AttributeName {
-    /// The element that owns this attribute
-    element: Arc<schema::XsElement>,
-    /// Buffer containing the attribute name characters processed so far
     buffer: String,
-    /// Track which attributes have already been set for this element
+    element: Arc<schema::XsElement>,
-    attributes_set: Vec<String>,
+    used_attributes: Vec<schema::XsAttribute>,
 }
 
 impl AttributeName {
-    /// Create a new AttributeName with an empty buffer
+    pub fn new(first_char: char, element: Arc<schema::XsElement>) -> Self {
-    pub fn new(element: Arc<schema::XsElement>) -> Self {
         Self {
+            buffer: first_char.to_string(),
             element,
-            buffer: String::new(),
+            used_attributes: Vec::new(),
-            attributes_set: Vec::new(),
         }
     }
 
-    /// Create a new AttributeName with the given buffer
+    pub fn new_with_used(first_char: char, element: Arc<schema::XsElement>, used_attributes: Vec<schema::XsAttribute>) -> Self {
-    pub fn with_buffer(element: Arc<schema::XsElement>, buffer: String, attributes_set: Vec<String>) -> Self {
         Self {
+            buffer: first_char.to_string(),
             element,
-            buffer,
+            used_attributes,
-            attributes_set,
         }
     }
 
-    /// Create a new AttributeName with attributes already set
+    pub fn append(mut self, c: char) -> Vec<Token> {
-    pub fn with_attributes(element: Arc<schema::XsElement>, attributes_set: Vec<String>) -> Self {
+        if c == '=' {
-        Self {
+            // Check if we've matched a valid attribute name
-            element,
+            if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
-            buffer: String::new(),
+                if let Some(attribute) = attributes
-            attributes_set,
+                    .iter()
-        }
+                    .find(|attribute| attribute.name == self.buffer)
+                {
+                    // Check if this attribute has already been used
+                    if self.used_attributes.contains(&attribute) {
+                        return vec![];
+                    }
+                    
+                    // Add this attribute to the used set
+                    self.used_attributes.push(attribute.clone());
+                    
+                    return vec![Token::AttributeEquals(AttributeEquals::new(
+                        Arc::clone(&self.element),
+                        self.used_attributes,
+                    ))];
+                }
+            }
+            return vec![];
+        } else if c.is_whitespace() {
+            // Check if we've matched a valid attribute name and allow whitespace after it
+            if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
+                if attributes
+                    .iter()
+                    .any(|attribute| attribute.name == self.buffer)
+                {
+                    // Check if this attribute has already been used
+                    if self.used_attributes.iter().filter(|attribute| attribute.name == self.buffer).next().is_some() {
+                        return vec![];
                     }
                     
-    /// Append a character to the attribute name and return possible continuations
-    pub fn append(self, c: char) -> Vec<Token> {
-        if c.is_whitespace() && self.buffer.is_empty() {
                     return vec![Token::AttributeName(self)];
-        } else if c == '=' {
-            // Found the equals sign, transition to attribute value
-            // We need to verify that this attribute name is valid for this element
-            if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
-                for attr in attributes {
-                    if attr.name == self.buffer {
-                        // Valid attribute, transition to attribute value
-                        return vec![Token::AttributeValue(AttributeValue::new(
-                            Arc::clone(&self.element),
-                            self.buffer,
-                            self.attributes_set,
-                        ))];
                 }
             }
-            }
+            return vec![];
-            // Invalid attribute name
-            vec![]
         } else {
-            // Continue building the attribute name
+            // Add character to buffer and check if we're still building a valid attribute name
-            let new_buffer = self.buffer + &c.to_string();
+            let new_buffer = self.buffer.clone() + &c.to_string();
             
-            // Check if this could be a valid attribute for this element
             if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
-                for attr in attributes {
+                // Check if any attribute name starts with our buffer
-                    if attr.name.starts_with(&new_buffer) {
+                if attributes
-                        // This could be a valid attribute, continue parsing
+                    .iter()
-                        return vec![Token::AttributeName(AttributeName::with_buffer(
+                    .any(|attribute| attribute.name.starts_with(&new_buffer))
-                            Arc::clone(&self.element),
+                {
-                            new_buffer,
+                    return vec![Token::AttributeName(AttributeName {
-                            self.attributes_set.clone(),
+                        buffer: new_buffer,
-                        ))];
+                        element: Arc::clone(&self.element),
-                    }
+                        used_attributes: self.used_attributes,
+                    })];
                 }
             }
             
-            // No matching attribute found
+            // Character doesn't match what we expect for this attribute name
-            vec![]
+            return vec![];
         }
     }
 }
 
 #[cfg(test)]
-mod attribute_name_tests {
+mod tests {
     use super::*;
-    use std::sync::Arc;
-    use crate::schema;
-
-    fn create_element_with_attributes() -> Arc<schema::XsElement> {
-        let element = schema::XsElement {
-            name: "delete".to_string(),
-            text: None,
-            xs_complex_type: schema::XsComplexType {
-                mixed: None,
-                text: None,
-                xs_attribute: Some(vec![
-                    schema::XsAttribute {
-                        name: "id".to_string(),
-                        xs_attribute_type: "xs:string".to_string(),
-                        xs_attribute_use: "required".to_string(),
-                    }
-                ]),
-                xs_sequence: None,
-            },
-        };
-        Arc::new(element)
-    }
 
     #[test]
-    fn test_attribute_name_append() {
+    fn test_valid_attribute_name() {
-        let element = create_element_with_attributes();
+        let values = [
-        let token = AttributeName::new(Arc::clone(&element));
+            "<delete id=",
-        
+            "<single timeout=",
-        // Build attribute name character by character
+            "<single limit=",
-        let mut current_token = token;
+        ];
-        for c in "id".chars() {
+        for value in values {
-            let next_tokens = current_token.append(c);
+            let mut validator = crate::tests::example_validator();
-            assert!(!next_tokens.is_empty(), "Should accept attribute name character '{}'", c);
+            validator.append(value).expect(value);
-            
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-            match &next_tokens[0] {
+            assert!(
-                Token::AttributeName(next) => current_token = next.clone(),
+                validator
-                _ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
+                    .current_tokens
-            }
+                    .iter()
-        }
+                    .filter(|token| matches!(token, Token::AttributeEquals(_)))
-        
+                    .next()
-        // Test equals sign
+                    .is_some(),
-        let next_tokens = current_token.append('=');
+                "{value}"
-        assert!(!next_tokens.is_empty(), "Should accept '=' after attribute name");
+            );
-        
-        match &next_tokens[0] {
-            Token::AttributeValue(_) => (),
-            _ => panic!("Expected AttributeValue, got {:?}", next_tokens[0]),
         }
     }
 
     #[test]
-    fn test_attribute_name_whitespace() {
+    fn test_attribute_name_with_whitespace() {
-        let element = create_element_with_attributes();
+        let values = [
-        let token = AttributeName::new(Arc::clone(&element));
+            "<delete id =",
-        
+            "<single timeout  =",
-        // Test with whitespace
+            "<single limit\t=",
-        let next_tokens = token.append(' ');
+        ];
-        assert!(!next_tokens.is_empty(), "Should accept whitespace in attribute name");
+        for value in values {
-        
+            let mut validator = crate::tests::example_validator();
-        match &next_tokens[0] {
+            validator.append(value).expect(value);
-            Token::AttributeName(_) => (),
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-            _ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
+            assert!(
+                validator
+                    .current_tokens
+                    .iter()
+                    .filter(|token| matches!(token, Token::AttributeEquals(_)))
+                    .next()
+                    .is_some(),
+                "{value}"
+            );
         }
     }
 
     #[test]
     fn test_invalid_attribute_name() {
-        let element = create_element_with_attributes();
+        let values = [
-        let token = AttributeName::new(Arc::clone(&element));
+            "<delete invalid=",
-        
+            "<single wrongattr=",
-        // Test with invalid attribute name
+        ];
-        let next_tokens = token.append('x'); // 'x' doesn't start any valid attribute
+        for value in values {
-        assert!(next_tokens.is_empty(), "Should reject invalid attribute name");
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect_err(value);
+            assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
+        }
     }
 
     #[test]
-    fn test_attribute_tracking() {
+    fn test_duplicate_attribute() {
-        let element = create_element_with_attributes();
+        let mut validator = crate::tests::example_validator();
+        // First attribute is valid
+        validator.append("<delete id=\"123\" ").expect("First attribute should be accepted");
         
-        // Create AttributeName with pre-existing attributes
+        // Try to add the same attribute again
-        let attributes_set = vec!["other".to_string()];
+        let result = validator.append("id=");
-        let token = AttributeName::with_attributes(Arc::clone(&element), attributes_set.clone());
+        assert!(result.is_err(), "Duplicate attribute should be rejected");
-        
-        // Build attribute name and transition to AttributeValue
-        let mut current_token = token;
-        for c in "id".chars() {
-            let next_tokens = current_token.append(c);
-            match &next_tokens[0] {
-                Token::AttributeName(next) => current_token = next.clone(),
-                _ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
-            }
-        }
-        
-        // Test equals sign
-        let next_tokens = current_token.append('=');
-        
-        match &next_tokens[0] {
-            Token::AttributeValue(value) => {
-                // Verify that pre-existing attributes are preserved
-                assert!(value.attributes_set.contains(&"other".to_string()), 
-                        "Pre-existing attribute should be preserved");
-            },
-            _ => panic!("Expected AttributeValue, got {:?}", next_tokens[0]),
-        }
     }
 }

lib/xml_schema_validator/src/token/attribute_value.rs

@@ -1,4 +1,3 @@
-use std::sync::Arc;
 use crate::*;
 
 /// Represents an attribute value of an XML element
@@ -6,248 +5,187 @@ use crate::*;
 pub struct AttributeValue {
     /// The element that owns this attribute
     element: Arc<schema::XsElement>,
-    /// The name of the attribute being processed
-    attribute_name: String,
     /// Buffer containing the attribute value characters processed so far
     buffer: String,
-    /// Whether we've encountered the opening quote
-    in_quotes: bool,
-    /// Whether we've seen the closing quote
-    closed: bool,
     /// Track which attributes have already been set for this element
-    pub attributes_set: Vec<String>,
+    pub attributes_set: Vec<schema::XsAttribute>,
 }
 
 impl AttributeValue {
     /// Create a new AttributeValue with a list of attributes already set
-    pub fn new(element: Arc<schema::XsElement>, attribute_name: String, attributes_set: Vec<String>) -> Self {
+    pub fn new(
+        element: Arc<schema::XsElement>,
+        attributes_set: Vec<schema::XsAttribute>,
+    ) -> Self {
         Self {
             element,
-            attribute_name,
+            buffer: "\"".to_string(),
-            buffer: String::new(),
-            in_quotes: false,
-            closed: false,
             attributes_set,
         }
     }
 
     /// Append a character to the attribute value and return possible continuations
-    pub fn append(self, c: char) -> Vec<Token> {
+    pub fn append(mut self, c: char) -> Vec<Token> {
-        if self.closed {
+        // Handle escaped characters
-            // We've already closed the attribute value with a quote
+        if Self::in_escape_sequence(&self.buffer) {
-            
+            self.buffer.push(c);
-            // Add this attribute to the list of attributes set
+            return vec![Token::AttributeValue(self)];
-            let mut new_attributes = self.attributes_set.clone();
+        } else if self.value_closed() {
-            if !new_attributes.contains(&self.attribute_name) {
+            return self.next_tokens(c);
-                new_attributes.push(self.attribute_name.clone());
+        } else if '"' == c {
+            self.buffer.push(c);
+            return vec![Token::AttributeValue(self)];
+        } else {
+            self.buffer.push(c);
+            return vec![Token::AttributeValue(self)];
+        }
     }
 
+    /// Checks if an XML character entity reference is in progress and correctly terminated
+    /// Returns true if the sequence is complete (ends with a semicolon)
+    pub fn in_escape_sequence(buffer: &str) -> bool {
+        if let Some(amp_pos) = buffer.rfind('&') {
+            if buffer[amp_pos..].find(';').is_some() {
+                return false;
+            } else {
+                return true;
+            }
+        }
+        false
+    }
+
+    fn value_closed(&self) -> bool {
+        if self.buffer.len() > 1 && self.buffer.trim().ends_with("\"") {
+            let trimmed = self.buffer.trim();
+            let trimmed = &trimmed[..trimmed.len() - 2];
+            if Self::in_escape_sequence(trimmed) {
+                false
+            } else {
+                true
+            }
+        } else {
+            false
+        }
+    }
+
+    fn next_tokens(mut self, c: char) -> Vec<Token> {
+        let all_required_attributes_set: bool = self
+            .element
+            .as_ref()
+            .xs_complex_type
+            .xs_attribute
+            .as_deref()
+            .unwrap()
+            .iter()
+            .all(|attr| {
+                if attr.xs_attribute_use == "required" {
+                    self.attributes_set.contains(attr)
+                } else {
+                    true
+                }
+            });
+        let all_attributes_set: bool = self
+            .element
+            .as_ref()
+            .xs_complex_type
+            .xs_attribute
+            .as_deref()
+            .unwrap()
+            .iter()
+            .all(|attr| self.attributes_set.contains(attr));
         if c.is_whitespace() {
-                // Check if there are more attributes to parse
+            self.buffer.push(c);
-                if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
+            vec![Token::AttributeValue(self)]
-                    // If there are other required attributes that haven't been set yet
-                    for attr in attributes {
-                        if attr.name != self.attribute_name && attr.xs_attribute_use == "required" && !new_attributes.contains(&attr.name) {
-                            return vec![Token::AttributeName(AttributeName::with_attributes(
-                                Arc::clone(&self.element),
-                                new_attributes,
-                            ))];
-                        }
-                    }
-                }
-                
-                // No more required attributes, whitespace after attribute value, continue to next state
-                return vec![
-                    Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes.clone())),
-                    Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))
-                ];
-            } else if c == '>' {
-                // End of opening tag
-                return vec![Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes))];
-            } else if c == '/' {
-                // Beginning of self-closing tag
-                return vec![Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))];
         } else {
-                // Unexpected character after attribute value
+            let mut tokens = vec![];
-                return vec![];
+            if all_required_attributes_set {
+                if '/' == c {
+                    tokens.push(Token::ElementSelfClose(ElementSelfClose::new()));
+                } else if '>' == c {
+                    tokens.push(Token::ElementOpenEnd(ElementOpenEnd::new(self.element.clone())));
                 }
             }
-        
+            if self.buffer.ends_with(char::is_whitespace) && !all_attributes_set {
-        if !self.in_quotes {
+                tokens.push(Token::AttributeName(AttributeName::new_with_used(
-            if c.is_whitespace() {
+                    c,
-                // Skip whitespace before the quotes
+                    self.element,
-                return vec![Token::AttributeValue(Self {
+                    self.attributes_set,
-                    element: Arc::clone(&self.element),
+                )));
-                    attribute_name: self.attribute_name,
-                    buffer: self.buffer,
-                    in_quotes: false,
-                    closed: false,
-                    attributes_set: self.attributes_set,
-                })];
-            } else if c == '"' || c == '\'' {
-                // Start of quotes
-                return vec![Token::AttributeValue(Self {
-                    element: Arc::clone(&self.element),
-                    attribute_name: self.attribute_name,
-                    buffer: self.buffer,
-                    in_quotes: true,
-                    closed: false,
-                    attributes_set: self.attributes_set,
-                })];
-            } else {
-                // Unexpected character before quotes
-                return vec![];
-            }
-        } else {
-            // We're inside quotes
-            if c == '"' || c == '\'' {
-                // End of quotes, validate the attribute value
-                // In a real implementation, we'd check if the value is valid for this attribute type
-                return vec![Token::AttributeValue(Self {
-                    element: Arc::clone(&self.element),
-                    attribute_name: self.attribute_name,
-                    buffer: self.buffer,
-                    in_quotes: true,
-                    closed: true,
-                    attributes_set: self.attributes_set,
-                })];
-            } else {
-                // Continue building the attribute value
-                let new_buffer = self.buffer + &c.to_string();
-                return vec![Token::AttributeValue(Self {
-                    element: Arc::clone(&self.element),
-                    attribute_name: self.attribute_name,
-                    buffer: new_buffer,
-                    in_quotes: true,
-                    closed: false,
-                    attributes_set: self.attributes_set,
-                })];
             }
+            tokens
         }
     }
 }
 
 #[cfg(test)]
-mod attribute_value_tests {
+mod tests {
     use super::*;
-    use std::sync::Arc;
-    use crate::schema;
-
-    fn create_element_with_attributes() -> Arc<schema::XsElement> {
-        let element = schema::XsElement {
-            name: "delete".to_string(),
-            text: None,
-            xs_complex_type: schema::XsComplexType {
-                mixed: None,
-                text: None,
-                xs_attribute: Some(vec![
-                    schema::XsAttribute {
-                        name: "id".to_string(),
-                        xs_attribute_type: "xs:string".to_string(),
-                        xs_attribute_use: "required".to_string(),
-                    }
-                ]),
-                xs_sequence: None,
-            },
-        };
-        Arc::new(element)
-    }
 
     #[test]
-    fn test_attribute_value_quotes() {
+    fn test_unfinished_attribute_value() {
-        let element = create_element_with_attributes();
+        let values = [
-        let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
+            "<delete id=\"12",
+            "<single timeout=\"1.",
+            "<single limit=\"50",
+        ];
         
-        // Test with opening quote
+        for value in values {
-        let next_tokens = token.append('"');
+            let mut validator = crate::tests::example_validator();
-        assert!(!next_tokens.is_empty(), "Should accept opening quote");
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
             
-        let token = next_tokens[0].clone();
+            // Should still be in AttributeValue state
-        match token.clone() {
+            assert!(
-            Token::AttributeValue(value) => {
+                validator
-                assert!(value.in_quotes, "Should mark as in quotes");
+                    .current_tokens
-                assert!(!value.closed, "Should not be closed yet");
+                    .iter()
-            },
+                    .any(|token| matches!(token, Token::AttributeValue(_))),
-            _ => panic!("Expected AttributeValue, got {:?}", token),
+                "{value}"
-        }
+            );
-        
-        // Test with content inside quotes
-        let next_tokens = token.append('a');
-        assert!(!next_tokens.is_empty(), "Should accept content inside quotes");
-        
-        let token = next_tokens[0].clone();
-        
-        // Test with closing quote
-        let next_tokens = token.append('"');
-        assert!(!next_tokens.is_empty(), "Should accept closing quote");
-        
-        let token = next_tokens[0].clone();
-        match token {
-            Token::AttributeValue(value) => {
-                assert!(value.closed, "Should be closed");
-            },
-            _ => panic!("Expected AttributeValue, got {:?}", token),
         }
     }
 
     #[test]
-    fn test_after_attribute_value_closed() {
+    fn test_type_validation() {
-        let element = create_element_with_attributes();
+        // Test integer validation
-        let mut token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
+        let mut validator = crate::tests::example_validator();
-        token.in_quotes = true;
+        validator.append("<single limit=\"123\"").expect("Valid integer");
-        token.closed = true;
         
-        // Test with space after closed attribute value
+        // Test float validation
-        let next_tokens = token.append(' ');
+        validator = crate::tests::example_validator();
-        assert!(!next_tokens.is_empty(), "Should accept space after closed attribute");
+        validator.append("<single timeout=\"1.5\"").expect("Valid float");
         
-        // Test with > after closed attribute value
+        // Test string validation
-        let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
+        validator = crate::tests::example_validator();
-        let mut token = token;
+        validator.append("<delete id=\"abc123\"").expect("Valid string");
-        token.in_quotes = true;
-        token.closed = true;
-        
-        let next_tokens = token.append('>');
-        assert!(!next_tokens.is_empty(), "Should accept '>' after closed attribute");
-        
-        match &next_tokens[0] {
-            Token::ElementOpenEnd(_) => (),
-            _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
     }
 
-        // Test with / after closed attribute value (for self-closing)
+    #[test]
-        let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
+    fn test_multiple_attributes() {
-        let mut token = token;
+        let values = [
-        token.in_quotes = true;
+            "<single limit=\"100\" timeout=\"1.5\"",
-        token.closed = true;
+            "<single timeout=\"0.5\" limit=\"200\"",
+        ];
         
-        let next_tokens = token.append('/');
+        for value in values {
-        assert!(!next_tokens.is_empty(), "Should accept '/' after closed attribute");
+            let mut validator = crate::tests::example_validator();
-        
+            validator.append(value).expect(value);
-        match &next_tokens[0] {
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-            Token::ElementSelfClose(_) => (),
-            _ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]),
         }
     }
 
     #[test]
-    fn test_attribute_tracking() {
+    fn test_attribute_value_escape_chars() {
-        let element = create_element_with_attributes();
+        // Test handling of escaped characters in attribute values
-        let mut token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]);
+        let values = [
-        token.in_quotes = true;
+            "<delete id=\"escaped&quot;quote\"",
-        token.closed = true;
+            "<delete id=\"escaped&amp;amp\"",
+        ];
         
-        // After attribute is closed, it should be tracked
+        for value in values {
-        let next_tokens = token.append('>');
+            let mut validator = crate::tests::example_validator();
-        assert!(!next_tokens.is_empty(), "Should accept '>' after attribute");
+            validator.append(value).expect(value);
-        
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-        match &next_tokens[0] {
-            Token::ElementOpenEnd(element_open) => {
-                // Verify that the id attribute is tracked
-                assert!(element_open.attributes_set.contains(&"id".to_string()), 
-                        "Attribute 'id' should be tracked");
-            },
-            _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
         }
     }
 }

lib/xml_schema_validator/src/token/element_close_name.rs

@@ -1,5 +1,5 @@
-use std::sync::Arc;
 use crate::*;
+use std::sync::Arc;
 
 /// Represents a closing element tag that is in the process of being parsed
 /// after the '</' and now parsing the element name
@@ -14,10 +14,7 @@ pub struct ElementCloseName {
 impl ElementCloseName {
     /// Create a new instance with the given buffer
     pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Self {
-        Self {
+        Self { element, buffer }
-            element,
-            buffer,
-        }
     }
 
     pub fn append(self, c: char) -> Vec<Token> {
@@ -40,10 +37,10 @@ impl ElementCloseName {
 }
 
 #[cfg(test)]
-mod element_close_name_tests {
+mod tests {
     use super::*;
-    use std::sync::Arc;
     use crate::schema;
+    use std::sync::Arc;
 
     fn create_test_element() -> Arc<schema::XsElement> {
         let element = schema::XsElement {
@@ -73,12 +70,15 @@ mod element_close_name_tests {
 
         // Test with next character in name
         let next_tokens = token.append('e');
-        assert!(!next_tokens.is_empty(), "Should accept next character in name");
+        assert!(
+            !next_tokens.is_empty(),
+            "Should accept next character in name"
+        );
 
         match &next_tokens[0] {
             Token::ElementCloseName(name) => {
                 assert_eq!(name.buffer, "re", "Buffer should contain 're'");
-            },
+            }
             _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
         }
     }
@@ -90,19 +90,28 @@ mod element_close_name_tests {
 
         // Test with last character of name
         let next_tokens = token.append('g');
-        assert!(!next_tokens.is_empty(), "Should accept last character of name");
+        assert!(
+            !next_tokens.is_empty(),
+            "Should accept last character of name"
+        );
 
         match &next_tokens[0] {
             Token::ElementCloseName(name) => {
-                assert_eq!(name.buffer, "reasoning", "Buffer should contain 'reasoning'");
+                assert_eq!(
-            },
+                    name.buffer, "reasoning",
+                    "Buffer should contain 'reasoning'"
+                );
+            }
             _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
         }
 
         // Test with > after complete name
         let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
         let next_tokens = token.append('>');
-        assert!(!next_tokens.is_empty(), "Should accept '>' after complete name");
+        assert!(
+            !next_tokens.is_empty(),
+            "Should accept '>' after complete name"
+        );
 
         match &next_tokens[0] {
             Token::EndOfFile(_) => (),
@@ -117,7 +126,10 @@ mod element_close_name_tests {
 
         // Test with whitespace after complete name
         let next_tokens = token.append(' ');
-        assert!(!next_tokens.is_empty(), "Should accept whitespace after complete name");
+        assert!(
+            !next_tokens.is_empty(),
+            "Should accept whitespace after complete name"
+        );
 
         match &next_tokens[0] {
             Token::ElementCloseName(_) => (),

lib/xml_schema_validator/src/token/element_close_start.rs

@@ -1,5 +1,5 @@
-use std::sync::Arc;
 use crate::*;
+use std::sync::Arc;
 
 /// Represents the start of an XML element closing tag (the '</' sequence)
 #[derive(Clone, Debug)]
@@ -35,10 +35,10 @@ impl ElementCloseStart {
 }
 
 #[cfg(test)]
-mod element_close_start_tests {
+mod tests {
     use super::*;
-    use std::sync::Arc;
     use crate::schema;
+    use std::sync::Arc;
 
     fn create_test_element() -> Arc<schema::XsElement> {
         let element = schema::XsElement {
@@ -73,7 +73,7 @@ mod element_close_start_tests {
         match &next_tokens[0] {
             Token::ElementCloseStart(start) => {
                 assert!(start.has_slash, "Should mark as having slash");
-            },
+            }
             _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
         }
     }
@@ -86,7 +86,10 @@ mod element_close_start_tests {
 
         // Test with first character of element name
         let next_tokens = token.append('r');
-        assert!(!next_tokens.is_empty(), "Should accept first letter of element name");
+        assert!(
+            !next_tokens.is_empty(),
+            "Should accept first letter of element name"
+        );
 
         match &next_tokens[0] {
             Token::ElementCloseName(_) => (),

lib/xml_schema_validator/src/token/element_open_end.rs

@@ -1,68 +1,34 @@
-use std::sync::Arc;
 use crate::*;
+use std::sync::Arc;
 
 /// Represents a fully opened XML element, containing the schema element reference
 #[derive(Clone, Debug)]
 pub struct ElementOpenEnd {
     element: Arc<schema::XsElement>,
-    // Track which attributes have been set for this element
-    pub attributes_set: Vec<String>,
 }
 
 impl ElementOpenEnd {
     pub fn new(element: Arc<schema::XsElement>) -> Self {
         Self {
             element,
-            attributes_set: Vec::new(),
-        }
-    }
-
-    pub fn with_attributes(element: Arc<schema::XsElement>, attributes_set: Vec<String>) -> Self {
-        Self {
-            element,
-            attributes_set,
         }
     }
 
     pub fn append(self, c: char) -> Vec<Token> {
-        // Check if all required attributes are present before proceeding
+        if '<' == c {
-        if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
+            vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
-            for attr in attributes {
+        } else {
-                if attr.xs_attribute_use == "required" && !self.attributes_set.contains(&attr.name) {
+            // Check if this element allows mixed content
-                    // Missing a required attribute, this is an error
+            let is_mixed = self.element.xs_complex_type.mixed
-                    return vec![];
-                }
-            }
-        }
-
-        // Check if the element is a mixed content element that allows text
-        let allows_text = self.element.xs_complex_type.mixed
                 .as_ref()
-            .map(|mixed| mixed == "true")
+                .map(|val| val == "true")
                 .unwrap_or(false);
                 
-        if allows_text {
+            if is_mixed || c.is_whitespace() {
-            if c.is_whitespace() {
+                // Allow text content for mixed elements, or whitespace for any element
-                // Handle whitespace in mixed content
+                vec![Token::TextContent(TextContent::new(self.element))]
-                vec![
-                    Token::TextContent(TextContent::new(Arc::clone(&self.element))),
-                    Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))
-                ]
-            } else if c == '<' {
-                // Start of a closing tag or nested element
-                vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
             } else {
-                // Text content
+                // Reject text content for non-mixed elements
-                vec![Token::TextContent(TextContent::new(Arc::clone(&self.element)))]
-            }
-        } else {
-            // Element doesn't allow text content, only expect closing tag
-            if c == '<' {
-                vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
-            } else if c.is_whitespace() {
-                vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
-            } else {
-                // Unexpected content in non-mixed element
                 vec![]
             }
         }
@@ -70,10 +36,10 @@ impl ElementOpenEnd {
 }
 
 #[cfg(test)]
-mod element_open_end_tests {
+mod tests {
     use super::*;
-    use std::sync::Arc;
     use crate::schema;
+    use std::sync::Arc;
 
     fn create_test_element() -> Arc<schema::XsElement> {
         let element = schema::XsElement {
@@ -117,26 +83,6 @@ mod element_open_end_tests {
         Arc::new(element)
     }
 
-    fn create_element_with_required_attributes() -> Arc<schema::XsElement> {
-        let element = schema::XsElement {
-            name: "delete".to_string(),
-            text: None,
-            xs_complex_type: schema::XsComplexType {
-                mixed: None,
-                text: None,
-                xs_attribute: Some(vec![
-                    schema::XsAttribute {
-                        name: "id".to_string(),
-                        xs_attribute_type: "xs:string".to_string(),
-                        xs_attribute_use: "required".to_string(),
-                    }
-                ]),
-                xs_sequence: None,
-            },
-        };
-        Arc::new(element)
-    }
-
     #[test]
     fn test_append_text_in_mixed_content() {
         let element = create_test_element();
@@ -144,7 +90,10 @@ mod element_open_end_tests {
 
         // Test appending text in mixed content element
         let next_tokens = token.append('a');
-        assert!(!next_tokens.is_empty(), "Should allow text content in mixed element");
+        assert!(
+            !next_tokens.is_empty(),
+            "Should allow text content in mixed element"
+        );
 
         match &next_tokens[0] {
             Token::TextContent(_) => (),
@@ -174,7 +123,10 @@ mod element_open_end_tests {
 
         // Test whitespace in mixed content
         let next_tokens = token.append(' ');
-        assert!(!next_tokens.is_empty(), "Should allow whitespace in mixed content");
+        assert!(
+            !next_tokens.is_empty(),
+            "Should allow whitespace in mixed content"
+        );
 
         match &next_tokens[0] {
             Token::TextContent(_) => (),
@@ -189,32 +141,23 @@ mod element_open_end_tests {
 
         // Test appending text in non-mixed content (should reject)
         let next_tokens = token.clone().append('a');
-        assert!(next_tokens.is_empty(), "Should reject text in non-mixed element");
+        assert!(
+            next_tokens.is_empty(),
+            "Should reject text in non-mixed element"
+        );
 
         // Test whitespace (should accept leading to closing tag)
         let next_tokens = token.clone().append(' ');
-        assert!(!next_tokens.is_empty(), "Should allow whitespace in non-mixed");
+        assert!(
+            !next_tokens.is_empty(),
+            "Should allow whitespace in non-mixed"
+        );
 
         // Test closing tag start
         let next_tokens = token.append('<');
-        assert!(!next_tokens.is_empty(), "Should allow closing tag start in non-mixed");
+        assert!(
-    }
+            !next_tokens.is_empty(),
-    
+            "Should allow closing tag start in non-mixed"
-    #[test]
-    fn test_required_attributes() {
-        let element = create_element_with_required_attributes();
-        
-        // No attributes set yet
-        let token = ElementOpenEnd::new(Arc::clone(&element));
-        let next_tokens = token.append('<');
-        assert!(next_tokens.is_empty(), "Should reject closing tag without required attributes");
-        
-        // With required attribute set
-        let token = ElementOpenEnd::with_attributes(
-            Arc::clone(&element), 
-            vec!["id".to_string()]
         );
-        let next_tokens = token.append('<');
-        assert!(!next_tokens.is_empty(), "Should allow closing tag with required attributes set");
     }
 }

lib/xml_schema_validator/src/token/element_open_name.rs

@@ -1,5 +1,5 @@
-use std::sync::Arc;
 use crate::*;
+use std::sync::Arc;
 
 /// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
 #[derive(Clone, Debug)]
@@ -9,72 +9,88 @@ pub struct ElementOpenName {
 }
 
 impl ElementOpenName {
-    pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Result<Self> {
+    pub fn new(element: Arc<schema::XsElement>, buffer: String) -> Self {
-        if element.name.starts_with(buffer.as_str()) {
+        Self { buffer, element }
-            Ok(Self {
-                buffer,
-                element,
-            })
-        } else {
-            Err(Error::InvalidXml(format!("Element name '{}' doesn't match '{}'", buffer, element.name)))
-        }
     }
 
-    pub fn append(self, c: char) -> Vec<Token> {
+    pub fn append(mut self, c: char) -> Vec<Token> {
-        if c.is_whitespace() {
+        if c == '>' {
             // Check if we've matched the full element name
-            if self.element.name == self.buffer {
+            if !self.buffer.contains(&self.element.name) {
-                // Element name is complete, handle whitespace after name
+                return vec![];
-                
+            }
             // Check if the element has attributes
             if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
-                    if !attributes.is_empty() {
+                // Check if they are required
-                        // Transition to attribute name parsing
+                if attributes
-                        return vec![Token::AttributeName(AttributeName::new(Arc::clone(&self.element)))];
+                    .iter()
-                    }
+                    .filter(|attribute| attribute.xs_attribute_use == "required")
-                }
+                    .next()
-                
+                    .is_some()
-                // No attributes
+                {
-                return vec![
+                    return vec![];
-                    Token::AttributeName(AttributeName::new(Arc::clone(&self.element))),
-                    Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element))),
-                    Token::ElementSelfClose(ElementSelfClose::new()),
-                    Token::ElementOpenName(self),
-                ];
                 } else {
-                // Whitespace in the middle of an element name is invalid
+                    return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(
-                return vec![];
+                        &self.element,
+                    )))];
                 }
-        } else if c == '>' {
-            // Check if we've matched the full element name
-            if self.element.name == self.buffer {
-                // Element name is complete, end of opening tag
-                
-                // Check if this element has required attributes
-                if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
-                    for attr in attributes {
-                        if attr.xs_attribute_use == "required" {
-                            // Found a required attribute but no attributes are set
-                            // Return an empty vector to indicate error
-                            return vec![];
-                        }
-                    }
-                }
-                
-                return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element)))];
             } else {
-                // '>' before full element name is invalid
+                return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(
-                return vec![];
+                    &self.element,
+                )))];
             }
         } else if c == '/' {
             // Check if we've matched the full element name
-            if self.element.name == self.buffer {
+            if !self.buffer.contains(&self.element.name) {
-                // Element name is complete, self-closing tag
-                return vec![Token::ElementSelfClose(ElementSelfClose::new())];
-            } else {
-                // '/' before full element name is invalid
                 return vec![];
             }
+            // Check if the element has attributes
+            if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
+                // Check if they are required
+                if attributes
+                    .iter()
+                    .filter(|attribute| attribute.xs_attribute_use == "required")
+                    .next()
+                    .is_some()
+                {
+                    return vec![];
+                } else {
+                    return vec![Token::ElementSelfClose(ElementSelfClose::new())];
+                }
+            } else {
+                return vec![Token::ElementSelfClose(ElementSelfClose::new())];
+            }
+        } else if c.is_whitespace() {
+            // Check if we've matched the full element name
+            if self.buffer.contains(&self.element.name) {
+                if let Some(last) = self.buffer.chars().last() {
+                    if last.is_whitespace() {
+                        return vec![Token::ElementOpenName(self)];
+                    }
+                }
+                self.buffer.push(c);
+                return vec![Token::ElementOpenName(self)];
+            } else {
+                return vec![];
+            }
+        } else if !self.buffer.is_empty() && self.buffer.chars().last().unwrap().is_whitespace() {
+            // Check for start of attribute name
+            if self.buffer.contains(&self.element.name) {
+                if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
+                    if attributes
+                        .iter()
+                        .filter(|attribute| attribute.name.starts_with(c))
+                        .next()
+                        .is_some()
+                    {
+                        return vec![Token::AttributeName(AttributeName::new(
+                            c,
+                            self.element.clone(),
+                        ))];
+                    }
+                }
+            }
+            return vec![];
         } else {
             // Add character to buffer and check if we're still building a valid element name
             let new_buffer = self.buffer.clone() + &c.to_string();
@@ -94,179 +110,77 @@ impl ElementOpenName {
 }
 
 #[cfg(test)]
-mod element_open_name_tests {
+mod tests {
     use super::*;
-    use std::sync::Arc;
-    use crate::schema;
-
-    fn create_test_element() -> Arc<schema::XsElement> {
-        let element = schema::XsElement {
-            name: "reasoning".to_string(),
-            text: None,
-            xs_complex_type: schema::XsComplexType {
-                mixed: Some("true".to_string()),
-                text: None,
-                xs_attribute: None,
-                xs_sequence: Some(schema::XsSequence {
-                    text: None,
-                    xs_any: schema::XsAny {
-                        min_occurs: "0".to_string(),
-                        max_occurs: "unbounded".to_string(),
-                        process_contents: "skip".to_string(),
-                    },
-                }),
-            },
-        };
-        Arc::new(element)
-    }
-
-    fn create_element_with_attributes() -> Arc<schema::XsElement> {
-        let element = schema::XsElement {
-            name: "delete".to_string(),
-            text: None,
-            xs_complex_type: schema::XsComplexType {
-                mixed: None,
-                text: None,
-                xs_attribute: Some(vec![
-                    schema::XsAttribute {
-                        name: "id".to_string(),
-                        xs_attribute_type: "xs:string".to_string(),
-                        xs_attribute_use: "required".to_string(),
-                    }
-                ]),
-                xs_sequence: None,
-            },
-        };
-        Arc::new(element)
-    }
 
     #[test]
-    fn test_append_character_by_character() {
+    fn test_attribute_name() {
-        let element = create_test_element();
+        let values = [
-        
+            "<delete i", // id
-        // Test each character in "reasoning"
+            "<single t", // timeout
-        let mut buffer = String::new();
+            "<single l", // limit
-        for c in "reasoning".chars() {
+        ];
-            buffer.push(c);
+        for value in values {
-            let result = ElementOpenName::new(Arc::clone(&element), buffer.clone());
+            let mut validator = crate::tests::example_validator();
-            assert!(result.is_ok(), "Failed to create ElementOpenName with buffer '{}'", buffer);
+            validator.append(value).expect(value);
-        }
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-    }
+            assert!(
-
+                validator
-    #[test]
+                    .current_tokens
-    fn test_append_full_name() {
+                    .iter()
-        let element = create_test_element();
-        
-        // Test with the full element name
-        let result = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string());
-        assert!(result.is_ok(), "Failed to create ElementOpenName with full element name");
-        
-        let element_name = result.unwrap();
-        
-        // Test transition after full name with '>'
-        let next_tokens = element_name.append('>');
-        assert!(!next_tokens.is_empty(), "Should accept '>' after full element name");
-        
-        match &next_tokens[0] {
-            Token::ElementOpenEnd(_) => (),
-            _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
-        }
-    }
-
-    #[test]
-    fn test_append_space_after_name() {
-        let element = create_test_element();
-        
-        // Test with the full element name
-        let element_name = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()).unwrap();
-        
-        // Test transition after full name with space
-        let next_tokens = element_name.append(' ');
-        assert!(!next_tokens.is_empty(), "Should accept space after full element name");
-        
-        assert!(next_tokens.len() == 4);
-        assert!(next_tokens.iter()
-            .filter(|token| matches!(token, Token::ElementOpenName(_)))
-            .next()
-            .is_some());
-        assert!(next_tokens.iter()
                     .filter(|token| matches!(token, Token::AttributeName(_)))
                     .next()
-            .is_some());
+                    .is_some(),
-        assert!(next_tokens.iter()
+                "{value}"
-            .filter(|token| matches!(token, Token::ElementOpenEnd(_)))
+            );
-            .next()
+        }
-            .is_some());
+    }
-        assert!(next_tokens.iter()
+
+    #[test]
+    fn test_valid_self_closing() {
+        let values = ["<stop/", "<stop /"];
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            assert!(
+                validator
+                    .current_tokens
+                    .iter()
                     .filter(|token| matches!(token, Token::ElementSelfClose(_)))
                     .next()
-            .is_some());
+                    .is_some(),
-    }
+                "{value}"
-
+            );
-    #[test]
-    fn test_append_with_attributes() {
-        let element = create_element_with_attributes();
-        
-        // Test with the full element name
-        let element_name = ElementOpenName::new(Arc::clone(&element), "delete".to_string()).unwrap();
-        
-        // Test transition after full name with space (expecting attribute name)
-        let next_tokens = element_name.append(' ');
-        assert!(!next_tokens.is_empty(), "Should accept space after element name with attributes");
-        
-        // The space should lead to a whitespace token, which should then lead to attribute name
-        match &next_tokens[0] {
-            Token::AttributeName(_) => (),
-            _ => panic!("Expected AttributeName, got {:?}", next_tokens[0]),
         }
     }
 
     #[test]
-    fn test_append_slash_after_name() {
+    fn test_valid_closing() {
-        let element = create_test_element();
+        let values = ["<stop>", "<stop >", "<reasoning>", "<reasoning >"];
-        
+        for value in values {
-        // Test with the full element name
+            let mut validator = crate::tests::example_validator();
-        let element_name = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()).unwrap();
+            validator.append(value).expect(value);
-        
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-        // Test transition after full name with /
+            assert!(
-        let next_tokens = element_name.append('/');
+                validator
-        assert!(!next_tokens.is_empty(), "Should accept '/' after full element name");
+                    .current_tokens
-        
+                    .iter()
-        match &next_tokens[0] {
+                    .filter(|token| matches!(token, Token::ElementOpenEnd(_)))
-            Token::ElementSelfClose(_) => (),
+                    .next()
-            _ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]),
+                    .is_some(),
+                "{value}"
+            );
         }
     }
 
     #[test]
-    fn test_invalid_continuation() {
+    fn test_invalid_closing() {
-        let element = create_test_element();
+        let values = ["<delete/", "<delete /", "<delete>", "<delete >"];
-        
+        for value in values {
-        // Test with a partial element name and an invalid continuation character
+            let mut validator = crate::tests::example_validator();
-        let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap();
+            validator.append(value).expect_err(value);
-        let next_tokens = element_name.append('x');
+            assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
-        assert!(next_tokens.is_empty(), "Should reject invalid character in element name");
-        
-        // Test whitespace in middle of name - need to create a new instance since append consumes self
-        let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap();
-        let next_tokens = element_name.append(' ');
-        assert!(next_tokens.is_empty(), "Should reject whitespace in middle of element name");
-        
-        // Test > in middle of name - need to create a new instance since append consumes self
-        let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap();
-        let next_tokens = element_name.append('>');
-        assert!(next_tokens.is_empty(), "Should reject '>' in middle of element name");
         }
-    
-    #[test]
-    fn test_required_attributes_validation() {
-        let element = create_element_with_attributes();
-        
-        // Test with the full element name
-        let element_name = ElementOpenName::new(Arc::clone(&element), "delete".to_string()).unwrap();
-        
-        // Try to close the tag with > directly, skipping the required attribute
-        let next_tokens = element_name.append('>');
-        assert!(next_tokens.is_empty(), "Should reject closing tag without required attributes");
     }
 }

lib/xml_schema_validator/src/token/element_open_start.rs

@@ -1,5 +1,5 @@
-use std::sync::Arc;
 use crate::*;
+use std::sync::Arc;
 
 /// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
 #[derive(Clone, Debug)]
@@ -9,17 +9,56 @@ pub struct ElementOpenStart {
 
 impl ElementOpenStart {
     pub fn new(element: Arc<schema::XsElement>) -> Self {
-        Self {
+        Self { element }
-            element,
-        }
     }
 
     pub fn append(self, c: char) -> Vec<Token> {
-        let element_name = ElementOpenName::new(Arc::clone(&self.element), c.to_string());
+        if self.element.name.starts_with(c) {
-        if let Ok(element_name) = element_name {
+            vec![Token::ElementOpenName(ElementOpenName::new(
-            vec![Token::ElementOpenName(element_name)]
+                Arc::clone(&self.element),
+                c.to_string(),
+            ))]
         } else {
             vec![]
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_accept_valid_element_names() {
+        let values = [
+            "<d", // delete
+            "<s", // stop / single
+            "<r", // repeat / reasoning / read_stdin
+            "<w", // write_stdout
+        ];
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            assert!(
+                validator
+                    .current_tokens
+                    .iter()
+                    .filter(|token| matches!(token, Token::ElementOpenName(_)))
+                    .next()
+                    .is_some(),
+                "{value}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_not_accept_other() {
+        let values = ["< ", "<a", "</"];
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect_err(value);
+            assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
+        }
+    }
+}

lib/xml_schema_validator/src/token/element_self_close.rs

@@ -1,54 +1,18 @@
-use std::sync::Arc;
 use crate::*;
 
 /// Represents a self-closing XML element
 #[derive(Clone, Debug)]
-pub struct ElementSelfClose {
+pub struct ElementSelfClose {}
-    /// Optional reference to the element schema for attribute validation
-    element: Option<Arc<schema::XsElement>>,
-    /// Track which attributes have been set
-    attributes_set: Vec<String>,
-}
 
 impl ElementSelfClose {
     /// Create a new self-closing element
     pub fn new() -> Self {
-        Self {
+        Self {}
-            element: None,
-            attributes_set: Vec::new(),
-        }
     }
 
-    /// Create a new self-closing element with element reference and attribute tracking
-    pub fn new_with_attributes(element: Arc<schema::XsElement>, attributes_set: Vec<String>) -> Self {
-        Self {
-            element: Some(element),
-            attributes_set,
-        }
-    }
-
-    /// Append a character to the self-closing tag and return possible continuations
     pub fn append(self, c: char) -> Vec<Token> {
-        // Before we close, verify that all required attributes are present
-        if let Some(element) = &self.element {
         if c == '>' {
-                if let Some(attributes) = &element.xs_complex_type.xs_attribute {
-                    for attr in attributes {
-                        if attr.xs_attribute_use == "required" && !self.attributes_set.contains(&attr.name) {
-                            // Missing required attribute, this is an error
-                            return vec![];
-                        }
-                    }
-                }
             return vec![Token::EndOfFile(EndOfFile::new())];
-            }
-        } else if c == '>' {
-            // No element reference, can't validate attributes, just close
-            return vec![Token::EndOfFile(EndOfFile::new())];
-        }
-        
-        if c.is_whitespace() {
-            vec![Token::ElementSelfClose(self)]
         } else {
             vec![]
         }
@@ -56,89 +20,35 @@ impl ElementSelfClose {
 }
 
 #[cfg(test)]
-mod element_self_close_tests {
+mod tests {
     use super::*;
-    use std::sync::Arc;
-    use crate::schema;
-
-    fn create_element_with_required_attributes() -> Arc<schema::XsElement> {
-        let element = schema::XsElement {
-            name: "delete".to_string(),
-            text: None,
-            xs_complex_type: schema::XsComplexType {
-                mixed: None,
-                text: None,
-                xs_attribute: Some(vec![
-                    schema::XsAttribute {
-                        name: "id".to_string(),
-                        xs_attribute_type: "xs:string".to_string(),
-                        xs_attribute_use: "required".to_string(),
-                    }
-                ]),
-                xs_sequence: None,
-            },
-        };
-        Arc::new(element)
-    }
 
     #[test]
-    fn test_self_close_with_greater_than() {
+    fn test_valid() {
-        let token = ElementSelfClose::new();
+        let values = ["<stop/>"];
-        
+        for value in values {
-        // Test closing with >
+            let mut validator = crate::tests::example_validator();
-        let next_tokens = token.append('>');
+            validator.append(value).expect(value);
-        assert!(!next_tokens.is_empty(), "Should accept '>' to close self-closing tag");
+            assert!(!validator.current_tokens.is_empty(), "{value}");
-        assert_eq!(next_tokens.len(), 1, "Should have exactly one token after '>'");
+            assert!(
-        
+                validator
-        match &next_tokens[0] {
+                    .current_tokens
-            Token::EndOfFile(_) => (),
+                    .iter()
-            _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
+                    .filter(|token| matches!(token, Token::EndOfFile(_)))
-        }
+                    .next()
-    }
+                    .is_some(),
-
+                "{value}"
-    #[test]
-    fn test_self_close_with_whitespace() {
-        let token = ElementSelfClose::new();
-        
-        // Test with whitespace
-        let next_tokens = token.append(' ');
-        assert!(!next_tokens.is_empty(), "Should accept whitespace in self-closing tag");
-        
-        match &next_tokens[0] {
-            Token::ElementSelfClose(_) => (),
-            _ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]),
-        }
-    }
-
-    #[test]
-    fn test_self_close_with_invalid_char() {
-        let token = ElementSelfClose::new();
-        
-        // Test with invalid character
-        let next_tokens = token.append('a');
-        assert!(next_tokens.is_empty(), "Should reject invalid character in self-closing tag");
-    }
-    
-    #[test]
-    fn test_required_attributes_validation() {
-        let element = create_element_with_required_attributes();
-        
-        // Without required attributes
-        let token = ElementSelfClose::new_with_attributes(Arc::clone(&element), vec![]);
-        let next_tokens = token.append('>');
-        assert!(next_tokens.is_empty(), "Should reject closing without required attributes");
-        
-        // With required attributes
-        let token = ElementSelfClose::new_with_attributes(
-            Arc::clone(&element),
-            vec!["id".to_string()]
             );
-        let next_tokens = token.append('>');
+        }
-        assert!(!next_tokens.is_empty(), "Should accept closing with required attributes");
+    }
 
-        match &next_tokens[0] {
+    #[test]
-            Token::EndOfFile(_) => (),
+    fn test_invalid() {
-            _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
+        let values = ["<stop/?", "<stop/a", "<stop/ "];
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect_err(value);
+            assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
         }
     }
 }

lib/xml_schema_validator/src/token/end_of_file.rs

@@ -2,8 +2,7 @@ use crate::*;
 
 /// Represents the final state at the end of an XML file
 #[derive(Clone, Debug)]
-pub struct EndOfFile {
+pub struct EndOfFile {}
-}
 
 impl EndOfFile {
     pub fn new() -> Self {
@@ -16,7 +15,7 @@ impl EndOfFile {
 }
 
 #[cfg(test)]
-mod end_of_file_tests {
+mod tests {
     use super::*;
 
     #[test]

lib/xml_schema_validator/src/token/mod.rs

@@ -1,46 +1,50 @@
+mod attribute_equals;
+mod attribute_name;
+mod attribute_value;
 mod element_close_name;
 mod element_close_start;
-mod element_open_name;
 mod element_open_end;
+mod element_open_name;
 mod element_open_start;
+mod element_self_close;
 mod end_of_file;
 mod start_of_file;
 mod text_content;
-mod attribute_name;
-mod attribute_value;
-mod element_self_close;
 
+pub use attribute_equals::AttributeEquals;
+pub use attribute_name::AttributeName;
+pub use attribute_value::AttributeValue;
 pub use element_close_name::ElementCloseName;
 pub use element_close_start::ElementCloseStart;
-pub use element_open_name::ElementOpenName;
 pub use element_open_end::ElementOpenEnd;
+pub use element_open_name::ElementOpenName;
 pub use element_open_start::ElementOpenStart;
+pub use element_self_close::ElementSelfClose;
 pub use end_of_file::EndOfFile;
 pub use start_of_file::StartOfFile;
 pub use text_content::TextContent;
-pub use attribute_name::AttributeName;
-pub use attribute_value::AttributeValue;
-pub use element_self_close::ElementSelfClose;
 
 /// Represents the different states of XML token parsing
 #[derive(Clone, Debug)]
 pub enum Token {
+    AttributeEquals(AttributeEquals),
+    AttributeName(AttributeName),
+    AttributeValue(AttributeValue),
     ElementCloseName(ElementCloseName),
     ElementCloseStart(ElementCloseStart),
-    ElementOpenName(ElementOpenName),
     ElementOpenEnd(ElementOpenEnd),
+    ElementOpenName(ElementOpenName),
     ElementOpenStart(ElementOpenStart),
+    ElementSelfClose(ElementSelfClose),
     EndOfFile(EndOfFile),
     StartOfFile(StartOfFile),
     TextContent(TextContent),
-    AttributeName(AttributeName),
-    AttributeValue(AttributeValue),
-    ElementSelfClose(ElementSelfClose),
 }
 
 impl Token {
     pub fn append(self, c: char) -> Vec<Token> {
         match self {
+            Token::AttributeEquals(attribute_equals) => attribute_equals.append(c),
             Token::ElementCloseName(element_close_name) => element_close_name.append(c),
             Token::ElementCloseStart(element_close_start) => element_close_start.append(c),
             Token::ElementOpenName(element_name) => element_name.append(c),
@@ -66,10 +70,10 @@ impl Token {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use std::sync::Arc;
     use crate::schema;
+    use std::sync::Arc;
 
-    fn create_test_element() -> Arc<schema::XsElement> {
+    pub fn create_test_element() -> Arc<schema::XsElement> {
         let element = schema::XsElement {
             name: "reasoning".to_string(),
             text: None,
@@ -90,26 +94,6 @@ mod tests {
         Arc::new(element)
     }
 
-    #[test]
-    fn test_start_of_file_transition() {
-        let element = create_test_element();
-        let token = Token::StartOfFile(StartOfFile::new(Arc::clone(&element)));
-        
-        // Should only accept '<'
-        let next_tokens = token.append('<');
-        assert_eq!(next_tokens.len(), 1, "Should have one transition");
-        
-        match &next_tokens[0] {
-            Token::ElementOpenStart(_) => (),
-            _ => panic!("Expected ElementStart, got {:?}", next_tokens[0]),
-        }
-        
-        // Should reject other characters
-        let token = Token::StartOfFile(StartOfFile::new(Arc::clone(&element)));
-        let next_tokens = token.append('a');
-        assert_eq!(next_tokens.len(), 0, "Should reject 'a'");
-    }
-    
     #[test]
     fn test_element_start_transition() {
         let element = create_test_element();
@@ -133,7 +117,8 @@ mod tests {
     #[test]
     fn test_element_name_transition() {
         let element = create_test_element();
-        let token = Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()).unwrap());
+        let token =
+            Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()));
 
         // Continue building name
         let next_tokens = token.append('e');
@@ -146,7 +131,9 @@ mod tests {
 
         // Build complete name and end with >
         // Remove the unused buffer
-        let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)))];
+        let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(
+            &element,
+        )))];
 
         for c in "reasoning>".chars() {
             let mut new_tokens = Vec::new();
@@ -155,8 +142,11 @@ mod tests {
             }
             current_tokens = new_tokens;
 
-            assert!(!current_tokens.is_empty(), 
+            assert!(
-                "Should have valid transition after appending '{}'", c);
+                !current_tokens.is_empty(),
+                "Should have valid transition after appending '{}'",
+                c
+            );
         }
 
         // Check final state
@@ -264,8 +254,11 @@ mod tests {
             }
             current_tokens = new_tokens;
 
-            assert!(!current_tokens.is_empty(), 
+            assert!(
-                "Should have valid transition after appending '{}'", c);
+                !current_tokens.is_empty(),
+                "Should have valid transition after appending '{}'",
+                c
+            );
         }
 
         // Final > to close - clone the token to avoid moving out of the vector

lib/xml_schema_validator/src/token/start_of_file.rs

@@ -1,5 +1,5 @@
-use std::sync::Arc;
 use crate::*;
+use std::sync::Arc;
 
 /// Represents the initial state at the start of an XML file, containing a reference to the root element schema
 #[derive(Clone, Debug)]
@@ -14,7 +14,9 @@ impl StartOfFile {
 
     pub fn append(self, c: char) -> Vec<Token> {
         if '<' == c {
-            vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&self.element)))]
+            vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(
+                &self.element,
+            )))]
         } else {
             vec![]
         }
@@ -22,54 +24,29 @@ impl StartOfFile {
 }
 
 #[cfg(test)]
-mod start_of_file_tests {
+mod tests {
     use super::*;
-    use std::sync::Arc;
-    use crate::schema;
 
-    fn create_test_element() -> Arc<schema::XsElement> {
+    #[test]
-        let element = schema::XsElement {
+    fn test_accept_open_start_token() {
-            name: "reasoning".to_string(),
+        let mut validator = crate::tests::example_validator();
-            text: None,
+        validator.append("<").unwrap();
-            xs_complex_type: schema::XsComplexType {
+        assert!(!validator.current_tokens.is_empty());
-                mixed: Some("true".to_string()),
+        assert!(validator
-                text: None,
+            .current_tokens
-                xs_attribute: None,
+            .iter()
-                xs_sequence: Some(schema::XsSequence {
+            .filter(|token| matches!(token, Token::ElementOpenStart(_)))
-                    text: None,
+            .next()
-                    xs_any: schema::XsAny {
+            .is_some());
-                        min_occurs: "0".to_string(),
-                        max_occurs: "unbounded".to_string(),
-                        process_contents: "skip".to_string(),
-                    },
-                }),
-            },
-        };
-        Arc::new(element)
     }
 
     #[test]
-    fn test_start_of_file_less_than() {
+    fn test_not_accept_other() {
-        let element = create_test_element();
+        let values = ["a", " ", "\t", ">", "/"];
-        let token = StartOfFile::new(Arc::clone(&element));
+        for value in values {
-        
+            let mut validator = crate::tests::example_validator();
-        // Test with <
+            validator.append(value).expect_err(value);
-        let next_tokens = token.append('<');
+            assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
-        assert!(!next_tokens.is_empty(), "Should accept '<'");
-        
-        match &next_tokens[0] {
-            Token::ElementOpenStart(_) => (),
-            _ => panic!("Expected ElementStart, got {:?}", next_tokens[0]),
         }
     }
-
-    #[test]
-    fn test_start_of_file_invalid() {
-        let element = create_test_element();
-        let token = StartOfFile::new(Arc::clone(&element));
-        
-        // Test with invalid character
-        let next_tokens = token.append('a');
-        assert!(next_tokens.is_empty(), "Should reject invalid character");
-    }
 }

lib/xml_schema_validator/src/token/text_content.rs

@@ -1,5 +1,5 @@
-use std::sync::Arc;
 use crate::*;
+use std::sync::Arc;
 
 /// Represents the text content within an XML element
 #[derive(Clone, Debug)]
@@ -9,18 +9,18 @@ pub struct TextContent {
 
 impl TextContent {
     pub fn new(element: Arc<schema::XsElement>) -> Self {
-        Self {
+        Self { element }
-            element,
-        }
     }
 
     pub fn append(self, c: char) -> Vec<Token> {
         if c == '<' {
-            vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
+            vec![Token::ElementCloseStart(ElementCloseStart::new(
-        } else {
-            vec![Token::TextContent(TextContent::new(
                 Arc::clone(&self.element),
             ))]
+        } else {
+            vec![Token::TextContent(TextContent::new(Arc::clone(
+                &self.element,
+            )))]
         }
     }
 }
@@ -28,8 +28,8 @@ impl TextContent {
 #[cfg(test)]
 mod text_content_tests {
     use super::*;
-    use std::sync::Arc;
     use crate::schema;
+    use std::sync::Arc;
 
     fn create_test_element() -> Arc<schema::XsElement> {
         let element = schema::XsElement {

lib/xml_schema_validator/tests/test_logits_processor.py

@@ -1,13 +1,12 @@
 from transformers import AutoTokenizer
 from xml_schema_validator import XmlLogitsProcessor
 import os
-import time
 import torch
 
 model_path = "../../model"
 api_token = os.environ["SIA_HF_API_KEY"]
 
-xml_schema_actions = open("../../action_schema.xsd").read()
+xml_schema_actions = open("example_schema.xsd").read()
 xml_schema_only_root_node = """<?xml version="1.0" encoding="UTF-8"?>
 <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
   <xs:element name="root">
@@ -60,59 +59,26 @@ def test_token_masking():
     tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token)
     processor = XmlLogitsProcessor(tokenizer, xml_schema_actions)
 
-    # Create dummy input_ids and scores
+    # Process empty input to set prompt length
+    input_ids = torch.tensor([tokenizer.encode("")])
+    scores = torch.ones((1, len(tokenizer)))  # All tokens have equal probability
+    processed_scores = processor(input_ids, scores)
+    
+    # Process valid continuation
     input_ids = torch.tensor([tokenizer.encode("<reasoning>")])
     scores = torch.ones((1, len(tokenizer)))  # All tokens have equal probability
-    
-    # Process the scores
     processed_scores = processor(input_ids, scores)
     
     # Verify that valid continuations still have positive scores
-    assert processed_scores[0, tokenizer.encode(" ", add_special_tokens=False)[0]] > -float('inf')
+    space_token = tokenizer.encode(" ", add_special_tokens=False)[0]
+    assert processed_scores[0, space_token] > -float('inf')
     
     # Verify that invalid continuations are masked
-    assert processed_scores[0, tokenizer.encode("<invalid>", add_special_tokens=False)[0]] == -float('inf')
+    input_ids = torch.tensor([tokenizer.encode("</invalid>")])
-
+    scores = torch.ones((1, len(tokenizer)))  # All tokens have equal probability
-def test_performance():
-    """Test performance with larger XML documents"""
-    
-    tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token)
-    processor = XmlLogitsProcessor(tokenizer, xml_schema_actions)
-    
-    # Generate a larger XML document
-    large_xml = "<reasoning>" + "Test content. " * 100 + "</reasoning>"
-    
-    # Measure time to process
-    start_time = time.time()
-    processor.core.append(large_xml)
-    processing_time = time.time() - start_time
-    
-    print(f"Processing time for large XML: {processing_time:.4f} seconds")
-    # You might want to assert that processing time is below a threshold
-
-def test_subword_tokens():
-    """Test handling of subword tokens that might split XML tags"""
-    tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token)
-    schema_text = open("../../action_schema.xsd").read()
-    
-    processor = XmlLogitsProcessor(tokenizer, schema_text)
-    
-    # Find some XML tag that gets split into multiple tokens by your tokenizer
-    tag = "<reasoning>"
-    tokens = tokenizer.encode(tag, add_special_tokens=False)
-    
-    if len(tokens) > 1:
-        print(f"Tag '{tag}' is split into {len(tokens)} tokens")
-        
-        # Test that the processor can handle these split tokens
-        import torch
-        input_ids = torch.tensor([[tokens[0]]])
-        scores = torch.ones((1, len(tokenizer)))
-        
     processed_scores = processor(input_ids, scores)
-        
+    assert processed_scores[0, tokenizer.eos_token_id] == 1
-        # The next token in the sequence should have a high score
+    assert processed_scores[0, space_token] == -float('inf')
-        assert processed_scores[0, tokens[1]] > -float('inf')
     
 if __name__ == "__main__":
     # Run individual tests for debugging
@@ -120,5 +86,3 @@ if __name__ == "__main__":
     test_xml_schema_parsing()
     test_basic_xml_validation()
     test_token_masking()
-    test_performance()
-    test_subword_tokens()