diff --git a/lib/xml_schema_validator/example_schema.xsd b/lib/xml_schema_validator/example_schema.xsd new file mode 100644 index 0000000..14f94c9 --- /dev/null +++ b/lib/xml_schema_validator/example_schema.xsd @@ -0,0 +1,118 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/xml_schema_validator/python/xml_schema_validator/__init__.py b/lib/xml_schema_validator/python/xml_schema_validator/__init__.py index c0393a0..c7cc733 100644 --- a/lib/xml_schema_validator/python/xml_schema_validator/__init__.py +++ b/lib/xml_schema_validator/python/xml_schema_validator/__init__.py @@ -12,7 +12,7 @@ class XmlLogitsProcessor(LogitsProcessor): by setting their logits to negative infinity. """ - def __init__(self, tokenizer: AutoTokenizer, schema_text: str = None, core=None): + def __init__(self, tokenizer: AutoTokenizer, schema_text: str = None, core: XmlLogitsProcessorCore=None): """ Initialize the processor with a schema and tokenizer. diff --git a/lib/xml_schema_validator/src/char_trie.rs b/lib/xml_schema_validator/src/char_trie.rs index 3e909f0..50e2658 100644 --- a/lib/xml_schema_validator/src/char_trie.rs +++ b/lib/xml_schema_validator/src/char_trie.rs @@ -1,103 +1,116 @@ -use crate::*; -use pyo3::prelude::*; - -// Node in our character trie -struct CharTrieNode { - // Store character and node pairs, keep sorted for binary search - children: Vec<(char, CharTrieNode)>, - // Store token IDs that are valid at this point - token_id: Option>, -} - -impl CharTrieNode { - fn new() -> Self { - Self { - children: Vec::new(), - token_id: None, - } - } - - // Add a child for character c, maintaining sorted order - fn add_child(&mut self, c: char) -> &mut CharTrieNode { - match self.children.binary_search_by_key(&c, |(ch, _)| *ch) { - Ok(index) => &mut self.children[index].1, - Err(index) => { - // Insert at the right position to maintain sort order - self.children.insert(index, (c, CharTrieNode::new())); - &mut self.children[index].1 - } - } - } - - // Add a token id to this node - fn add_token(&mut self, token_id: Py) { - self.token_id = Some(token_id); - } - - pub fn clone_ref(&self, py: Python<'_>) -> Self { - Self { - children: self.children.iter().map(|(ch, node)| (*ch, node.clone_ref(py))).collect(), - token_id: self.token_id.iter().map(|token_id| token_id.clone_ref(py)).nth(0), - } - } -} - -// The main trie structure -pub struct CharTrie { - root: CharTrieNode, -} - -impl CharTrie { - pub fn new() -> Self { - Self { - root: CharTrieNode::new(), - } - } - - // Insert a token's character sequence into the trie - pub fn insert(&mut self, token_text: &str, token_id: Py) { - let mut current = &mut self.root; - - // Add each character in the token text - for c in token_text.chars() { - current = current.add_child(c); - } - - // Mark this node as representing token_id - current.add_token(token_id); - } - - // Find all tokens that could be invalid given the current XML state - pub fn find_invalid_tokens(&self, py: Python<'_>, xml_validator: &XmlSchemaValidator, token_map: &Vec<(Py, String)>) -> Vec> { - let mut invalid_tokens = Vec::new(); - - for (token_id, token_text) in token_map { - // Check if this token would make the XML invalid - if !self.is_valid_continuation(xml_validator, token_text) { - invalid_tokens.push(token_id.clone_ref(py)); - } - } - - invalid_tokens - } - - // Check if a token would be a valid continuation - fn is_valid_continuation(&self, xml_validator: &XmlSchemaValidator, token_text: &str) -> bool { - // First, quick check with just the first character - if let Some(first_char) = token_text.chars().next() { - if !xml_validator.can_continue_with(first_char) { - return false; - } - } - - // If first character is valid, try the full token - let mut validator_clone = xml_validator.clone(); - validator_clone.append(token_text).is_ok() - } - - pub fn clone_ref(&self, py: Python<'_>) -> Self { - Self { - root: self.root.clone_ref(py), - } - } -} \ No newline at end of file +use crate::*; +use pyo3::prelude::*; + +// Node in our character trie +struct CharTrieNode { + // Store character and node pairs, keep sorted for binary search + children: Vec<(char, CharTrieNode)>, + // Store token IDs that are valid at this point + token_id: Option>, +} + +impl CharTrieNode { + fn new() -> Self { + Self { + children: Vec::new(), + token_id: None, + } + } + + // Add a child for character c, maintaining sorted order + fn add_child(&mut self, c: char) -> &mut CharTrieNode { + match self.children.binary_search_by_key(&c, |(ch, _)| *ch) { + Ok(index) => &mut self.children[index].1, + Err(index) => { + // Insert at the right position to maintain sort order + self.children.insert(index, (c, CharTrieNode::new())); + &mut self.children[index].1 + } + } + } + + // Add a token id to this node + fn add_token(&mut self, token_id: Py) { + self.token_id = Some(token_id); + } + + pub fn clone_ref(&self, py: Python<'_>) -> Self { + Self { + children: self + .children + .iter() + .map(|(ch, node)| (*ch, node.clone_ref(py))) + .collect(), + token_id: self + .token_id + .iter() + .map(|token_id| token_id.clone_ref(py)) + .nth(0), + } + } +} + +// The main trie structure +pub struct CharTrie { + root: CharTrieNode, +} + +impl CharTrie { + pub fn new() -> Self { + Self { + root: CharTrieNode::new(), + } + } + + // Insert a token's character sequence into the trie + pub fn insert(&mut self, token_text: &str, token_id: Py) { + let mut current = &mut self.root; + + // Add each character in the token text + for c in token_text.chars() { + current = current.add_child(c); + } + + // Mark this node as representing token_id + current.add_token(token_id); + } + + // Find all tokens that could be invalid given the current XML state + pub fn find_invalid_tokens( + &self, + py: Python<'_>, + xml_validator: &XmlSchemaValidator, + token_map: &Vec<(Py, String)>, + ) -> Vec> { + let mut invalid_tokens = Vec::new(); + + for (token_id, token_text) in token_map { + // Check if this token would make the XML invalid + if !self.is_valid_continuation(xml_validator, token_text) { + invalid_tokens.push(token_id.clone_ref(py)); + } + } + + invalid_tokens + } + + // Check if a token would be a valid continuation + fn is_valid_continuation(&self, xml_validator: &XmlSchemaValidator, token_text: &str) -> bool { + // First, quick check with just the first character + if let Some(first_char) = token_text.chars().next() { + if !xml_validator.can_continue_with(first_char) { + return false; + } + } + + // If first character is valid, try the full token + let mut validator_clone = xml_validator.clone(); + validator_clone.append(token_text).is_ok() + } + + pub fn clone_ref(&self, py: Python<'_>) -> Self { + Self { + root: self.root.clone_ref(py), + } + } +} diff --git a/lib/xml_schema_validator/src/error.rs b/lib/xml_schema_validator/src/error.rs index 4a899a6..c511a9c 100644 --- a/lib/xml_schema_validator/src/error.rs +++ b/lib/xml_schema_validator/src/error.rs @@ -1,14 +1,14 @@ -/// Error types that can occur during XML validation - -#[derive(Clone, Debug, thiserror::Error)] -pub enum Error { - /// Indicates that the provided XSD schema is invalid - #[error("Invalid schema: {0}")] - InvalidSchema(#[from] quick_xml::DeError), - /// Indicates that the provided XML fragment is invalid - #[error("Invalid XML: {0}")] - InvalidXml(String), - /// Not implemented yet - #[error("Not implemented yet")] - NotImplemented, -} \ No newline at end of file +/// Error types that can occur during XML validation + +#[derive(Clone, Debug, thiserror::Error)] +pub enum Error { + /// Indicates that the provided XSD schema is invalid + #[error("Invalid schema: {0}")] + InvalidSchema(#[from] quick_xml::DeError), + /// Indicates that the provided XML fragment is invalid + #[error("Invalid XML: {0}")] + InvalidXml(String), + /// Not implemented yet + #[error("Not implemented yet")] + NotImplemented, +} diff --git a/lib/xml_schema_validator/src/lib.rs b/lib/xml_schema_validator/src/lib.rs index de3d1c0..795e6b9 100644 --- a/lib/xml_schema_validator/src/lib.rs +++ b/lib/xml_schema_validator/src/lib.rs @@ -1,619 +1,739 @@ -#![warn(missing_docs)] -#![warn(rust_2018_idioms)] - -//! A streaming XML validator that checks XML fragments against an XSD schema - -mod char_trie; -mod error; -mod py_xml_logits_processor_core; -mod py_xml_schema_validator; -mod python; -mod schema; -mod token; - -use std::sync::Arc; -use error::Error; -use token::*; - -/// An XML validator that checks XML fragments against an XSD schema -#[derive(Clone, Debug)] -pub struct XmlSchemaValidator { - current_tokens: Vec, - schema: Arc, -} - -/// The Result type used throughout this crate -/// -/// This is a type alias for `std::result::Result` with our custom `Error` type -pub type Result = std::result::Result; - -impl XmlSchemaValidator { - /// Create a new validator with the given schema text - pub fn new(schema_text: &str) -> Result { - let schema: schema::XsSchema = quick_xml::de::from_str(schema_text)?; - let schema_arc = Arc::new(schema); - - let current_tokens = schema_arc.xs_element.iter().map(|element| { - let element_arc = Arc::new(element.clone()); - Token::StartOfFile(StartOfFile::new(element_arc)) - }).collect(); - - Ok(Self { - current_tokens, - schema: schema_arc, - }) - } - - /// Append a fragment of XML to the validator - pub fn append(&mut self, fragment: &str) -> Result<()> { - for c in fragment.chars() { - self.append_char(c)?; - } - Ok(()) - } - - /// Check if the validator has reached the end of the XML - pub fn eof(&self) -> bool { - self.current_tokens.is_empty() || self.current_tokens.iter().all(Token::is_eof) - } - - /// Append a single character to the validator - fn append_char(&mut self, c: char) -> Result<()> { - let mut new_tokens = vec![]; - for token in self.current_tokens.drain(..) { - new_tokens.append(&mut token.append(c)); - } - if new_tokens.is_empty() { - return Err(Error::InvalidXml(format!("No valid continuations for character: '{}'", c))); - } - self.current_tokens = new_tokens; - Ok(()) - } - - /// Try to validate a continuation character - pub fn can_continue_with(&self, c: char) -> bool { - // Clone the current state and try to append the character - let mut validator_clone = self.clone(); - validator_clone.append_char(c).is_ok() - } - - /// Get a list of all element names in the schema - pub fn get_element_names(&self) -> Vec { - self.schema.xs_element.iter() - .map(|element| element.name.clone()) - .collect() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_element_open() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); - let input = ""; - let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); - validator.append(input).unwrap(); - } - - #[test] - fn test_element_open_whitespace() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); - let input = ""; - let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); - validator.append(input).unwrap(); - } - - #[test] - fn test_element_open_invalid_name() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); - let input = ""#; - let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); - validator.append(input).unwrap(); - assert!(validator.eof()); - } - - #[test] - fn test_delete_missing_required_attribute() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); - // Create validator - let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); - - // First, validate up to the opening tag - assert!(validator.append("").is_err(), "Should fail with missing required attribute"); - - // Create a new validator and try closing tag approach - let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); - // Try to validate an incomplete delete tag that's missing the required id attribute - let input = ""; - let result = validator.append(input); - assert!(result.is_err(), "Should fail without required id attribute"); - - // Check the error message - note the test now checks for either "required" OR "No valid continuations" - if let Err(err) = result { - println!("Error message: {}", err); - let error_str = err.to_string().to_lowercase(); - assert!(error_str.contains("required") || error_str.contains("no valid continuations"), - "Error should mention missing required attribute or invalid continuation"); - } - } - - #[test] - fn test_invalid_nesting() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); - let validator = XmlSchemaValidator::new(&schema_text).unwrap(); - validator.clone().append("").expect_err("Should fail because of nested tag"); - validator.clone().append(" ").expect_err("Should fail because of nested tag, even with whitespace"); - validator.clone().append("foo").expect_err("Should fail because of nested tag, even with text"); - validator.clone().append(" foo ").expect_err("Should fail because of nested tag, even with text and whitespace"); - validator.clone().append(" foo ").expect_err("Should fail because of nested tag, even with self closing tag"); - } - - #[test] - fn test_stop_self_closing() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); - let input = ""; - let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); - validator.append(input).unwrap(); - assert!(validator.eof()); - } - - #[test] - fn test_read_stdin_self_closing() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); - let input = ""; - let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); - validator.append(input).unwrap(); - assert!(validator.eof()); - } - - #[test] - fn test_single_with_attributes() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); - let input = r#"ls -la"#; - let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); - validator.append(input).unwrap(); - assert!(validator.eof()); - } - - #[test] - fn test_reasoning_with_content() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); - let input = "I should explore the file system for interesting files."; - let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); - validator.append(input).unwrap(); - assert!(validator.eof()); - } - - #[test] - fn test_write_stdout_with_content() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); - let input = "Hello world!"; - let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); - validator.append(input).unwrap(); - assert!(validator.eof()); - } - - #[test] - fn test_create_validator() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); - let validator = XmlSchemaValidator::new(&schema_text); - assert!(validator.is_ok(), "Failed to create validator: {:?}", validator.err()); - } - - #[test] - fn test_get_element_names() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); - let validator = XmlSchemaValidator::new(&schema_text).unwrap(); - let names = validator.get_element_names(); - assert!(names.contains(&"reasoning".to_string()), - "Element names should contain 'reasoning', got: {:?}", names); - } - - // Test basic XML fragment validation - debugging the "" failure - #[test] - fn test_append_reasoning_tag() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); - let validator = XmlSchemaValidator::new(&schema_text).unwrap(); - // Test character by character for precise failure point - assert!(validator.can_continue_with('<'), "Should accept '<'"); - // Clone at each step to debug exact point of failure - let mut v1 = validator.clone(); - assert!(v1.append_char('<').is_ok(), "Failed to append '<'"); - let mut v2 = v1.clone(); - assert!(v2.append_char('r').is_ok(), "Failed to append 'r'"); - // Continue with each character in "reasoning>" - let mut v3 = v2.clone(); - for c in "easoning>".chars() { - assert!(v3.append_char(c).is_ok(), "Failed to append '{}'", c); - } - // Try the whole string at once - let mut v4 = validator.clone(); - let result = v4.append(""); - assert!(result.is_ok(), "Failed to append '': {:?}", result.err()); - } - - // Test validation of multiple fragments - #[test] - fn test_multiple_append() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); - let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); - // Test sequential appends - assert!(validator.append("<").is_ok(), "Failed to append '<'"); - assert!(validator.append("reasoning").is_ok(), "Failed to append 'reasoning'"); - assert!(validator.append(">").is_ok(), "Failed to append '>'"); - assert!(validator.append("test content").is_ok(), "Failed to append content"); - assert!(validator.append("").is_ok(), "Failed to append closing tag"); - // Should be at EOF now - assert!(validator.eof(), "Should be at EOF after complete XML"); - } - - // Test token continuation - critical for the token masking issue - #[test] - fn test_can_continue_with() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); - // Test at start - let validator = XmlSchemaValidator::new(&schema_text).unwrap(); - assert!(validator.can_continue_with('<'), "Should accept '<' at start"); - assert!(!validator.can_continue_with('a'), "Should reject 'a' at start"); - // Test after opening tag - let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); - validator.append("").unwrap(); - // Any character should be valid inside a mixed content element - assert!(validator.can_continue_with('a'), "Should accept 'a' inside element"); - assert!(validator.can_continue_with('1'), "Should accept '1' inside element"); - assert!(validator.can_continue_with('<'), "Should accept '<' inside element"); - } -} - -#[cfg(test)] -mod diagnostic_tests { - use super::*; - use std::sync::Arc; - - // Simple schema with just the reasoning tag for cleaner debugging - fn get_simple_schema() -> String { - r#" - - - - - - - - - "#.to_string() - } - - // Test every character in an opening tag one by one - #[test] - fn test_char_by_char_open_tag() { - let schema_text = get_simple_schema(); - let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); - - // Test each character individually - assert!(validator.can_continue_with('<'), "Should accept '<'"); - assert!(validator.append_char('<').is_ok(), "Failed to append '<'"); - - assert!(validator.can_continue_with('r'), "Should accept 'r'"); - assert!(validator.append_char('r').is_ok(), "Failed to append 'r'"); - - for c in "easoning".chars() { - assert!(validator.can_continue_with(c), "Should accept '{}'", c); - assert!(validator.append_char(c).is_ok(), "Failed to append '{}'", c); - } - - // This is where many tests are failing - let's check the state before and after - println!("State before '>': {:?}", validator.current_tokens); - let result = validator.append_char('>'); - println!("Result of append '>': {:?}", result); - if result.is_ok() { - println!("State after '>': {:?}", validator.current_tokens); - } - assert!(result.is_ok(), "Failed to append '>'"); - } - - // Test the token transitions specifically - #[test] - fn test_token_transitions() { - // Create minimal elements for testing token transitions directly - let test_element = Arc::new(schema::XsElement { - name: "reasoning".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: Some("true".to_string()), - text: None, - xs_attribute: None, - xs_sequence: Some(schema::XsSequence { - text: None, - xs_any: schema::XsAny { - min_occurs: "0".to_string(), - max_occurs: "unbounded".to_string(), - process_contents: "skip".to_string(), - }, - }), - }, - }); - - // Test the transition from ElementOpenName to ElementOpenEnd - let element_name = token::ElementOpenName::new(Arc::clone(&test_element), "reasoning".to_string()).unwrap(); - let tokens = element_name.append('>'); - assert!(!tokens.is_empty(), "Should produce valid tokens after '>'"); - println!("Token after appending '>' to ElementOpenName: {:?}", tokens[0]); - - // Check what type the token is - match &tokens[0] { - Token::ElementOpenEnd(element_open) => { - println!("Successfully transitioned to ElementOpen: {:?}", element_open); - }, - other => { - panic!("Expected ElementOpen, got {:?}", other); - } - } - } - - // Test element self-closing functionality - #[test] - fn test_element_self_close_transition() { - let test_element = Arc::new(schema::XsElement { - name: "stop".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: None, - text: None, - xs_attribute: None, - xs_sequence: None, - }, - }); - - // Test transition from ElementName to ElementSelfClose - let element_name = token::ElementOpenName::new(Arc::clone(&test_element), "stop".to_string()).unwrap(); - let tokens = element_name.append('/'); - assert!(!tokens.is_empty(), "Should handle '/' after element name"); - - match &tokens[0] { - Token::ElementSelfClose(element_self_close) => { - println!("Successfully transitioned to ElementSelfClose"); - // Now test transition to EndOfFile - let next_tokens = element_self_close.clone().append('>'); - assert!(!next_tokens.is_empty(), "Should handle '>' after '/'"); - - match &next_tokens[0] { - Token::EndOfFile(_) => println!("Successfully transitioned to EndOfFile"), - other => panic!("Expected EndOfFile, got {:?}", other), - } - }, - other => panic!("Expected ElementSelfClose, got {:?}", other), - } - } - - // Test whitespace handling - #[test] - fn test_whitespace_handling() { - let schema_text = get_simple_schema(); - let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); - - // Test opening tag with whitespace - for c in " - assert!(validator.can_continue_with('>'), "Should accept '>' after whitespace"); - assert!(validator.append_char('>').is_ok(), "Failed to append '>' after whitespace"); - } - - // Test attribute handling - #[test] - fn test_attribute_handling() { - let schema_text = r#" - - - - - - - "#; - - let mut validator = XmlSchemaValidator::new(schema_text).unwrap(); - - // Test opening tag with attribute - for c in "".chars() { - assert!(validator.can_continue_with(c), "Should accept '{}' to close tag", c); - assert!(validator.append_char(c).is_ok(), "Failed to append '{}' to close tag", c); - } - - // Check if we've reached EOF - assert!(validator.eof(), "Should be at EOF after self-closing tag"); - } - - // Test for the specific error patterns seen in the test output - #[test] - fn test_specific_error_cases() { - let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap_or_else(|_| get_simple_schema()); - - // Test case 1: Simple opening tag (noticed in `test_element_open`) - let input1 = ""; - let mut validator1 = XmlSchemaValidator::new(&schema_text).unwrap(); - let result1 = validator1.append(input1); - println!("Result appending '': {:?}", result1); - assert!(result1.is_ok(), "Failed on ''"); - - // Test case 2: Element with whitespace (noticed in `test_element_open_whitespace`) - let input2 = ""; - let mut validator2 = XmlSchemaValidator::new(&schema_text).unwrap(); - let result2 = validator2.append(input2); - println!("Result appending '': {:?}", result2); - assert!(result2.is_ok(), "Failed on ''"); - - // Test case 3: Self-closing tags (noticed in `test_read_stdin_self_closing`) - let input3 = ""; - let mut validator3 = XmlSchemaValidator::new(&schema_text).unwrap(); - let result3 = validator3.append(input3); - println!("Result appending '': {:?}", result3); - assert!(result3.is_ok(), "Failed on ''"); - - // Test case 4: Attributes (noticed in `test_single_with_attributes`) - let input4 = r#""#; - let mut validator4 = XmlSchemaValidator::new(&schema_text).unwrap(); - let result4 = validator4.append(input4); - println!("Result appending attribute: {:?}", result4); - assert!(result4.is_ok(), "Failed on attribute handling"); - } - - // Test the basic token state machine with a minimal element - #[test] - fn test_basic_token_operations() { - // Create a minimal element for testing - let element = Arc::new(schema::XsElement { - name: "test".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: Some("true".to_string()), - text: None, - xs_attribute: None, - xs_sequence: None, - }, - }); - - // Create and test each token type - let start_of_file = token::StartOfFile::new(Arc::clone(&element)); - let tokens = start_of_file.append('<'); - assert!(!tokens.is_empty(), "StartOfFile should accept '<'"); - - let element_start = match &tokens[0] { - Token::ElementOpenStart(es) => es.clone(), - _ => panic!("Expected ElementStart"), - }; - - let tokens = element_start.append('t'); - assert!(!tokens.is_empty(), "ElementStart should accept 't'"); - - let element_name = match &tokens[0] { - Token::ElementOpenName(en) => en.clone(), - _ => panic!("Expected ElementName"), - }; - - // This is often where things break - println!("About to append 'e' to ElementName..."); - let tokens = element_name.append('e'); - assert!(!tokens.is_empty(), "ElementName should accept 'e'"); - - // Build the rest of the name - let mut current_token = tokens[0].clone(); - for c in "st".chars() { - let next_tokens = current_token.append(c); - assert!(!next_tokens.is_empty(), "Token should accept '{}'", c); - current_token = next_tokens[0].clone(); - } - - // Now try to append '>' - println!("About to append '>' to token: {:?}", current_token); - let tokens = current_token.append('>'); - println!("Result of appending '>': {:?}", tokens); - assert!(!tokens.is_empty(), "Token should accept '>'"); - - // Verify transition to ElementOpen - match &tokens[0] { - Token::ElementOpenEnd(_) => println!("Successfully transitioned to ElementOpen"), - other => panic!("Expected ElementOpen, got {:?}", other), - } - } - - // This test will help identify issues with token creation for specific elements - #[test] - fn test_element_specific_issues() { - if let Ok(schema_text) = std::fs::read_to_string("../../action_schema.xsd") { - let validator = XmlSchemaValidator::new(&schema_text).unwrap(); - - // Get all element names in the schema - let element_names = validator.get_element_names(); - println!("Elements in schema: {:?}", element_names); - - // Test each element separately - for name in &element_names { - println!("\nTesting element: {}", name); - - let mut element_validator = XmlSchemaValidator::new(&schema_text).unwrap(); - let open_tag = format!("<{}>", name); - - // Test character by character - println!("Testing character by character for <{}>", name); - let mut progress = String::new(); - for c in open_tag.chars() { - progress.push(c); - let result = element_validator.can_continue_with(c); - println!(" Can continue with '{}' after '{}': {}", - c, &progress[..progress.len()-1], result); - - if !result { - // This is helpful to diagnose where the process fails - println!(" ❌ Failed on character '{}' for element '{}'", c, name); - break; - } - - if let Err(e) = element_validator.append_char(c) { - println!(" ❌ Failed to append '{}' for element '{}': {:?}", c, name, e); - break; - } - } - - // Test complete tag at once - let mut complete_validator = XmlSchemaValidator::new(&schema_text).unwrap(); - match complete_validator.append(&open_tag) { - Ok(_) => println!(" ✅ Successfully appended <{}>", name), - Err(e) => println!(" ❌ Failed to append <{}>: {:?}", name, e), - } - } - } else { - println!("Schema file not found, skipping element-specific tests"); - } - } -} \ No newline at end of file +#![warn(missing_docs)] +#![warn(rust_2018_idioms)] + +//! A streaming XML validator that checks XML fragments against an XSD schema + +mod char_trie; +mod error; +mod py_xml_logits_processor_core; +mod py_xml_schema_validator; +mod python; +mod schema; +mod token; + +use error::Error; +use std::sync::Arc; +use token::*; + +/// An XML validator that checks XML fragments against an XSD schema +#[derive(Clone, Debug)] +pub struct XmlSchemaValidator { + current_tokens: Vec, + schema: Arc, +} + +/// The Result type used throughout this crate +/// +/// This is a type alias for `std::result::Result` with our custom `Error` type +pub type Result = std::result::Result; + +impl XmlSchemaValidator { + /// Create a new validator with the given schema text + pub fn new(schema_text: &str) -> Result { + let schema: schema::XsSchema = quick_xml::de::from_str(schema_text)?; + let schema_arc = Arc::new(schema); + + let current_tokens = schema_arc + .xs_element + .iter() + .map(|element| { + let element_arc = Arc::new(element.clone()); + Token::StartOfFile(StartOfFile::new(element_arc)) + }) + .collect(); + + Ok(Self { + current_tokens, + schema: schema_arc, + }) + } + + /// Append a fragment of XML to the validator + pub fn append(&mut self, fragment: &str) -> Result<()> { + for c in fragment.chars() { + self.append_char(c)?; + } + Ok(()) + } + + /// Check if the validator has reached the end of the XML + pub fn eof(&self) -> bool { + self.current_tokens.is_empty() || self.current_tokens.iter().all(Token::is_eof) + } + + /// Append a single character to the validator + fn append_char(&mut self, c: char) -> Result<()> { + let mut new_tokens = vec![]; + for token in self.current_tokens.drain(..) { + new_tokens.append(&mut token.append(c)); + } + if new_tokens.is_empty() { + return Err(Error::InvalidXml(format!( + "No valid continuations for character: '{}'", + c + ))); + } + self.current_tokens = new_tokens; + Ok(()) + } + + /// Try to validate a continuation character + pub fn can_continue_with(&self, c: char) -> bool { + // Clone the current state and try to append the character + let mut validator_clone = self.clone(); + validator_clone.append_char(c).is_ok() + } + + /// Get a list of all element names in the schema + pub fn get_element_names(&self) -> Vec { + self.schema + .xs_element + .iter() + .map(|element| element.name.clone()) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + pub fn example_validator() -> XmlSchemaValidator { + let schema_text = std::fs::read_to_string("example_schema.xsd").unwrap(); + XmlSchemaValidator::new(&schema_text).unwrap() + } + + #[test] + fn test_element_open() { + let mut validator = example_validator(); + let input = ""; + validator.append(input).unwrap(); + } + + #[test] + fn test_element_open_whitespace() { + let mut validator = example_validator(); + let input = ""; + validator.append(input).unwrap(); + } + + #[test] + fn test_element_open_invalid_name() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let input = ""#; + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + validator.append(input).unwrap(); + assert!(validator.eof()); + } + + #[test] + fn test_delete_missing_required_attribute() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + // Create validator + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + + // First, validate up to the opening tag + assert!( + validator.append("").is_err(), + "Should fail with missing required attribute" + ); + + // Create a new validator and try closing tag approach + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + // Try to validate an incomplete delete tag that's missing the required id attribute + let input = ""; + let result = validator.append(input); + assert!(result.is_err(), "Should fail without required id attribute"); + + // Check the error message - note the test now checks for either "required" OR "No valid continuations" + if let Err(err) = result { + println!("Error message: {}", err); + let error_str = err.to_string().to_lowercase(); + assert!( + error_str.contains("required") || error_str.contains("no valid continuations"), + "Error should mention missing required attribute or invalid continuation" + ); + } + } + + #[test] + fn test_invalid_nesting() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let validator = XmlSchemaValidator::new(&schema_text).unwrap(); + validator + .clone() + .append("") + .expect_err("Should fail because of nested tag"); + validator + .clone() + .append(" ") + .expect_err("Should fail because of nested tag, even with whitespace"); + validator + .clone() + .append("foo") + .expect_err("Should fail because of nested tag, even with text"); + validator + .clone() + .append(" foo ") + .expect_err("Should fail because of nested tag, even with text and whitespace"); + validator + .clone() + .append(" foo ") + .expect_err("Should fail because of nested tag, even with self closing tag"); + } + + #[test] + fn test_stop_self_closing() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let input = ""; + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + validator.append(input).unwrap(); + assert!(validator.eof()); + } + + #[test] + fn test_read_stdin_self_closing() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let input = ""; + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + validator.append(input).unwrap(); + assert!(validator.eof()); + } + + #[test] + fn test_single_with_attributes() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let input = r#"ls -la"#; + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + validator.append(input).unwrap(); + assert!(validator.eof()); + } + + #[test] + fn test_reasoning_with_content() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let input = + "I should explore the file system for interesting files."; + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + validator.append(input).unwrap(); + assert!(validator.eof()); + } + + #[test] + fn test_write_stdout_with_content() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let input = "Hello world!"; + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + validator.append(input).unwrap(); + assert!(validator.eof()); + } + + #[test] + fn test_create_validator() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let validator = XmlSchemaValidator::new(&schema_text); + assert!( + validator.is_ok(), + "Failed to create validator: {:?}", + validator.err() + ); + } + + #[test] + fn test_get_element_names() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let validator = XmlSchemaValidator::new(&schema_text).unwrap(); + let names = validator.get_element_names(); + assert!( + names.contains(&"reasoning".to_string()), + "Element names should contain 'reasoning', got: {:?}", + names + ); + } + + // Test basic XML fragment validation - debugging the "" failure + #[test] + fn test_append_reasoning_tag() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let validator = XmlSchemaValidator::new(&schema_text).unwrap(); + // Test character by character for precise failure point + assert!(validator.can_continue_with('<'), "Should accept '<'"); + // Clone at each step to debug exact point of failure + let mut v1 = validator.clone(); + assert!(v1.append_char('<').is_ok(), "Failed to append '<'"); + let mut v2 = v1.clone(); + assert!(v2.append_char('r').is_ok(), "Failed to append 'r'"); + // Continue with each character in "reasoning>" + let mut v3 = v2.clone(); + for c in "easoning>".chars() { + assert!(v3.append_char(c).is_ok(), "Failed to append '{}'", c); + } + // Try the whole string at once + let mut v4 = validator.clone(); + let result = v4.append(""); + assert!( + result.is_ok(), + "Failed to append '': {:?}", + result.err() + ); + } + + // Test validation of multiple fragments + #[test] + fn test_multiple_append() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + // Test sequential appends + assert!(validator.append("<").is_ok(), "Failed to append '<'"); + assert!( + validator.append("reasoning").is_ok(), + "Failed to append 'reasoning'" + ); + assert!(validator.append(">").is_ok(), "Failed to append '>'"); + assert!( + validator.append("test content").is_ok(), + "Failed to append content" + ); + assert!( + validator.append("").is_ok(), + "Failed to append closing tag" + ); + // Should be at EOF now + assert!(validator.eof(), "Should be at EOF after complete XML"); + } + + // Test token continuation - critical for the token masking issue + #[test] + fn test_can_continue_with() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + // Test at start + let validator = XmlSchemaValidator::new(&schema_text).unwrap(); + assert!( + validator.can_continue_with('<'), + "Should accept '<' at start" + ); + assert!( + !validator.can_continue_with('a'), + "Should reject 'a' at start" + ); + // Test after opening tag + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + validator.append("").unwrap(); + // Any character should be valid inside a mixed content element + assert!( + validator.can_continue_with('a'), + "Should accept 'a' inside element" + ); + assert!( + validator.can_continue_with('1'), + "Should accept '1' inside element" + ); + assert!( + validator.can_continue_with('<'), + "Should accept '<' inside element" + ); + } +} + +#[cfg(test)] +mod diagnostic_tests { + use super::*; + use std::sync::Arc; + + // Simple schema with just the reasoning tag for cleaner debugging + fn get_simple_schema() -> String { + r#" + + + + + + + + + "# + .to_string() + } + + // Test every character in an opening tag one by one + #[test] + fn test_char_by_char_open_tag() { + let schema_text = get_simple_schema(); + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + + // Test each character individually + assert!(validator.can_continue_with('<'), "Should accept '<'"); + assert!(validator.append_char('<').is_ok(), "Failed to append '<'"); + + assert!(validator.can_continue_with('r'), "Should accept 'r'"); + assert!(validator.append_char('r').is_ok(), "Failed to append 'r'"); + + for c in "easoning".chars() { + assert!(validator.can_continue_with(c), "Should accept '{}'", c); + assert!(validator.append_char(c).is_ok(), "Failed to append '{}'", c); + } + + // This is where many tests are failing - let's check the state before and after + println!("State before '>': {:?}", validator.current_tokens); + let result = validator.append_char('>'); + println!("Result of append '>': {:?}", result); + if result.is_ok() { + println!("State after '>': {:?}", validator.current_tokens); + } + assert!(result.is_ok(), "Failed to append '>'"); + } + + // Test the token transitions specifically + #[test] + fn test_token_transitions() { + // Create minimal elements for testing token transitions directly + let test_element = Arc::new(schema::XsElement { + name: "reasoning".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }); + + // Test the transition from ElementOpenName to ElementOpenEnd + let element_name = + token::ElementOpenName::new(Arc::clone(&test_element), "reasoning".to_string()); + let tokens = element_name.append('>'); + assert!(!tokens.is_empty(), "Should produce valid tokens after '>'"); + println!( + "Token after appending '>' to ElementOpenName: {:?}", + tokens[0] + ); + + // Check what type the token is + match &tokens[0] { + Token::ElementOpenEnd(element_open) => { + println!( + "Successfully transitioned to ElementOpen: {:?}", + element_open + ); + } + other => { + panic!("Expected ElementOpen, got {:?}", other); + } + } + } + + // Test element self-closing functionality + #[test] + fn test_element_self_close_transition() { + let test_element = Arc::new(schema::XsElement { + name: "stop".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: None, + text: None, + xs_attribute: None, + xs_sequence: None, + }, + }); + + // Test transition from ElementName to ElementSelfClose + let element_name = + token::ElementOpenName::new(Arc::clone(&test_element), "stop".to_string()); + let tokens = element_name.append('/'); + assert!(!tokens.is_empty(), "Should handle '/' after element name"); + + match &tokens[0] { + Token::ElementSelfClose(element_self_close) => { + println!("Successfully transitioned to ElementSelfClose"); + // Now test transition to EndOfFile + let next_tokens = element_self_close.clone().append('>'); + assert!(!next_tokens.is_empty(), "Should handle '>' after '/'"); + + match &next_tokens[0] { + Token::EndOfFile(_) => println!("Successfully transitioned to EndOfFile"), + other => panic!("Expected EndOfFile, got {:?}", other), + } + } + other => panic!("Expected ElementSelfClose, got {:?}", other), + } + } + + // Test whitespace handling + #[test] + fn test_whitespace_handling() { + let schema_text = get_simple_schema(); + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + + // Test opening tag with whitespace + for c in " + assert!( + validator.can_continue_with('>'), + "Should accept '>' after whitespace" + ); + assert!( + validator.append_char('>').is_ok(), + "Failed to append '>' after whitespace" + ); + } + + // Test attribute handling + #[test] + fn test_attribute_handling() { + let schema_text = r#" + + + + + + + "#; + + let mut validator = XmlSchemaValidator::new(schema_text).unwrap(); + + // Test opening tag with attribute + for c in "".chars() { + assert!( + validator.can_continue_with(c), + "Should accept '{}' to close tag", + c + ); + assert!( + validator.append_char(c).is_ok(), + "Failed to append '{}' to close tag", + c + ); + } + + // Check if we've reached EOF + assert!(validator.eof(), "Should be at EOF after self-closing tag"); + } + + // Test for the specific error patterns seen in the test output + #[test] + fn test_specific_error_cases() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd") + .unwrap_or_else(|_| get_simple_schema()); + + // Test case 1: Simple opening tag (noticed in `test_element_open`) + let input1 = ""; + let mut validator1 = XmlSchemaValidator::new(&schema_text).unwrap(); + let result1 = validator1.append(input1); + println!("Result appending '': {:?}", result1); + assert!(result1.is_ok(), "Failed on ''"); + + // Test case 2: Element with whitespace (noticed in `test_element_open_whitespace`) + let input2 = ""; + let mut validator2 = XmlSchemaValidator::new(&schema_text).unwrap(); + let result2 = validator2.append(input2); + println!("Result appending '': {:?}", result2); + assert!(result2.is_ok(), "Failed on ''"); + + // Test case 3: Self-closing tags (noticed in `test_read_stdin_self_closing`) + let input3 = ""; + let mut validator3 = XmlSchemaValidator::new(&schema_text).unwrap(); + let result3 = validator3.append(input3); + println!("Result appending '': {:?}", result3); + assert!(result3.is_ok(), "Failed on ''"); + + // Test case 4: Attributes (noticed in `test_single_with_attributes`) + let input4 = r#""#; + let mut validator4 = XmlSchemaValidator::new(&schema_text).unwrap(); + let result4 = validator4.append(input4); + println!("Result appending attribute: {:?}", result4); + assert!(result4.is_ok(), "Failed on attribute handling"); + } + + // Test the basic token state machine with a minimal element + #[test] + fn test_basic_token_operations() { + // Create a minimal element for testing + let element = Arc::new(schema::XsElement { + name: "test".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: None, + }, + }); + + // Create and test each token type + let start_of_file = token::StartOfFile::new(Arc::clone(&element)); + let tokens = start_of_file.append('<'); + assert!(!tokens.is_empty(), "StartOfFile should accept '<'"); + + let element_start = match &tokens[0] { + Token::ElementOpenStart(es) => es.clone(), + _ => panic!("Expected ElementStart"), + }; + + let tokens = element_start.append('t'); + assert!(!tokens.is_empty(), "ElementStart should accept 't'"); + + let element_name = match &tokens[0] { + Token::ElementOpenName(en) => en.clone(), + _ => panic!("Expected ElementName"), + }; + + // This is often where things break + println!("About to append 'e' to ElementName..."); + let tokens = element_name.append('e'); + assert!(!tokens.is_empty(), "ElementName should accept 'e'"); + + // Build the rest of the name + let mut current_token = tokens[0].clone(); + for c in "st".chars() { + let next_tokens = current_token.append(c); + assert!(!next_tokens.is_empty(), "Token should accept '{}'", c); + current_token = next_tokens[0].clone(); + } + + // Now try to append '>' + println!("About to append '>' to token: {:?}", current_token); + let tokens = current_token.append('>'); + println!("Result of appending '>': {:?}", tokens); + assert!(!tokens.is_empty(), "Token should accept '>'"); + + // Verify transition to ElementOpen + match &tokens[0] { + Token::ElementOpenEnd(_) => println!("Successfully transitioned to ElementOpen"), + other => panic!("Expected ElementOpen, got {:?}", other), + } + } + + // This test will help identify issues with token creation for specific elements + #[test] + fn test_element_specific_issues() { + if let Ok(schema_text) = std::fs::read_to_string("../../action_schema.xsd") { + let validator = XmlSchemaValidator::new(&schema_text).unwrap(); + + // Get all element names in the schema + let element_names = validator.get_element_names(); + println!("Elements in schema: {:?}", element_names); + + // Test each element separately + for name in &element_names { + println!("\nTesting element: {}", name); + + let mut element_validator = XmlSchemaValidator::new(&schema_text).unwrap(); + let open_tag = format!("<{}>", name); + + // Test character by character + println!("Testing character by character for <{}>", name); + let mut progress = String::new(); + for c in open_tag.chars() { + progress.push(c); + let result = element_validator.can_continue_with(c); + println!( + " Can continue with '{}' after '{}': {}", + c, + &progress[..progress.len() - 1], + result + ); + + if !result { + // This is helpful to diagnose where the process fails + println!(" ❌ Failed on character '{}' for element '{}'", c, name); + break; + } + + if let Err(e) = element_validator.append_char(c) { + println!( + " ❌ Failed to append '{}' for element '{}': {:?}", + c, name, e + ); + break; + } + } + + // Test complete tag at once + let mut complete_validator = XmlSchemaValidator::new(&schema_text).unwrap(); + match complete_validator.append(&open_tag) { + Ok(_) => println!(" ✅ Successfully appended <{}>", name), + Err(e) => println!(" ❌ Failed to append <{}>: {:?}", name, e), + } + } + } else { + println!("Schema file not found, skipping element-specific tests"); + } + } +} diff --git a/lib/xml_schema_validator/src/py_xml_logits_processor_core.rs b/lib/xml_schema_validator/src/py_xml_logits_processor_core.rs index cbded0e..011aa6a 100644 --- a/lib/xml_schema_validator/src/py_xml_logits_processor_core.rs +++ b/lib/xml_schema_validator/src/py_xml_logits_processor_core.rs @@ -1,6 +1,6 @@ +use crate::*; use pyo3::prelude::*; use pyo3::types::PyDict; -use crate::*; /// A minimal LogitsProcessor that enforces valid XML according to a schema. #[pyclass] @@ -18,7 +18,10 @@ impl XmlLogitsProcessorCore { Ok(xml_schema_validator) => xml_schema_validator, Err(e) => return Err(pyo3::exceptions::PyValueError::new_err(e.to_string())), }; - let tokens: Vec<(Py, String)> = tokens.iter().map(|(id, value)| (id.unbind(), value.unbind().to_string())).collect(); + let tokens: Vec<(Py, String)> = tokens + .iter() + .map(|(id, value)| (id.unbind(), value.unbind().to_string())) + .collect(); let mut trie = char_trie::CharTrie::new(); for (token_id, token_text) in tokens.iter() { trie.insert(token_text, token_id.clone_ref(py)); @@ -29,7 +32,7 @@ impl XmlLogitsProcessorCore { trie, }) } - + /// Append a fragment of XML to the validator /// Returns true if the append was successful, false otherwise fn append(&mut self, fragment: &str) -> bool { @@ -38,7 +41,9 @@ impl XmlLogitsProcessorCore { /// Get a list of tokens that would lead to invalid XML fn get_invalid_tokens(&mut self, py: Python<'_>) -> PyResult> { - Ok(self.trie.find_invalid_tokens(py, &self.xml_schema_validator, &self.tokens)) + Ok(self + .trie + .find_invalid_tokens(py, &self.xml_schema_validator, &self.tokens)) } /// Check if the validator has reached the end of the XML @@ -50,16 +55,20 @@ impl XmlLogitsProcessorCore { fn copy(&self, py: Python<'_>) -> PyResult { Ok(Self { xml_schema_validator: self.xml_schema_validator.clone(), - tokens: self.tokens.iter().map(|(a, b)|(a.clone_ref(py), b.clone())).collect(), + tokens: self + .tokens + .iter() + .map(|(a, b)| (a.clone_ref(py), b.clone())) + .collect(), trie: self.trie.clone_ref(py), }) } - + /// Get a list of all element names in the schema fn get_element_names(&self) -> PyResult> { Ok(self.xml_schema_validator.get_element_names()) } - + /// Check if a specific character is valid as the next character fn can_continue_with(&self, c: &str) -> PyResult { if let Some(first_char) = c.chars().next() { @@ -79,8 +88,8 @@ pub fn register_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> #[cfg(test)] mod tests { use super::*; - use pyo3::Python; use pyo3::types::PyDict; + use pyo3::Python; fn create_test_processor<'py>(py: Python<'py>) -> PyResult { let tokens = PyDict::new(py); @@ -91,7 +100,7 @@ mod tests { tokens.set_item(3, " ")?; tokens.set_item(4, "a")?; tokens.set_item(5, "")?; - + // Simple schema with only reasoning element let schema_text = r#" @@ -103,7 +112,7 @@ mod tests { "#; - + XmlLogitsProcessorCore::new(py, tokens.into(), schema_text) } @@ -111,24 +120,29 @@ mod tests { fn test_append_valid_xml() { Python::with_gil(|py| { let mut processor = create_test_processor(py).unwrap(); - + // Test various valid XML fragments assert!(processor.append(""), "Should accept opening tag"); - + let mut processor = create_test_processor(py).unwrap(); - assert!(processor.append("test"), - "Should accept complete element with content"); - + assert!( + processor.append("test"), + "Should accept complete element with content" + ); + // Test with real schema from disk if available if let Ok(schema_text) = std::fs::read_to_string("../../action_schema.xsd") { let tokens = PyDict::new(py); tokens.set_item(0, "<").unwrap(); tokens.set_item(1, "reasoning").unwrap(); tokens.set_item(2, ">").unwrap(); - - let mut processor = XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap(); - assert!(processor.append(""), - "Should accept with actual schema"); + + let mut processor = + XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap(); + assert!( + processor.append(""), + "Should accept with actual schema" + ); } }); } @@ -137,13 +151,18 @@ mod tests { fn test_append_invalid_xml() { Python::with_gil(|py| { let mut processor = create_test_processor(py).unwrap(); - + // Test invalid XML fragments - assert!(!processor.append(""), "Should reject invalid element"); - + assert!( + !processor.append(""), + "Should reject invalid element" + ); + let mut processor = create_test_processor(py).unwrap(); - assert!(!processor.append(""), - "Should reject mismatched tags"); + assert!( + !processor.append(""), + "Should reject mismatched tags" + ); }); } @@ -151,25 +170,36 @@ mod tests { fn test_eof_detection() { Python::with_gil(|py| { let processor = create_test_processor(py).unwrap(); - + // At start, not at EOF assert!(!processor.eof().unwrap(), "Should not be at EOF at start"); - + // After complete document, should be at EOF let mut processor = create_test_processor(py).unwrap(); // Use a properly formed XML document that matches the schema - assert!(processor.append("test"), - "Should accept complete document"); - + assert!( + processor.append("test"), + "Should accept complete document" + ); + // DEBUG: Print the token state to see why EOF detection fails - println!("Current tokens: {:?}", processor.xml_schema_validator.current_tokens); - - assert!(processor.eof().unwrap(), "Should be at EOF after complete document"); - + println!( + "Current tokens: {:?}", + processor.xml_schema_validator.current_tokens + ); + + assert!( + processor.eof().unwrap(), + "Should be at EOF after complete document" + ); + // After opening tag, should not be at EOF let mut processor = create_test_processor(py).unwrap(); assert!(processor.append(""), "Should accept opening tag"); - assert!(!processor.eof().unwrap(), "Should not be at EOF after opening tag"); + assert!( + !processor.eof().unwrap(), + "Should not be at EOF after opening tag" + ); }); } @@ -177,20 +207,30 @@ mod tests { fn test_can_continue_with() { Python::with_gil(|py| { let processor = create_test_processor(py).unwrap(); - + // At start, only '<' is valid - assert!(processor.can_continue_with("<").unwrap(), "Should accept '<' at start"); - assert!(!processor.can_continue_with("a").unwrap(), "Should reject 'a' at start"); - + assert!( + processor.can_continue_with("<").unwrap(), + "Should accept '<' at start" + ); + assert!( + !processor.can_continue_with("a").unwrap(), + "Should reject 'a' at start" + ); + // Test after opening tag let mut processor = create_test_processor(py).unwrap(); assert!(processor.append(""), "Should accept opening tag"); - + // In mixed content, any character should be valid - assert!(processor.can_continue_with("a").unwrap(), - "Should accept 'a' in mixed content"); - assert!(processor.can_continue_with("<").unwrap(), - "Should accept '<' in mixed content"); + assert!( + processor.can_continue_with("a").unwrap(), + "Should accept 'a' in mixed content" + ); + assert!( + processor.can_continue_with("<").unwrap(), + "Should accept '<' in mixed content" + ); }); } @@ -198,14 +238,17 @@ mod tests { fn test_copy() { Python::with_gil(|py| { let processor = create_test_processor(py).unwrap(); - + // Make a copy let copy = processor.copy(py).unwrap(); - + // Original and copy should behave the same - assert_eq!(processor.eof().unwrap(), copy.eof().unwrap(), - "Copy should have same EOF state"); - + assert_eq!( + processor.eof().unwrap(), + copy.eof().unwrap(), + "Copy should have same EOF state" + ); + // Modify copy, original should be unchanged let mut copy = processor.copy(py).unwrap(); assert!(copy.append(""), "Copy should accept XML"); @@ -218,28 +261,32 @@ mod tests { fn test_get_element_names() { Python::with_gil(|py| { let processor = create_test_processor(py).unwrap(); - + let names = processor.get_element_names().unwrap(); - assert!(names.contains(&"reasoning".to_string()), - "Should contain 'reasoning' element"); + assert!( + names.contains(&"reasoning".to_string()), + "Should contain 'reasoning' element" + ); assert_eq!(names.len(), 1, "Should have exactly one element"); - + let tokens = PyDict::new(py); tokens.set_item(0, "<").unwrap(); - + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); let processor = XmlLogitsProcessorCore::new(py, tokens.into(), &schema_text).unwrap(); let names = processor.get_element_names().unwrap(); - + // Print out all elements from schema for debugging println!("Elements in schema: {:?}", names); - + // Check if reasoning exists - assert!(names.contains(&"reasoning".to_string()), - "Real schema should contain 'reasoning'"); + assert!( + names.contains(&"reasoning".to_string()), + "Real schema should contain 'reasoning'" + ); }); } - + #[test] fn test_debug_reasoning_tag_validation() { Python::with_gil(|py| { @@ -254,7 +301,7 @@ mod tests { "#; - + // Create a minimal token dictionary let tokens = PyDict::new(py); tokens.set_item(0, "<").unwrap(); @@ -268,56 +315,67 @@ mod tests { tokens.set_item(8, "n").unwrap(); tokens.set_item(9, "g").unwrap(); tokens.set_item(10, ">").unwrap(); - + // Create processor with this schema and tokens - let processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap(); - + let processor = + XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap(); + // Test basic operations to verify core functionality works let element_names = processor.get_element_names().unwrap(); println!("Element names in test schema: {:?}", element_names); - assert!(element_names.contains(&"reasoning".to_string()), - "Test schema should contain 'reasoning'"); - + assert!( + element_names.contains(&"reasoning".to_string()), + "Test schema should contain 'reasoning'" + ); + // Now try to append each character of "" one by one let chars = vec!['<', 'r', 'e', 'a', 's', 'o', 'n', 'i', 'n', 'g', '>']; let mut full_text = String::new(); let processor = processor.copy(py).unwrap(); - + for c in chars { full_text.push(c); let fragment = c.to_string(); let result = processor.can_continue_with(&fragment).unwrap(); - println!("Can continue with '{}' after '{}': {}", - c, &full_text[..full_text.len()-1], result); - + println!( + "Can continue with '{}' after '{}': {}", + c, + &full_text[..full_text.len() - 1], + result + ); + // Create a new processor to test appending the fragment built so far - let mut fresh_processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap(); + let mut fresh_processor = + XmlLogitsProcessorCore::new(py, tokens.clone().into(), schema_text).unwrap(); let append_result = fresh_processor.append(&full_text); println!("Appending full text '{}': {}", full_text, append_result); } - + // Now test with actual schema from disk if available if let Ok(actual_schema) = std::fs::read_to_string("../../action_schema.xsd") { println!("\nTesting with actual schema from disk:"); - - let processor = XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap(); + + let processor = + XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap(); let element_names = processor.get_element_names().unwrap(); println!("Element names in actual schema: {:?}", element_names); - + if element_names.contains(&"reasoning".to_string()) { println!("'reasoning' element found in actual schema"); - + // Test appending "" to a fresh processor let mut processor = processor.copy(py).unwrap(); let result = processor.append(""); println!("Appending '' to fresh processor: {}", result); - + // Test character by character let mut text = String::new(); for c in "".chars() { text.push(c); - let result = XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema).unwrap() - .append(&text); + let result = + XmlLogitsProcessorCore::new(py, tokens.clone().into(), &actual_schema) + .unwrap() + .append(&text); println!("Appending '{}' to fresh processor: {}", text, result); } } else { @@ -326,4 +384,4 @@ mod tests { } }); } -} \ No newline at end of file +} diff --git a/lib/xml_schema_validator/src/py_xml_schema_validator.rs b/lib/xml_schema_validator/src/py_xml_schema_validator.rs index 1cc8c31..05150e4 100644 --- a/lib/xml_schema_validator/src/py_xml_schema_validator.rs +++ b/lib/xml_schema_validator/src/py_xml_schema_validator.rs @@ -1,71 +1,71 @@ -use pyo3::prelude::*; -use pyo3::exceptions::PyValueError; - -/// Python wrapper for XmlValidator -#[pyclass] -struct XmlSchemaValidator { - validator: crate::XmlSchemaValidator, -} - -#[pymethods] -impl XmlSchemaValidator { - #[new] - fn new(schema_text: &str) -> PyResult { - let validator = crate::XmlSchemaValidator::new(schema_text) - .map_err(|e| PyValueError::new_err(format!("Failed to create validator: {}", e)))?; - - Ok(Self { validator }) - } - - /// Append a fragment of XML to the validator - /// Returns a tuple containing: - /// - A boolean indicating success - /// - An optional error message - fn append(&mut self, fragment: &str) -> PyResult<(bool, Option)> { - match self.validator.append(fragment) { - Ok(()) => PyResult::Ok((true, None)), - Err(e) => PyResult::Ok((false, Some(e.to_string()))), - } - } - - /// Check if the validator has reached the end of the XML - fn eof(&self) -> PyResult { - Ok(self.validator.eof()) - } - - /// Create a deep copy of this validator - fn copy(&self) -> PyResult { - Ok(Self { - validator: self.validator.clone(), - }) - } - - /// Get a list of all element names in the schema - fn get_element_names(&self) -> PyResult> { - Ok(self.validator.get_element_names()) - } - - /// Check if a specific character is valid as the next character - fn can_continue_with(&self, c: &str) -> PyResult { - if let Some(first_char) = c.chars().next() { - Ok(self.validator.can_continue_with(first_char)) - } else { - Ok(false) - } - } - - /// Validate an entire XML string and check if it's valid according to the schema - fn validate(&self, xml_string: &str) -> PyResult { - let mut validator_clone = self.validator.clone(); - match validator_clone.append(xml_string) { - Ok(()) => Ok(validator_clone.eof()), // Valid only if we reached EOF - Err(_) => Ok(false), // Invalid XML - } - } -} - -/// Register the class with the Python module -pub fn register_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_class::()?; - Ok(()) -} \ No newline at end of file +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; + +/// Python wrapper for XmlValidator +#[pyclass] +struct XmlSchemaValidator { + validator: crate::XmlSchemaValidator, +} + +#[pymethods] +impl XmlSchemaValidator { + #[new] + fn new(schema_text: &str) -> PyResult { + let validator = crate::XmlSchemaValidator::new(schema_text) + .map_err(|e| PyValueError::new_err(format!("Failed to create validator: {}", e)))?; + + Ok(Self { validator }) + } + + /// Append a fragment of XML to the validator + /// Returns a tuple containing: + /// - A boolean indicating success + /// - An optional error message + fn append(&mut self, fragment: &str) -> PyResult<(bool, Option)> { + match self.validator.append(fragment) { + Ok(()) => PyResult::Ok((true, None)), + Err(e) => PyResult::Ok((false, Some(e.to_string()))), + } + } + + /// Check if the validator has reached the end of the XML + fn eof(&self) -> PyResult { + Ok(self.validator.eof()) + } + + /// Create a deep copy of this validator + fn copy(&self) -> PyResult { + Ok(Self { + validator: self.validator.clone(), + }) + } + + /// Get a list of all element names in the schema + fn get_element_names(&self) -> PyResult> { + Ok(self.validator.get_element_names()) + } + + /// Check if a specific character is valid as the next character + fn can_continue_with(&self, c: &str) -> PyResult { + if let Some(first_char) = c.chars().next() { + Ok(self.validator.can_continue_with(first_char)) + } else { + Ok(false) + } + } + + /// Validate an entire XML string and check if it's valid according to the schema + fn validate(&self, xml_string: &str) -> PyResult { + let mut validator_clone = self.validator.clone(); + match validator_clone.append(xml_string) { + Ok(()) => Ok(validator_clone.eof()), // Valid only if we reached EOF + Err(_) => Ok(false), // Invalid XML + } + } +} + +/// Register the class with the Python module +pub fn register_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + Ok(()) +} diff --git a/lib/xml_schema_validator/src/python.rs b/lib/xml_schema_validator/src/python.rs index dafeb47..86fb367 100644 --- a/lib/xml_schema_validator/src/python.rs +++ b/lib/xml_schema_validator/src/python.rs @@ -1,9 +1,9 @@ -use pyo3::prelude::*; - -/// Python module for XML Schema validation -#[pymodule] -fn _rs(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - crate::py_xml_logits_processor_core::register_module(py, m)?; - crate::py_xml_schema_validator::register_module(py, m)?; - Ok(()) -} \ No newline at end of file +use pyo3::prelude::*; + +/// Python module for XML Schema validation +#[pymodule] +fn _rs(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + crate::py_xml_logits_processor_core::register_module(py, m)?; + crate::py_xml_schema_validator::register_module(py, m)?; + Ok(()) +} diff --git a/lib/xml_schema_validator/src/schema.rs b/lib/xml_schema_validator/src/schema.rs index d026cdf..f4ef89f 100644 --- a/lib/xml_schema_validator/src/schema.rs +++ b/lib/xml_schema_validator/src/schema.rs @@ -1,6 +1,14 @@ use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive( + Clone, + Debug, + Deserialize, + Eq, + Hash, + PartialEq, + Serialize, +)] pub struct XsSchema { #[serde(rename = "@xmlns:xs")] pub xmlns_xs: String, @@ -12,7 +20,15 @@ pub struct XsSchema { pub xs_element: Vec, } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive( + Clone, + Debug, + Deserialize, + Eq, + Hash, + PartialEq, + Serialize, +)] pub struct XsElement { #[serde(rename = "@name")] pub name: String, @@ -22,7 +38,15 @@ pub struct XsElement { pub xs_complex_type: XsComplexType, } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive( + Clone, + Debug, + Deserialize, + Eq, + Hash, + PartialEq, + Serialize, +)] pub struct XsComplexType { #[serde(rename = "@mixed")] pub mixed: Option, @@ -34,7 +58,15 @@ pub struct XsComplexType { pub xs_sequence: Option, } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive( + Clone, + Debug, + Deserialize, + Eq, + Hash, + PartialEq, + Serialize, +)] pub struct XsAttribute { #[serde(rename = "@name")] pub name: String, @@ -44,7 +76,15 @@ pub struct XsAttribute { pub xs_attribute_use: String, } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive( + Clone, + Debug, + Deserialize, + Eq, + Hash, + PartialEq, + Serialize, +)] pub struct XsSequence { #[serde(rename = "$text")] pub text: Option, @@ -52,7 +92,15 @@ pub struct XsSequence { pub xs_any: XsAny, } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive( + Clone, + Debug, + Deserialize, + Eq, + Hash, + PartialEq, + Serialize, +)] pub struct XsAny { #[serde(rename = "@minOccurs")] pub min_occurs: String, @@ -61,4 +109,3 @@ pub struct XsAny { #[serde(rename = "@processContents")] pub process_contents: String, } - diff --git a/lib/xml_schema_validator/src/token/attribute_equals.rs b/lib/xml_schema_validator/src/token/attribute_equals.rs new file mode 100644 index 0000000..7ab5996 --- /dev/null +++ b/lib/xml_schema_validator/src/token/attribute_equals.rs @@ -0,0 +1,102 @@ +use crate::*; +use std::sync::Arc; + +/// Represents the equals sign after an XML attribute name +#[derive(Clone, Debug)] +pub struct AttributeEquals { + element: Arc, + used_attributes: Vec, +} + +impl AttributeEquals { + pub fn new( + element: Arc, + used_attributes: Vec, + ) -> Self { + Self { + element, + used_attributes, + } + } + + pub fn append(self, c: char) -> Vec { + if c == '"' { + // Opening quote starts the attribute value + return vec![Token::AttributeValue(AttributeValue::new( + Arc::clone(&self.element), + self.used_attributes, + ))]; + } else if c.is_whitespace() { + // Allow whitespace between equals sign and opening quote + return vec![Token::AttributeEquals(self)]; + } else { + // Any other character is invalid + return vec![]; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_attribute_equals_quote() { + let values = [ + ", - /// Buffer containing the attribute name characters processed so far - buffer: String, - /// Track which attributes have already been set for this element - attributes_set: Vec, -} - -impl AttributeName { - /// Create a new AttributeName with an empty buffer - pub fn new(element: Arc) -> Self { - Self { - element, - buffer: String::new(), - attributes_set: Vec::new(), - } - } - - /// Create a new AttributeName with the given buffer - pub fn with_buffer(element: Arc, buffer: String, attributes_set: Vec) -> Self { - Self { - element, - buffer, - attributes_set, - } - } - - /// Create a new AttributeName with attributes already set - pub fn with_attributes(element: Arc, attributes_set: Vec) -> Self { - Self { - element, - buffer: String::new(), - attributes_set, - } - } - - /// Append a character to the attribute name and return possible continuations - pub fn append(self, c: char) -> Vec { - if c.is_whitespace() && self.buffer.is_empty() { - return vec![Token::AttributeName(self)]; - } else if c == '=' { - // Found the equals sign, transition to attribute value - // We need to verify that this attribute name is valid for this element - if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { - for attr in attributes { - if attr.name == self.buffer { - // Valid attribute, transition to attribute value - return vec![Token::AttributeValue(AttributeValue::new( - Arc::clone(&self.element), - self.buffer, - self.attributes_set, - ))]; - } - } - } - // Invalid attribute name - vec![] - } else { - // Continue building the attribute name - let new_buffer = self.buffer + &c.to_string(); - - // Check if this could be a valid attribute for this element - if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { - for attr in attributes { - if attr.name.starts_with(&new_buffer) { - // This could be a valid attribute, continue parsing - return vec![Token::AttributeName(AttributeName::with_buffer( - Arc::clone(&self.element), - new_buffer, - self.attributes_set.clone(), - ))]; - } - } - } - - // No matching attribute found - vec![] - } - } -} - -#[cfg(test)] -mod attribute_name_tests { - use super::*; - use std::sync::Arc; - use crate::schema; - - fn create_element_with_attributes() -> Arc { - let element = schema::XsElement { - name: "delete".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: None, - text: None, - xs_attribute: Some(vec![ - schema::XsAttribute { - name: "id".to_string(), - xs_attribute_type: "xs:string".to_string(), - xs_attribute_use: "required".to_string(), - } - ]), - xs_sequence: None, - }, - }; - Arc::new(element) - } - - #[test] - fn test_attribute_name_append() { - let element = create_element_with_attributes(); - let token = AttributeName::new(Arc::clone(&element)); - - // Build attribute name character by character - let mut current_token = token; - for c in "id".chars() { - let next_tokens = current_token.append(c); - assert!(!next_tokens.is_empty(), "Should accept attribute name character '{}'", c); - - match &next_tokens[0] { - Token::AttributeName(next) => current_token = next.clone(), - _ => panic!("Expected AttributeName, got {:?}", next_tokens[0]), - } - } - - // Test equals sign - let next_tokens = current_token.append('='); - assert!(!next_tokens.is_empty(), "Should accept '=' after attribute name"); - - match &next_tokens[0] { - Token::AttributeValue(_) => (), - _ => panic!("Expected AttributeValue, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_attribute_name_whitespace() { - let element = create_element_with_attributes(); - let token = AttributeName::new(Arc::clone(&element)); - - // Test with whitespace - let next_tokens = token.append(' '); - assert!(!next_tokens.is_empty(), "Should accept whitespace in attribute name"); - - match &next_tokens[0] { - Token::AttributeName(_) => (), - _ => panic!("Expected AttributeName, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_invalid_attribute_name() { - let element = create_element_with_attributes(); - let token = AttributeName::new(Arc::clone(&element)); - - // Test with invalid attribute name - let next_tokens = token.append('x'); // 'x' doesn't start any valid attribute - assert!(next_tokens.is_empty(), "Should reject invalid attribute name"); - } - - #[test] - fn test_attribute_tracking() { - let element = create_element_with_attributes(); - - // Create AttributeName with pre-existing attributes - let attributes_set = vec!["other".to_string()]; - let token = AttributeName::with_attributes(Arc::clone(&element), attributes_set.clone()); - - // Build attribute name and transition to AttributeValue - let mut current_token = token; - for c in "id".chars() { - let next_tokens = current_token.append(c); - match &next_tokens[0] { - Token::AttributeName(next) => current_token = next.clone(), - _ => panic!("Expected AttributeName, got {:?}", next_tokens[0]), - } - } - - // Test equals sign - let next_tokens = current_token.append('='); - - match &next_tokens[0] { - Token::AttributeValue(value) => { - // Verify that pre-existing attributes are preserved - assert!(value.attributes_set.contains(&"other".to_string()), - "Pre-existing attribute should be preserved"); - }, - _ => panic!("Expected AttributeValue, got {:?}", next_tokens[0]), - } - } +use crate::*; +use std::sync::Arc; + +/// Represents an XML attribute name that is in the process of being parsed +#[derive(Clone, Debug)] +pub struct AttributeName { + buffer: String, + element: Arc, + used_attributes: Vec, +} + +impl AttributeName { + pub fn new(first_char: char, element: Arc) -> Self { + Self { + buffer: first_char.to_string(), + element, + used_attributes: Vec::new(), + } + } + + pub fn new_with_used(first_char: char, element: Arc, used_attributes: Vec) -> Self { + Self { + buffer: first_char.to_string(), + element, + used_attributes, + } + } + + pub fn append(mut self, c: char) -> Vec { + if c == '=' { + // Check if we've matched a valid attribute name + if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { + if let Some(attribute) = attributes + .iter() + .find(|attribute| attribute.name == self.buffer) + { + // Check if this attribute has already been used + if self.used_attributes.contains(&attribute) { + return vec![]; + } + + // Add this attribute to the used set + self.used_attributes.push(attribute.clone()); + + return vec![Token::AttributeEquals(AttributeEquals::new( + Arc::clone(&self.element), + self.used_attributes, + ))]; + } + } + return vec![]; + } else if c.is_whitespace() { + // Check if we've matched a valid attribute name and allow whitespace after it + if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { + if attributes + .iter() + .any(|attribute| attribute.name == self.buffer) + { + // Check if this attribute has already been used + if self.used_attributes.iter().filter(|attribute| attribute.name == self.buffer).next().is_some() { + return vec![]; + } + + return vec![Token::AttributeName(self)]; + } + } + return vec![]; + } else { + // Add character to buffer and check if we're still building a valid attribute name + let new_buffer = self.buffer.clone() + &c.to_string(); + + if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { + // Check if any attribute name starts with our buffer + if attributes + .iter() + .any(|attribute| attribute.name.starts_with(&new_buffer)) + { + return vec![Token::AttributeName(AttributeName { + buffer: new_buffer, + element: Arc::clone(&self.element), + used_attributes: self.used_attributes, + })]; + } + } + + // Character doesn't match what we expect for this attribute name + return vec![]; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_valid_attribute_name() { + let values = [ + ", - /// The name of the attribute being processed - attribute_name: String, - /// Buffer containing the attribute value characters processed so far - buffer: String, - /// Whether we've encountered the opening quote - in_quotes: bool, - /// Whether we've seen the closing quote - closed: bool, - /// Track which attributes have already been set for this element - pub attributes_set: Vec, -} - -impl AttributeValue { - /// Create a new AttributeValue with a list of attributes already set - pub fn new(element: Arc, attribute_name: String, attributes_set: Vec) -> Self { - Self { - element, - attribute_name, - buffer: String::new(), - in_quotes: false, - closed: false, - attributes_set, - } - } - - /// Append a character to the attribute value and return possible continuations - pub fn append(self, c: char) -> Vec { - if self.closed { - // We've already closed the attribute value with a quote - - // Add this attribute to the list of attributes set - let mut new_attributes = self.attributes_set.clone(); - if !new_attributes.contains(&self.attribute_name) { - new_attributes.push(self.attribute_name.clone()); - } - - if c.is_whitespace() { - // Check if there are more attributes to parse - if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { - // If there are other required attributes that haven't been set yet - for attr in attributes { - if attr.name != self.attribute_name && attr.xs_attribute_use == "required" && !new_attributes.contains(&attr.name) { - return vec![Token::AttributeName(AttributeName::with_attributes( - Arc::clone(&self.element), - new_attributes, - ))]; - } - } - } - - // No more required attributes, whitespace after attribute value, continue to next state - return vec![ - Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes.clone())), - Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes)) - ]; - } else if c == '>' { - // End of opening tag - return vec![Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes))]; - } else if c == '/' { - // Beginning of self-closing tag - return vec![Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))]; - } else { - // Unexpected character after attribute value - return vec![]; - } - } - - if !self.in_quotes { - if c.is_whitespace() { - // Skip whitespace before the quotes - return vec![Token::AttributeValue(Self { - element: Arc::clone(&self.element), - attribute_name: self.attribute_name, - buffer: self.buffer, - in_quotes: false, - closed: false, - attributes_set: self.attributes_set, - })]; - } else if c == '"' || c == '\'' { - // Start of quotes - return vec![Token::AttributeValue(Self { - element: Arc::clone(&self.element), - attribute_name: self.attribute_name, - buffer: self.buffer, - in_quotes: true, - closed: false, - attributes_set: self.attributes_set, - })]; - } else { - // Unexpected character before quotes - return vec![]; - } - } else { - // We're inside quotes - if c == '"' || c == '\'' { - // End of quotes, validate the attribute value - // In a real implementation, we'd check if the value is valid for this attribute type - return vec![Token::AttributeValue(Self { - element: Arc::clone(&self.element), - attribute_name: self.attribute_name, - buffer: self.buffer, - in_quotes: true, - closed: true, - attributes_set: self.attributes_set, - })]; - } else { - // Continue building the attribute value - let new_buffer = self.buffer + &c.to_string(); - return vec![Token::AttributeValue(Self { - element: Arc::clone(&self.element), - attribute_name: self.attribute_name, - buffer: new_buffer, - in_quotes: true, - closed: false, - attributes_set: self.attributes_set, - })]; - } - } - } -} - -#[cfg(test)] -mod attribute_value_tests { - use super::*; - use std::sync::Arc; - use crate::schema; - - fn create_element_with_attributes() -> Arc { - let element = schema::XsElement { - name: "delete".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: None, - text: None, - xs_attribute: Some(vec![ - schema::XsAttribute { - name: "id".to_string(), - xs_attribute_type: "xs:string".to_string(), - xs_attribute_use: "required".to_string(), - } - ]), - xs_sequence: None, - }, - }; - Arc::new(element) - } - - #[test] - fn test_attribute_value_quotes() { - let element = create_element_with_attributes(); - let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]); - - // Test with opening quote - let next_tokens = token.append('"'); - assert!(!next_tokens.is_empty(), "Should accept opening quote"); - - let token = next_tokens[0].clone(); - match token.clone() { - Token::AttributeValue(value) => { - assert!(value.in_quotes, "Should mark as in quotes"); - assert!(!value.closed, "Should not be closed yet"); - }, - _ => panic!("Expected AttributeValue, got {:?}", token), - } - - // Test with content inside quotes - let next_tokens = token.append('a'); - assert!(!next_tokens.is_empty(), "Should accept content inside quotes"); - - let token = next_tokens[0].clone(); - - // Test with closing quote - let next_tokens = token.append('"'); - assert!(!next_tokens.is_empty(), "Should accept closing quote"); - - let token = next_tokens[0].clone(); - match token { - Token::AttributeValue(value) => { - assert!(value.closed, "Should be closed"); - }, - _ => panic!("Expected AttributeValue, got {:?}", token), - } - } - - #[test] - fn test_after_attribute_value_closed() { - let element = create_element_with_attributes(); - let mut token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]); - token.in_quotes = true; - token.closed = true; - - // Test with space after closed attribute value - let next_tokens = token.append(' '); - assert!(!next_tokens.is_empty(), "Should accept space after closed attribute"); - - // Test with > after closed attribute value - let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]); - let mut token = token; - token.in_quotes = true; - token.closed = true; - - let next_tokens = token.append('>'); - assert!(!next_tokens.is_empty(), "Should accept '>' after closed attribute"); - - match &next_tokens[0] { - Token::ElementOpenEnd(_) => (), - _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]), - } - - // Test with / after closed attribute value (for self-closing) - let token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]); - let mut token = token; - token.in_quotes = true; - token.closed = true; - - let next_tokens = token.append('/'); - assert!(!next_tokens.is_empty(), "Should accept '/' after closed attribute"); - - match &next_tokens[0] { - Token::ElementSelfClose(_) => (), - _ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_attribute_tracking() { - let element = create_element_with_attributes(); - let mut token = AttributeValue::new(Arc::clone(&element), "id".to_string(), vec![]); - token.in_quotes = true; - token.closed = true; - - // After attribute is closed, it should be tracked - let next_tokens = token.append('>'); - assert!(!next_tokens.is_empty(), "Should accept '>' after attribute"); - - match &next_tokens[0] { - Token::ElementOpenEnd(element_open) => { - // Verify that the id attribute is tracked - assert!(element_open.attributes_set.contains(&"id".to_string()), - "Attribute 'id' should be tracked"); - }, - _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]), - } - } +use crate::*; + +/// Represents an attribute value of an XML element +#[derive(Clone, Debug)] +pub struct AttributeValue { + /// The element that owns this attribute + element: Arc, + /// Buffer containing the attribute value characters processed so far + buffer: String, + /// Track which attributes have already been set for this element + pub attributes_set: Vec, +} + +impl AttributeValue { + /// Create a new AttributeValue with a list of attributes already set + pub fn new( + element: Arc, + attributes_set: Vec, + ) -> Self { + Self { + element, + buffer: "\"".to_string(), + attributes_set, + } + } + + /// Append a character to the attribute value and return possible continuations + pub fn append(mut self, c: char) -> Vec { + // Handle escaped characters + if Self::in_escape_sequence(&self.buffer) { + self.buffer.push(c); + return vec![Token::AttributeValue(self)]; + } else if self.value_closed() { + return self.next_tokens(c); + } else if '"' == c { + self.buffer.push(c); + return vec![Token::AttributeValue(self)]; + } else { + self.buffer.push(c); + return vec![Token::AttributeValue(self)]; + } + } + + /// Checks if an XML character entity reference is in progress and correctly terminated + /// Returns true if the sequence is complete (ends with a semicolon) + pub fn in_escape_sequence(buffer: &str) -> bool { + if let Some(amp_pos) = buffer.rfind('&') { + if buffer[amp_pos..].find(';').is_some() { + return false; + } else { + return true; + } + } + false + } + + fn value_closed(&self) -> bool { + if self.buffer.len() > 1 && self.buffer.trim().ends_with("\"") { + let trimmed = self.buffer.trim(); + let trimmed = &trimmed[..trimmed.len() - 2]; + if Self::in_escape_sequence(trimmed) { + false + } else { + true + } + } else { + false + } + } + + fn next_tokens(mut self, c: char) -> Vec { + let all_required_attributes_set: bool = self + .element + .as_ref() + .xs_complex_type + .xs_attribute + .as_deref() + .unwrap() + .iter() + .all(|attr| { + if attr.xs_attribute_use == "required" { + self.attributes_set.contains(attr) + } else { + true + } + }); + let all_attributes_set: bool = self + .element + .as_ref() + .xs_complex_type + .xs_attribute + .as_deref() + .unwrap() + .iter() + .all(|attr| self.attributes_set.contains(attr)); + if c.is_whitespace() { + self.buffer.push(c); + vec![Token::AttributeValue(self)] + } else { + let mut tokens = vec![]; + if all_required_attributes_set { + if '/' == c { + tokens.push(Token::ElementSelfClose(ElementSelfClose::new())); + } else if '>' == c { + tokens.push(Token::ElementOpenEnd(ElementOpenEnd::new(self.element.clone()))); + } + } + if self.buffer.ends_with(char::is_whitespace) && !all_attributes_set { + tokens.push(Token::AttributeName(AttributeName::new_with_used( + c, + self.element, + self.attributes_set, + ))); + } + tokens + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_unfinished_attribute_value() { + let values = [ + ", - /// Current buffer of characters processed for the element name - buffer: String, -} - -impl ElementCloseName { - /// Create a new instance with the given buffer - pub fn new(element: Arc, buffer: String) -> Self { - Self { - element, - buffer, - } - } - - pub fn append(self, c: char) -> Vec { - if self.element.name == self.buffer && c.is_whitespace() { - return vec![Token::ElementCloseName(self)]; - } - if self.element.name == self.buffer && c == '>' { - return vec![Token::EndOfFile(EndOfFile::new())]; - } - let new_buffer = self.buffer + &c.to_string(); - if self.element.name.starts_with(&new_buffer) { - return vec![Token::ElementCloseName(ElementCloseName::new( - Arc::clone(&self.element), - new_buffer, - ))]; - } else { - vec![] - } - } -} - -#[cfg(test)] -mod element_close_name_tests { - use super::*; - use std::sync::Arc; - use crate::schema; - - fn create_test_element() -> Arc { - let element = schema::XsElement { - name: "reasoning".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: Some("true".to_string()), - text: None, - xs_attribute: None, - xs_sequence: Some(schema::XsSequence { - text: None, - xs_any: schema::XsAny { - min_occurs: "0".to_string(), - max_occurs: "unbounded".to_string(), - process_contents: "skip".to_string(), - }, - }), - }, - }; - Arc::new(element) - } - - #[test] - fn test_close_name_partial() { - let element = create_test_element(); - let token = ElementCloseName::new(Arc::clone(&element), "r".to_string()); - - // Test with next character in name - let next_tokens = token.append('e'); - assert!(!next_tokens.is_empty(), "Should accept next character in name"); - - match &next_tokens[0] { - Token::ElementCloseName(name) => { - assert_eq!(name.buffer, "re", "Buffer should contain 're'"); - }, - _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_close_name_complete() { - let element = create_test_element(); - let token = ElementCloseName::new(Arc::clone(&element), "reasonin".to_string()); - - // Test with last character of name - let next_tokens = token.append('g'); - assert!(!next_tokens.is_empty(), "Should accept last character of name"); - - match &next_tokens[0] { - Token::ElementCloseName(name) => { - assert_eq!(name.buffer, "reasoning", "Buffer should contain 'reasoning'"); - }, - _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]), - } - - // Test with > after complete name - let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string()); - let next_tokens = token.append('>'); - assert!(!next_tokens.is_empty(), "Should accept '>' after complete name"); - - match &next_tokens[0] { - Token::EndOfFile(_) => (), - _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_close_name_whitespace() { - let element = create_test_element(); - let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string()); - - // Test with whitespace after complete name - let next_tokens = token.append(' '); - assert!(!next_tokens.is_empty(), "Should accept whitespace after complete name"); - - match &next_tokens[0] { - Token::ElementCloseName(_) => (), - _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_close_name_invalid() { - let element = create_test_element(); - let token = ElementCloseName::new(Arc::clone(&element), "r".to_string()); - - // Test with invalid next character - let next_tokens = token.append('x'); - assert!(next_tokens.is_empty(), "Should reject invalid character"); - } -} \ No newline at end of file +use crate::*; +use std::sync::Arc; + +/// Represents a closing element tag that is in the process of being parsed +/// after the ', + /// Current buffer of characters processed for the element name + buffer: String, +} + +impl ElementCloseName { + /// Create a new instance with the given buffer + pub fn new(element: Arc, buffer: String) -> Self { + Self { element, buffer } + } + + pub fn append(self, c: char) -> Vec { + if self.element.name == self.buffer && c.is_whitespace() { + return vec![Token::ElementCloseName(self)]; + } + if self.element.name == self.buffer && c == '>' { + return vec![Token::EndOfFile(EndOfFile::new())]; + } + let new_buffer = self.buffer + &c.to_string(); + if self.element.name.starts_with(&new_buffer) { + return vec![Token::ElementCloseName(ElementCloseName::new( + Arc::clone(&self.element), + new_buffer, + ))]; + } else { + vec![] + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::schema; + use std::sync::Arc; + + fn create_test_element() -> Arc { + let element = schema::XsElement { + name: "reasoning".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }; + Arc::new(element) + } + + #[test] + fn test_close_name_partial() { + let element = create_test_element(); + let token = ElementCloseName::new(Arc::clone(&element), "r".to_string()); + + // Test with next character in name + let next_tokens = token.append('e'); + assert!( + !next_tokens.is_empty(), + "Should accept next character in name" + ); + + match &next_tokens[0] { + Token::ElementCloseName(name) => { + assert_eq!(name.buffer, "re", "Buffer should contain 're'"); + } + _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_close_name_complete() { + let element = create_test_element(); + let token = ElementCloseName::new(Arc::clone(&element), "reasonin".to_string()); + + // Test with last character of name + let next_tokens = token.append('g'); + assert!( + !next_tokens.is_empty(), + "Should accept last character of name" + ); + + match &next_tokens[0] { + Token::ElementCloseName(name) => { + assert_eq!( + name.buffer, "reasoning", + "Buffer should contain 'reasoning'" + ); + } + _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]), + } + + // Test with > after complete name + let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string()); + let next_tokens = token.append('>'); + assert!( + !next_tokens.is_empty(), + "Should accept '>' after complete name" + ); + + match &next_tokens[0] { + Token::EndOfFile(_) => (), + _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_close_name_whitespace() { + let element = create_test_element(); + let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string()); + + // Test with whitespace after complete name + let next_tokens = token.append(' '); + assert!( + !next_tokens.is_empty(), + "Should accept whitespace after complete name" + ); + + match &next_tokens[0] { + Token::ElementCloseName(_) => (), + _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_close_name_invalid() { + let element = create_test_element(); + let token = ElementCloseName::new(Arc::clone(&element), "r".to_string()); + + // Test with invalid next character + let next_tokens = token.append('x'); + assert!(next_tokens.is_empty(), "Should reject invalid character"); + } +} diff --git a/lib/xml_schema_validator/src/token/element_close_start.rs b/lib/xml_schema_validator/src/token/element_close_start.rs index 01828f9..e51c2ab 100644 --- a/lib/xml_schema_validator/src/token/element_close_start.rs +++ b/lib/xml_schema_validator/src/token/element_close_start.rs @@ -1,107 +1,110 @@ -use std::sync::Arc; -use crate::*; - -/// Represents the start of an XML element closing tag (the ', - has_slash: bool, -} - -impl ElementCloseStart { - pub fn new(element: Arc) -> Self { - Self { - element, - has_slash: false, - } - } - - pub fn append(self, c: char) -> Vec { - if !self.has_slash && '/' == c { - vec![Token::ElementCloseStart(Self { - element: Arc::clone(&self.element), - has_slash: true, - })] - } else if self.has_slash && self.element.name.starts_with(c) { - vec![Token::ElementCloseName(ElementCloseName::new( - Arc::clone(&self.element), - c.to_string(), - ))] - } else { - vec![] - } - } -} - -#[cfg(test)] -mod element_close_start_tests { - use super::*; - use std::sync::Arc; - use crate::schema; - - fn create_test_element() -> Arc { - let element = schema::XsElement { - name: "reasoning".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: Some("true".to_string()), - text: None, - xs_attribute: None, - xs_sequence: Some(schema::XsSequence { - text: None, - xs_any: schema::XsAny { - min_occurs: "0".to_string(), - max_occurs: "unbounded".to_string(), - process_contents: "skip".to_string(), - }, - }), - }, - }; - Arc::new(element) - } - - #[test] - fn test_close_start_slash() { - let element = create_test_element(); - let token = ElementCloseStart::new(Arc::clone(&element)); - - // Test with slash - let next_tokens = token.append('/'); - assert!(!next_tokens.is_empty(), "Should accept '/'"); - - match &next_tokens[0] { - Token::ElementCloseStart(start) => { - assert!(start.has_slash, "Should mark as having slash"); - }, - _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_close_start_element_name() { - let element = create_test_element(); - let mut token = ElementCloseStart::new(Arc::clone(&element)); - token.has_slash = true; - - // Test with first character of element name - let next_tokens = token.append('r'); - assert!(!next_tokens.is_empty(), "Should accept first letter of element name"); - - match &next_tokens[0] { - Token::ElementCloseName(_) => (), - _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_close_start_invalid_name() { - let element = create_test_element(); - let mut token = ElementCloseStart::new(Arc::clone(&element)); - token.has_slash = true; - - // Test with invalid first character of element name - let next_tokens = token.append('x'); - assert!(next_tokens.is_empty(), "Should reject invalid first letter"); - } -} \ No newline at end of file +use crate::*; +use std::sync::Arc; + +/// Represents the start of an XML element closing tag (the ', + has_slash: bool, +} + +impl ElementCloseStart { + pub fn new(element: Arc) -> Self { + Self { + element, + has_slash: false, + } + } + + pub fn append(self, c: char) -> Vec { + if !self.has_slash && '/' == c { + vec![Token::ElementCloseStart(Self { + element: Arc::clone(&self.element), + has_slash: true, + })] + } else if self.has_slash && self.element.name.starts_with(c) { + vec![Token::ElementCloseName(ElementCloseName::new( + Arc::clone(&self.element), + c.to_string(), + ))] + } else { + vec![] + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::schema; + use std::sync::Arc; + + fn create_test_element() -> Arc { + let element = schema::XsElement { + name: "reasoning".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }; + Arc::new(element) + } + + #[test] + fn test_close_start_slash() { + let element = create_test_element(); + let token = ElementCloseStart::new(Arc::clone(&element)); + + // Test with slash + let next_tokens = token.append('/'); + assert!(!next_tokens.is_empty(), "Should accept '/'"); + + match &next_tokens[0] { + Token::ElementCloseStart(start) => { + assert!(start.has_slash, "Should mark as having slash"); + } + _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_close_start_element_name() { + let element = create_test_element(); + let mut token = ElementCloseStart::new(Arc::clone(&element)); + token.has_slash = true; + + // Test with first character of element name + let next_tokens = token.append('r'); + assert!( + !next_tokens.is_empty(), + "Should accept first letter of element name" + ); + + match &next_tokens[0] { + Token::ElementCloseName(_) => (), + _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_close_start_invalid_name() { + let element = create_test_element(); + let mut token = ElementCloseStart::new(Arc::clone(&element)); + token.has_slash = true; + + // Test with invalid first character of element name + let next_tokens = token.append('x'); + assert!(next_tokens.is_empty(), "Should reject invalid first letter"); + } +} diff --git a/lib/xml_schema_validator/src/token/element_open_end.rs b/lib/xml_schema_validator/src/token/element_open_end.rs index 5ed07b9..fdab91b 100644 --- a/lib/xml_schema_validator/src/token/element_open_end.rs +++ b/lib/xml_schema_validator/src/token/element_open_end.rs @@ -1,220 +1,163 @@ -use std::sync::Arc; -use crate::*; - -/// Represents a fully opened XML element, containing the schema element reference -#[derive(Clone, Debug)] -pub struct ElementOpenEnd { - element: Arc, - // Track which attributes have been set for this element - pub attributes_set: Vec, -} - -impl ElementOpenEnd { - pub fn new(element: Arc) -> Self { - Self { - element, - attributes_set: Vec::new(), - } - } - - pub fn with_attributes(element: Arc, attributes_set: Vec) -> Self { - Self { - element, - attributes_set, - } - } - - pub fn append(self, c: char) -> Vec { - // Check if all required attributes are present before proceeding - if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { - for attr in attributes { - if attr.xs_attribute_use == "required" && !self.attributes_set.contains(&attr.name) { - // Missing a required attribute, this is an error - return vec![]; - } - } - } - - // Check if the element is a mixed content element that allows text - let allows_text = self.element.xs_complex_type.mixed - .as_ref() - .map(|mixed| mixed == "true") - .unwrap_or(false); - - if allows_text { - if c.is_whitespace() { - // Handle whitespace in mixed content - vec![ - Token::TextContent(TextContent::new(Arc::clone(&self.element))), - Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element))) - ] - } else if c == '<' { - // Start of a closing tag or nested element - vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))] - } else { - // Text content - vec![Token::TextContent(TextContent::new(Arc::clone(&self.element)))] - } - } else { - // Element doesn't allow text content, only expect closing tag - if c == '<' { - vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))] - } else if c.is_whitespace() { - vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))] - } else { - // Unexpected content in non-mixed element - vec![] - } - } - } -} - -#[cfg(test)] -mod element_open_end_tests { - use super::*; - use std::sync::Arc; - use crate::schema; - - fn create_test_element() -> Arc { - let element = schema::XsElement { - name: "reasoning".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: Some("true".to_string()), - text: None, - xs_attribute: None, - xs_sequence: Some(schema::XsSequence { - text: None, - xs_any: schema::XsAny { - min_occurs: "0".to_string(), - max_occurs: "unbounded".to_string(), - process_contents: "skip".to_string(), - }, - }), - }, - }; - Arc::new(element) - } - - fn create_element_non_mixed() -> Arc { - let element = schema::XsElement { - name: "element".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: None, // Not mixed content - text: None, - xs_attribute: None, - xs_sequence: Some(schema::XsSequence { - text: None, - xs_any: schema::XsAny { - min_occurs: "0".to_string(), - max_occurs: "unbounded".to_string(), - process_contents: "skip".to_string(), - }, - }), - }, - }; - Arc::new(element) - } - - fn create_element_with_required_attributes() -> Arc { - let element = schema::XsElement { - name: "delete".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: None, - text: None, - xs_attribute: Some(vec![ - schema::XsAttribute { - name: "id".to_string(), - xs_attribute_type: "xs:string".to_string(), - xs_attribute_use: "required".to_string(), - } - ]), - xs_sequence: None, - }, - }; - Arc::new(element) - } - - #[test] - fn test_append_text_in_mixed_content() { - let element = create_test_element(); - let token = ElementOpenEnd::new(Arc::clone(&element)); - - // Test appending text in mixed content element - let next_tokens = token.append('a'); - assert!(!next_tokens.is_empty(), "Should allow text content in mixed element"); - - match &next_tokens[0] { - Token::TextContent(_) => (), - _ => panic!("Expected TextContent, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_append_closing_tag_start() { - let element = create_test_element(); - let token = ElementOpenEnd::new(Arc::clone(&element)); - - // Test starting a closing tag - let next_tokens = token.append('<'); - assert!(!next_tokens.is_empty(), "Should allow closing tag start"); - - match &next_tokens[0] { - Token::ElementCloseStart(_) => (), - _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_append_whitespace_in_mixed() { - let element = create_test_element(); - let token = ElementOpenEnd::new(Arc::clone(&element)); - - // Test whitespace in mixed content - let next_tokens = token.append(' '); - assert!(!next_tokens.is_empty(), "Should allow whitespace in mixed content"); - - match &next_tokens[0] { - Token::TextContent(_) => (), - _ => panic!("Expected TextContent, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_append_in_non_mixed() { - let element = create_element_non_mixed(); - let token = ElementOpenEnd::new(Arc::clone(&element)); - - // Test appending text in non-mixed content (should reject) - let next_tokens = token.clone().append('a'); - assert!(next_tokens.is_empty(), "Should reject text in non-mixed element"); - - // Test whitespace (should accept leading to closing tag) - let next_tokens = token.clone().append(' '); - assert!(!next_tokens.is_empty(), "Should allow whitespace in non-mixed"); - - // Test closing tag start - let next_tokens = token.append('<'); - assert!(!next_tokens.is_empty(), "Should allow closing tag start in non-mixed"); - } - - #[test] - fn test_required_attributes() { - let element = create_element_with_required_attributes(); - - // No attributes set yet - let token = ElementOpenEnd::new(Arc::clone(&element)); - let next_tokens = token.append('<'); - assert!(next_tokens.is_empty(), "Should reject closing tag without required attributes"); - - // With required attribute set - let token = ElementOpenEnd::with_attributes( - Arc::clone(&element), - vec!["id".to_string()] - ); - let next_tokens = token.append('<'); - assert!(!next_tokens.is_empty(), "Should allow closing tag with required attributes set"); - } -} \ No newline at end of file +use crate::*; +use std::sync::Arc; + +/// Represents a fully opened XML element, containing the schema element reference +#[derive(Clone, Debug)] +pub struct ElementOpenEnd { + element: Arc, +} + +impl ElementOpenEnd { + pub fn new(element: Arc) -> Self { + Self { + element, + } + } + + pub fn append(self, c: char) -> Vec { + if '<' == c { + vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))] + } else { + // Check if this element allows mixed content + let is_mixed = self.element.xs_complex_type.mixed + .as_ref() + .map(|val| val == "true") + .unwrap_or(false); + + if is_mixed || c.is_whitespace() { + // Allow text content for mixed elements, or whitespace for any element + vec![Token::TextContent(TextContent::new(self.element))] + } else { + // Reject text content for non-mixed elements + vec![] + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::schema; + use std::sync::Arc; + + fn create_test_element() -> Arc { + let element = schema::XsElement { + name: "reasoning".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }; + Arc::new(element) + } + + fn create_element_non_mixed() -> Arc { + let element = schema::XsElement { + name: "element".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: None, // Not mixed content + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }; + Arc::new(element) + } + + #[test] + fn test_append_text_in_mixed_content() { + let element = create_test_element(); + let token = ElementOpenEnd::new(Arc::clone(&element)); + + // Test appending text in mixed content element + let next_tokens = token.append('a'); + assert!( + !next_tokens.is_empty(), + "Should allow text content in mixed element" + ); + + match &next_tokens[0] { + Token::TextContent(_) => (), + _ => panic!("Expected TextContent, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_append_closing_tag_start() { + let element = create_test_element(); + let token = ElementOpenEnd::new(Arc::clone(&element)); + + // Test starting a closing tag + let next_tokens = token.append('<'); + assert!(!next_tokens.is_empty(), "Should allow closing tag start"); + + match &next_tokens[0] { + Token::ElementCloseStart(_) => (), + _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_append_whitespace_in_mixed() { + let element = create_test_element(); + let token = ElementOpenEnd::new(Arc::clone(&element)); + + // Test whitespace in mixed content + let next_tokens = token.append(' '); + assert!( + !next_tokens.is_empty(), + "Should allow whitespace in mixed content" + ); + + match &next_tokens[0] { + Token::TextContent(_) => (), + _ => panic!("Expected TextContent, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_append_in_non_mixed() { + let element = create_element_non_mixed(); + let token = ElementOpenEnd::new(Arc::clone(&element)); + + // Test appending text in non-mixed content (should reject) + let next_tokens = token.clone().append('a'); + assert!( + next_tokens.is_empty(), + "Should reject text in non-mixed element" + ); + + // Test whitespace (should accept leading to closing tag) + let next_tokens = token.clone().append(' '); + assert!( + !next_tokens.is_empty(), + "Should allow whitespace in non-mixed" + ); + + // Test closing tag start + let next_tokens = token.append('<'); + assert!( + !next_tokens.is_empty(), + "Should allow closing tag start in non-mixed" + ); + } +} diff --git a/lib/xml_schema_validator/src/token/element_open_name.rs b/lib/xml_schema_validator/src/token/element_open_name.rs index eb6bbcd..5a5b082 100644 --- a/lib/xml_schema_validator/src/token/element_open_name.rs +++ b/lib/xml_schema_validator/src/token/element_open_name.rs @@ -1,272 +1,186 @@ -use std::sync::Arc; -use crate::*; - -/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference -#[derive(Clone, Debug)] -pub struct ElementOpenName { - buffer: String, - element: Arc, -} - -impl ElementOpenName { - pub fn new(element: Arc, buffer: String) -> Result { - if element.name.starts_with(buffer.as_str()) { - Ok(Self { - buffer, - element, - }) - } else { - Err(Error::InvalidXml(format!("Element name '{}' doesn't match '{}'", buffer, element.name))) - } - } - - pub fn append(self, c: char) -> Vec { - if c.is_whitespace() { - // Check if we've matched the full element name - if self.element.name == self.buffer { - // Element name is complete, handle whitespace after name - - // Check if the element has attributes - if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { - if !attributes.is_empty() { - // Transition to attribute name parsing - return vec![Token::AttributeName(AttributeName::new(Arc::clone(&self.element)))]; - } - } - - // No attributes - return vec![ - Token::AttributeName(AttributeName::new(Arc::clone(&self.element))), - Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element))), - Token::ElementSelfClose(ElementSelfClose::new()), - Token::ElementOpenName(self), - ]; - } else { - // Whitespace in the middle of an element name is invalid - return vec![]; - } - } else if c == '>' { - // Check if we've matched the full element name - if self.element.name == self.buffer { - // Element name is complete, end of opening tag - - // Check if this element has required attributes - if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { - for attr in attributes { - if attr.xs_attribute_use == "required" { - // Found a required attribute but no attributes are set - // Return an empty vector to indicate error - return vec![]; - } - } - } - - return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element)))]; - } else { - // '>' before full element name is invalid - return vec![]; - } - } else if c == '/' { - // Check if we've matched the full element name - if self.element.name == self.buffer { - // Element name is complete, self-closing tag - return vec![Token::ElementSelfClose(ElementSelfClose::new())]; - } else { - // '/' before full element name is invalid - return vec![]; - } - } else { - // Add character to buffer and check if we're still building a valid element name - let new_buffer = self.buffer.clone() + &c.to_string(); - - if self.element.name.starts_with(&new_buffer) { - // Still building the element name - return vec![Token::ElementOpenName(ElementOpenName { - buffer: new_buffer, - element: Arc::clone(&self.element), - })]; - } else { - // Character doesn't match what we expect for this element name - return vec![]; - } - } - } -} - -#[cfg(test)] -mod element_open_name_tests { - use super::*; - use std::sync::Arc; - use crate::schema; - - fn create_test_element() -> Arc { - let element = schema::XsElement { - name: "reasoning".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: Some("true".to_string()), - text: None, - xs_attribute: None, - xs_sequence: Some(schema::XsSequence { - text: None, - xs_any: schema::XsAny { - min_occurs: "0".to_string(), - max_occurs: "unbounded".to_string(), - process_contents: "skip".to_string(), - }, - }), - }, - }; - Arc::new(element) - } - - fn create_element_with_attributes() -> Arc { - let element = schema::XsElement { - name: "delete".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: None, - text: None, - xs_attribute: Some(vec![ - schema::XsAttribute { - name: "id".to_string(), - xs_attribute_type: "xs:string".to_string(), - xs_attribute_use: "required".to_string(), - } - ]), - xs_sequence: None, - }, - }; - Arc::new(element) - } - - #[test] - fn test_append_character_by_character() { - let element = create_test_element(); - - // Test each character in "reasoning" - let mut buffer = String::new(); - for c in "reasoning".chars() { - buffer.push(c); - let result = ElementOpenName::new(Arc::clone(&element), buffer.clone()); - assert!(result.is_ok(), "Failed to create ElementOpenName with buffer '{}'", buffer); - } - } - - #[test] - fn test_append_full_name() { - let element = create_test_element(); - - // Test with the full element name - let result = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()); - assert!(result.is_ok(), "Failed to create ElementOpenName with full element name"); - - let element_name = result.unwrap(); - - // Test transition after full name with '>' - let next_tokens = element_name.append('>'); - assert!(!next_tokens.is_empty(), "Should accept '>' after full element name"); - - match &next_tokens[0] { - Token::ElementOpenEnd(_) => (), - _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_append_space_after_name() { - let element = create_test_element(); - - // Test with the full element name - let element_name = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()).unwrap(); - - // Test transition after full name with space - let next_tokens = element_name.append(' '); - assert!(!next_tokens.is_empty(), "Should accept space after full element name"); - - assert!(next_tokens.len() == 4); - assert!(next_tokens.iter() - .filter(|token| matches!(token, Token::ElementOpenName(_))) - .next() - .is_some()); - assert!(next_tokens.iter() - .filter(|token| matches!(token, Token::AttributeName(_))) - .next() - .is_some()); - assert!(next_tokens.iter() - .filter(|token| matches!(token, Token::ElementOpenEnd(_))) - .next() - .is_some()); - assert!(next_tokens.iter() - .filter(|token| matches!(token, Token::ElementSelfClose(_))) - .next() - .is_some()); - } - - #[test] - fn test_append_with_attributes() { - let element = create_element_with_attributes(); - - // Test with the full element name - let element_name = ElementOpenName::new(Arc::clone(&element), "delete".to_string()).unwrap(); - - // Test transition after full name with space (expecting attribute name) - let next_tokens = element_name.append(' '); - assert!(!next_tokens.is_empty(), "Should accept space after element name with attributes"); - - // The space should lead to a whitespace token, which should then lead to attribute name - match &next_tokens[0] { - Token::AttributeName(_) => (), - _ => panic!("Expected AttributeName, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_append_slash_after_name() { - let element = create_test_element(); - - // Test with the full element name - let element_name = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()).unwrap(); - - // Test transition after full name with / - let next_tokens = element_name.append('/'); - assert!(!next_tokens.is_empty(), "Should accept '/' after full element name"); - - match &next_tokens[0] { - Token::ElementSelfClose(_) => (), - _ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_invalid_continuation() { - let element = create_test_element(); - - // Test with a partial element name and an invalid continuation character - let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap(); - let next_tokens = element_name.append('x'); - assert!(next_tokens.is_empty(), "Should reject invalid character in element name"); - - // Test whitespace in middle of name - need to create a new instance since append consumes self - let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap(); - let next_tokens = element_name.append(' '); - assert!(next_tokens.is_empty(), "Should reject whitespace in middle of element name"); - - // Test > in middle of name - need to create a new instance since append consumes self - let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap(); - let next_tokens = element_name.append('>'); - assert!(next_tokens.is_empty(), "Should reject '>' in middle of element name"); - } - - #[test] - fn test_required_attributes_validation() { - let element = create_element_with_attributes(); - - // Test with the full element name - let element_name = ElementOpenName::new(Arc::clone(&element), "delete".to_string()).unwrap(); - - // Try to close the tag with > directly, skipping the required attribute - let next_tokens = element_name.append('>'); - assert!(next_tokens.is_empty(), "Should reject closing tag without required attributes"); - } -} \ No newline at end of file +use crate::*; +use std::sync::Arc; + +/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference +#[derive(Clone, Debug)] +pub struct ElementOpenName { + buffer: String, + element: Arc, +} + +impl ElementOpenName { + pub fn new(element: Arc, buffer: String) -> Self { + Self { buffer, element } + } + + pub fn append(mut self, c: char) -> Vec { + if c == '>' { + // Check if we've matched the full element name + if !self.buffer.contains(&self.element.name) { + return vec![]; + } + // Check if the element has attributes + if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { + // Check if they are required + if attributes + .iter() + .filter(|attribute| attribute.xs_attribute_use == "required") + .next() + .is_some() + { + return vec![]; + } else { + return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone( + &self.element, + )))]; + } + } else { + return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone( + &self.element, + )))]; + } + } else if c == '/' { + // Check if we've matched the full element name + if !self.buffer.contains(&self.element.name) { + return vec![]; + } + // Check if the element has attributes + if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { + // Check if they are required + if attributes + .iter() + .filter(|attribute| attribute.xs_attribute_use == "required") + .next() + .is_some() + { + return vec![]; + } else { + return vec![Token::ElementSelfClose(ElementSelfClose::new())]; + } + } else { + return vec![Token::ElementSelfClose(ElementSelfClose::new())]; + } + } else if c.is_whitespace() { + // Check if we've matched the full element name + if self.buffer.contains(&self.element.name) { + if let Some(last) = self.buffer.chars().last() { + if last.is_whitespace() { + return vec![Token::ElementOpenName(self)]; + } + } + self.buffer.push(c); + return vec![Token::ElementOpenName(self)]; + } else { + return vec![]; + } + } else if !self.buffer.is_empty() && self.buffer.chars().last().unwrap().is_whitespace() { + // Check for start of attribute name + if self.buffer.contains(&self.element.name) { + if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { + if attributes + .iter() + .filter(|attribute| attribute.name.starts_with(c)) + .next() + .is_some() + { + return vec![Token::AttributeName(AttributeName::new( + c, + self.element.clone(), + ))]; + } + } + } + return vec![]; + } else { + // Add character to buffer and check if we're still building a valid element name + let new_buffer = self.buffer.clone() + &c.to_string(); + + if self.element.name.starts_with(&new_buffer) { + // Still building the element name + return vec![Token::ElementOpenName(ElementOpenName { + buffer: new_buffer, + element: Arc::clone(&self.element), + })]; + } else { + // Character doesn't match what we expect for this element name + return vec![]; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_attribute_name() { + let values = [ + "", "", "", ""]; + for value in values { + let mut validator = crate::tests::example_validator(); + validator.append(value).expect(value); + assert!(!validator.current_tokens.is_empty(), "{value}"); + assert!( + validator + .current_tokens + .iter() + .filter(|token| matches!(token, Token::ElementOpenEnd(_))) + .next() + .is_some(), + "{value}" + ); + } + } + + #[test] + fn test_invalid_closing() { + let values = ["", ""]; + for value in values { + let mut validator = crate::tests::example_validator(); + validator.append(value).expect_err(value); + assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'"); + } + } +} diff --git a/lib/xml_schema_validator/src/token/element_open_start.rs b/lib/xml_schema_validator/src/token/element_open_start.rs index cc0d9d7..92f0dcc 100644 --- a/lib/xml_schema_validator/src/token/element_open_start.rs +++ b/lib/xml_schema_validator/src/token/element_open_start.rs @@ -1,25 +1,64 @@ -use std::sync::Arc; -use crate::*; - -/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference -#[derive(Clone, Debug)] -pub struct ElementOpenStart { - element: Arc, -} - -impl ElementOpenStart { - pub fn new(element: Arc) -> Self { - Self { - element, - } - } - - pub fn append(self, c: char) -> Vec { - let element_name = ElementOpenName::new(Arc::clone(&self.element), c.to_string()); - if let Ok(element_name) = element_name { - vec![Token::ElementOpenName(element_name)] - } else { - vec![] - } - } -} \ No newline at end of file +use crate::*; +use std::sync::Arc; + +/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference +#[derive(Clone, Debug)] +pub struct ElementOpenStart { + element: Arc, +} + +impl ElementOpenStart { + pub fn new(element: Arc) -> Self { + Self { element } + } + + pub fn append(self, c: char) -> Vec { + if self.element.name.starts_with(c) { + vec![Token::ElementOpenName(ElementOpenName::new( + Arc::clone(&self.element), + c.to_string(), + ))] + } else { + vec![] + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_accept_valid_element_names() { + let values = [ + ">, - /// Track which attributes have been set - attributes_set: Vec, -} - -impl ElementSelfClose { - /// Create a new self-closing element - pub fn new() -> Self { - Self { - element: None, - attributes_set: Vec::new(), - } - } - - /// Create a new self-closing element with element reference and attribute tracking - pub fn new_with_attributes(element: Arc, attributes_set: Vec) -> Self { - Self { - element: Some(element), - attributes_set, - } - } - - /// Append a character to the self-closing tag and return possible continuations - pub fn append(self, c: char) -> Vec { - // Before we close, verify that all required attributes are present - if let Some(element) = &self.element { - if c == '>' { - if let Some(attributes) = &element.xs_complex_type.xs_attribute { - for attr in attributes { - if attr.xs_attribute_use == "required" && !self.attributes_set.contains(&attr.name) { - // Missing required attribute, this is an error - return vec![]; - } - } - } - return vec![Token::EndOfFile(EndOfFile::new())]; - } - } else if c == '>' { - // No element reference, can't validate attributes, just close - return vec![Token::EndOfFile(EndOfFile::new())]; - } - - if c.is_whitespace() { - vec![Token::ElementSelfClose(self)] - } else { - vec![] - } - } -} - -#[cfg(test)] -mod element_self_close_tests { - use super::*; - use std::sync::Arc; - use crate::schema; - - fn create_element_with_required_attributes() -> Arc { - let element = schema::XsElement { - name: "delete".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: None, - text: None, - xs_attribute: Some(vec![ - schema::XsAttribute { - name: "id".to_string(), - xs_attribute_type: "xs:string".to_string(), - xs_attribute_use: "required".to_string(), - } - ]), - xs_sequence: None, - }, - }; - Arc::new(element) - } - - #[test] - fn test_self_close_with_greater_than() { - let token = ElementSelfClose::new(); - - // Test closing with > - let next_tokens = token.append('>'); - assert!(!next_tokens.is_empty(), "Should accept '>' to close self-closing tag"); - assert_eq!(next_tokens.len(), 1, "Should have exactly one token after '>'"); - - match &next_tokens[0] { - Token::EndOfFile(_) => (), - _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_self_close_with_whitespace() { - let token = ElementSelfClose::new(); - - // Test with whitespace - let next_tokens = token.append(' '); - assert!(!next_tokens.is_empty(), "Should accept whitespace in self-closing tag"); - - match &next_tokens[0] { - Token::ElementSelfClose(_) => (), - _ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_self_close_with_invalid_char() { - let token = ElementSelfClose::new(); - - // Test with invalid character - let next_tokens = token.append('a'); - assert!(next_tokens.is_empty(), "Should reject invalid character in self-closing tag"); - } - - #[test] - fn test_required_attributes_validation() { - let element = create_element_with_required_attributes(); - - // Without required attributes - let token = ElementSelfClose::new_with_attributes(Arc::clone(&element), vec![]); - let next_tokens = token.append('>'); - assert!(next_tokens.is_empty(), "Should reject closing without required attributes"); - - // With required attributes - let token = ElementSelfClose::new_with_attributes( - Arc::clone(&element), - vec!["id".to_string()] - ); - let next_tokens = token.append('>'); - assert!(!next_tokens.is_empty(), "Should accept closing with required attributes"); - - match &next_tokens[0] { - Token::EndOfFile(_) => (), - _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]), - } - } -} \ No newline at end of file +use crate::*; + +/// Represents a self-closing XML element +#[derive(Clone, Debug)] +pub struct ElementSelfClose {} + +impl ElementSelfClose { + /// Create a new self-closing element + pub fn new() -> Self { + Self {} + } + + pub fn append(self, c: char) -> Vec { + if c == '>' { + return vec![Token::EndOfFile(EndOfFile::new())]; + } else { + vec![] + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_valid() { + let values = [""]; + for value in values { + let mut validator = crate::tests::example_validator(); + validator.append(value).expect(value); + assert!(!validator.current_tokens.is_empty(), "{value}"); + assert!( + validator + .current_tokens + .iter() + .filter(|token| matches!(token, Token::EndOfFile(_))) + .next() + .is_some(), + "{value}" + ); + } + } + + #[test] + fn test_invalid() { + let values = [" Self { - Self {} - } - - pub fn append(self, _c: char) -> Vec { - vec![] - } -} - -#[cfg(test)] -mod end_of_file_tests { - use super::*; - - #[test] - fn test_end_of_file_any_char() { - let token = EndOfFile::new(); - - // Test with any character (should reject) - let next_tokens = token.append('a'); - assert!(next_tokens.is_empty(), "Should reject any character at EOF"); - } -} \ No newline at end of file +use crate::*; + +/// Represents the final state at the end of an XML file +#[derive(Clone, Debug)] +pub struct EndOfFile {} + +impl EndOfFile { + pub fn new() -> Self { + Self {} + } + + pub fn append(self, _c: char) -> Vec { + vec![] + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_end_of_file_any_char() { + let token = EndOfFile::new(); + + // Test with any character (should reject) + let next_tokens = token.append('a'); + assert!(next_tokens.is_empty(), "Should reject any character at EOF"); + } +} diff --git a/lib/xml_schema_validator/src/token/mod.rs b/lib/xml_schema_validator/src/token/mod.rs index af22365..b92ea93 100644 --- a/lib/xml_schema_validator/src/token/mod.rs +++ b/lib/xml_schema_validator/src/token/mod.rs @@ -1,292 +1,285 @@ -mod element_close_name; -mod element_close_start; -mod element_open_name; -mod element_open_end; -mod element_open_start; -mod end_of_file; -mod start_of_file; -mod text_content; -mod attribute_name; -mod attribute_value; -mod element_self_close; - -pub use element_close_name::ElementCloseName; -pub use element_close_start::ElementCloseStart; -pub use element_open_name::ElementOpenName; -pub use element_open_end::ElementOpenEnd; -pub use element_open_start::ElementOpenStart; -pub use end_of_file::EndOfFile; -pub use start_of_file::StartOfFile; -pub use text_content::TextContent; -pub use attribute_name::AttributeName; -pub use attribute_value::AttributeValue; -pub use element_self_close::ElementSelfClose; - -/// Represents the different states of XML token parsing -#[derive(Clone, Debug)] -pub enum Token { - ElementCloseName(ElementCloseName), - ElementCloseStart(ElementCloseStart), - ElementOpenName(ElementOpenName), - ElementOpenEnd(ElementOpenEnd), - ElementOpenStart(ElementOpenStart), - EndOfFile(EndOfFile), - StartOfFile(StartOfFile), - TextContent(TextContent), - AttributeName(AttributeName), - AttributeValue(AttributeValue), - ElementSelfClose(ElementSelfClose), -} - -impl Token { - pub fn append(self, c: char) -> Vec { - match self { - Token::ElementCloseName(element_close_name) => element_close_name.append(c), - Token::ElementCloseStart(element_close_start) => element_close_start.append(c), - Token::ElementOpenName(element_name) => element_name.append(c), - Token::ElementOpenEnd(element_open) => element_open.append(c), - Token::ElementOpenStart(element_start) => element_start.append(c), - Token::EndOfFile(end_of_file) => end_of_file.append(c), - Token::StartOfFile(start_of_file) => start_of_file.append(c), - Token::TextContent(text_content) => text_content.append(c), - Token::AttributeName(attribute_name) => attribute_name.append(c), - Token::AttributeValue(attribute_value) => attribute_value.append(c), - Token::ElementSelfClose(element_self_close) => element_self_close.append(c), - } - } - - pub fn is_eof(&self) -> bool { - match self { - Token::EndOfFile(_) => true, - _ => false, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::Arc; - use crate::schema; - - fn create_test_element() -> Arc { - let element = schema::XsElement { - name: "reasoning".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: Some("true".to_string()), - text: None, - xs_attribute: None, - xs_sequence: Some(schema::XsSequence { - text: None, - xs_any: schema::XsAny { - min_occurs: "0".to_string(), - max_occurs: "unbounded".to_string(), - process_contents: "skip".to_string(), - }, - }), - }, - }; - Arc::new(element) - } - - #[test] - fn test_start_of_file_transition() { - let element = create_test_element(); - let token = Token::StartOfFile(StartOfFile::new(Arc::clone(&element))); - - // Should only accept '<' - let next_tokens = token.append('<'); - assert_eq!(next_tokens.len(), 1, "Should have one transition"); - - match &next_tokens[0] { - Token::ElementOpenStart(_) => (), - _ => panic!("Expected ElementStart, got {:?}", next_tokens[0]), - } - - // Should reject other characters - let token = Token::StartOfFile(StartOfFile::new(Arc::clone(&element))); - let next_tokens = token.append('a'); - assert_eq!(next_tokens.len(), 0, "Should reject 'a'"); - } - - #[test] - fn test_element_start_transition() { - let element = create_test_element(); - let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element))); - - // Should accept first char of element name - let next_tokens = token.append('r'); - assert_eq!(next_tokens.len(), 1, "Should have one transition"); - - match &next_tokens[0] { - Token::ElementOpenName(_) => (), - _ => panic!("Expected ElementName, got {:?}", next_tokens[0]), - } - - // Should reject invalid char - let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element))); - let next_tokens = token.append('x'); - assert_eq!(next_tokens.len(), 0, "Should reject 'x'"); - } - - #[test] - fn test_element_name_transition() { - let element = create_test_element(); - let token = Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()).unwrap()); - - // Continue building name - let next_tokens = token.append('e'); - assert_eq!(next_tokens.len(), 1, "Should have one transition"); - - match &next_tokens[0] { - Token::ElementOpenName(_) => (), - _ => panic!("Expected ElementName, got {:?}", next_tokens[0]), - } - - // Build complete name and end with > - // Remove the unused buffer - let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)))]; - - for c in "reasoning>".chars() { - let mut new_tokens = Vec::new(); - for token in current_tokens { - new_tokens.append(&mut token.append(c)); - } - current_tokens = new_tokens; - - assert!(!current_tokens.is_empty(), - "Should have valid transition after appending '{}'", c); - } - - // Check final state - assert_eq!(current_tokens.len(), 1, "Should have one final transition"); - match ¤t_tokens[0] { - Token::ElementOpenEnd(_) => (), - _ => panic!("Expected ElementOpen, got {:?}", current_tokens[0]), - } - } - - #[test] - fn test_element_open_end_transition() { - let element = create_test_element(); - let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element))); - - // Should accept text content in mixed element - let next_tokens = token.append('t'); - assert_eq!(next_tokens.len(), 1, "Should accept text"); - - match &next_tokens[0] { - Token::TextContent(_) => (), - _ => panic!("Expected TextContent, got {:?}", next_tokens[0]), - } - - // Should accept closing tag start - let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element))); - let next_tokens = token.append('<'); - assert_eq!(next_tokens.len(), 1, "Should accept '<'"); - - match &next_tokens[0] { - Token::ElementCloseStart(_) => (), - _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_text_content_transition() { - let element = create_test_element(); - let token = Token::TextContent(TextContent::new(Arc::clone(&element))); - - // Should accept more text - let next_tokens = token.append('a'); - assert_eq!(next_tokens.len(), 1, "Should accept more text"); - - match &next_tokens[0] { - Token::TextContent(_) => (), - _ => panic!("Expected TextContent, got {:?}", next_tokens[0]), - } - - // Should accept closing tag start - let token = Token::TextContent(TextContent::new(Arc::clone(&element))); - let next_tokens = token.append('<'); - assert_eq!(next_tokens.len(), 1, "Should accept '<'"); - - match &next_tokens[0] { - Token::ElementCloseStart(_) => (), - _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_element_close_start_transition() { - let element = create_test_element(); - let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element))); - - // Should accept / - let next_tokens = token.append('/'); - assert_eq!(next_tokens.len(), 1, "Should accept '/'"); - - let token = next_tokens[0].clone(); - match token { - Token::ElementCloseStart(_) => (), - _ => panic!("Expected ElementCloseStart with slash, got {:?}", token), - } - - // After /, should accept first char of element name - let next_tokens = token.append('r'); - assert_eq!(next_tokens.len(), 1, "Should accept 'r'"); - - match &next_tokens[0] { - Token::ElementCloseName(_) => (), - _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_element_close_name_transition() { - let element = create_test_element(); - let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element))); - - // Add / first - let next_tokens = token.append('/'); - assert_eq!(next_tokens.len(), 1, "Should accept '/'"); - - // Start building name - clone the token to avoid moving out of the vector - let next_tokens = next_tokens[0].clone().append('r'); - assert_eq!(next_tokens.len(), 1, "Should accept 'r'"); - - // Continue with each character - let mut current_tokens = next_tokens; - for c in "easoning".chars() { - let mut new_tokens = Vec::new(); - for token in current_tokens { - new_tokens.append(&mut token.append(c)); - } - current_tokens = new_tokens; - - assert!(!current_tokens.is_empty(), - "Should have valid transition after appending '{}'", c); - } - - // Final > to close - clone the token to avoid moving out of the vector - let next_tokens = current_tokens[0].clone().append('>'); - assert_eq!(next_tokens.len(), 1, "Should accept '>'"); - - match &next_tokens[0] { - Token::EndOfFile(_) => (), - _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_end_of_file_properties() { - let token = Token::EndOfFile(EndOfFile::new()); - - // Should be identified as EOF - assert!(token.is_eof(), "EndOfFile should be recognized as EOF"); - - // Should not accept any more input - let next_tokens = token.append('a'); - assert_eq!(next_tokens.len(), 0, "Should not accept any more input"); - } -} \ No newline at end of file +mod attribute_equals; +mod attribute_name; +mod attribute_value; +mod element_close_name; +mod element_close_start; +mod element_open_end; +mod element_open_name; +mod element_open_start; +mod element_self_close; +mod end_of_file; +mod start_of_file; +mod text_content; + +pub use attribute_equals::AttributeEquals; +pub use attribute_name::AttributeName; +pub use attribute_value::AttributeValue; +pub use element_close_name::ElementCloseName; +pub use element_close_start::ElementCloseStart; +pub use element_open_end::ElementOpenEnd; +pub use element_open_name::ElementOpenName; +pub use element_open_start::ElementOpenStart; +pub use element_self_close::ElementSelfClose; +pub use end_of_file::EndOfFile; +pub use start_of_file::StartOfFile; +pub use text_content::TextContent; + +/// Represents the different states of XML token parsing +#[derive(Clone, Debug)] +pub enum Token { + AttributeEquals(AttributeEquals), + AttributeName(AttributeName), + AttributeValue(AttributeValue), + ElementCloseName(ElementCloseName), + ElementCloseStart(ElementCloseStart), + ElementOpenEnd(ElementOpenEnd), + ElementOpenName(ElementOpenName), + ElementOpenStart(ElementOpenStart), + ElementSelfClose(ElementSelfClose), + EndOfFile(EndOfFile), + StartOfFile(StartOfFile), + TextContent(TextContent), +} + +impl Token { + pub fn append(self, c: char) -> Vec { + match self { + Token::AttributeEquals(attribute_equals) => attribute_equals.append(c), + Token::ElementCloseName(element_close_name) => element_close_name.append(c), + Token::ElementCloseStart(element_close_start) => element_close_start.append(c), + Token::ElementOpenName(element_name) => element_name.append(c), + Token::ElementOpenEnd(element_open) => element_open.append(c), + Token::ElementOpenStart(element_start) => element_start.append(c), + Token::EndOfFile(end_of_file) => end_of_file.append(c), + Token::StartOfFile(start_of_file) => start_of_file.append(c), + Token::TextContent(text_content) => text_content.append(c), + Token::AttributeName(attribute_name) => attribute_name.append(c), + Token::AttributeValue(attribute_value) => attribute_value.append(c), + Token::ElementSelfClose(element_self_close) => element_self_close.append(c), + } + } + + pub fn is_eof(&self) -> bool { + match self { + Token::EndOfFile(_) => true, + _ => false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::schema; + use std::sync::Arc; + + pub fn create_test_element() -> Arc { + let element = schema::XsElement { + name: "reasoning".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }; + Arc::new(element) + } + + #[test] + fn test_element_start_transition() { + let element = create_test_element(); + let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element))); + + // Should accept first char of element name + let next_tokens = token.append('r'); + assert_eq!(next_tokens.len(), 1, "Should have one transition"); + + match &next_tokens[0] { + Token::ElementOpenName(_) => (), + _ => panic!("Expected ElementName, got {:?}", next_tokens[0]), + } + + // Should reject invalid char + let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element))); + let next_tokens = token.append('x'); + assert_eq!(next_tokens.len(), 0, "Should reject 'x'"); + } + + #[test] + fn test_element_name_transition() { + let element = create_test_element(); + let token = + Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string())); + + // Continue building name + let next_tokens = token.append('e'); + assert_eq!(next_tokens.len(), 1, "Should have one transition"); + + match &next_tokens[0] { + Token::ElementOpenName(_) => (), + _ => panic!("Expected ElementName, got {:?}", next_tokens[0]), + } + + // Build complete name and end with > + // Remove the unused buffer + let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone( + &element, + )))]; + + for c in "reasoning>".chars() { + let mut new_tokens = Vec::new(); + for token in current_tokens { + new_tokens.append(&mut token.append(c)); + } + current_tokens = new_tokens; + + assert!( + !current_tokens.is_empty(), + "Should have valid transition after appending '{}'", + c + ); + } + + // Check final state + assert_eq!(current_tokens.len(), 1, "Should have one final transition"); + match ¤t_tokens[0] { + Token::ElementOpenEnd(_) => (), + _ => panic!("Expected ElementOpen, got {:?}", current_tokens[0]), + } + } + + #[test] + fn test_element_open_end_transition() { + let element = create_test_element(); + let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element))); + + // Should accept text content in mixed element + let next_tokens = token.append('t'); + assert_eq!(next_tokens.len(), 1, "Should accept text"); + + match &next_tokens[0] { + Token::TextContent(_) => (), + _ => panic!("Expected TextContent, got {:?}", next_tokens[0]), + } + + // Should accept closing tag start + let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element))); + let next_tokens = token.append('<'); + assert_eq!(next_tokens.len(), 1, "Should accept '<'"); + + match &next_tokens[0] { + Token::ElementCloseStart(_) => (), + _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_text_content_transition() { + let element = create_test_element(); + let token = Token::TextContent(TextContent::new(Arc::clone(&element))); + + // Should accept more text + let next_tokens = token.append('a'); + assert_eq!(next_tokens.len(), 1, "Should accept more text"); + + match &next_tokens[0] { + Token::TextContent(_) => (), + _ => panic!("Expected TextContent, got {:?}", next_tokens[0]), + } + + // Should accept closing tag start + let token = Token::TextContent(TextContent::new(Arc::clone(&element))); + let next_tokens = token.append('<'); + assert_eq!(next_tokens.len(), 1, "Should accept '<'"); + + match &next_tokens[0] { + Token::ElementCloseStart(_) => (), + _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_element_close_start_transition() { + let element = create_test_element(); + let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element))); + + // Should accept / + let next_tokens = token.append('/'); + assert_eq!(next_tokens.len(), 1, "Should accept '/'"); + + let token = next_tokens[0].clone(); + match token { + Token::ElementCloseStart(_) => (), + _ => panic!("Expected ElementCloseStart with slash, got {:?}", token), + } + + // After /, should accept first char of element name + let next_tokens = token.append('r'); + assert_eq!(next_tokens.len(), 1, "Should accept 'r'"); + + match &next_tokens[0] { + Token::ElementCloseName(_) => (), + _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_element_close_name_transition() { + let element = create_test_element(); + let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element))); + + // Add / first + let next_tokens = token.append('/'); + assert_eq!(next_tokens.len(), 1, "Should accept '/'"); + + // Start building name - clone the token to avoid moving out of the vector + let next_tokens = next_tokens[0].clone().append('r'); + assert_eq!(next_tokens.len(), 1, "Should accept 'r'"); + + // Continue with each character + let mut current_tokens = next_tokens; + for c in "easoning".chars() { + let mut new_tokens = Vec::new(); + for token in current_tokens { + new_tokens.append(&mut token.append(c)); + } + current_tokens = new_tokens; + + assert!( + !current_tokens.is_empty(), + "Should have valid transition after appending '{}'", + c + ); + } + + // Final > to close - clone the token to avoid moving out of the vector + let next_tokens = current_tokens[0].clone().append('>'); + assert_eq!(next_tokens.len(), 1, "Should accept '>'"); + + match &next_tokens[0] { + Token::EndOfFile(_) => (), + _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_end_of_file_properties() { + let token = Token::EndOfFile(EndOfFile::new()); + + // Should be identified as EOF + assert!(token.is_eof(), "EndOfFile should be recognized as EOF"); + + // Should not accept any more input + let next_tokens = token.append('a'); + assert_eq!(next_tokens.len(), 0, "Should not accept any more input"); + } +} diff --git a/lib/xml_schema_validator/src/token/start_of_file.rs b/lib/xml_schema_validator/src/token/start_of_file.rs index 582b2f5..fb460fa 100644 --- a/lib/xml_schema_validator/src/token/start_of_file.rs +++ b/lib/xml_schema_validator/src/token/start_of_file.rs @@ -1,75 +1,52 @@ -use std::sync::Arc; -use crate::*; - -/// Represents the initial state at the start of an XML file, containing a reference to the root element schema -#[derive(Clone, Debug)] -pub struct StartOfFile { - element: Arc, -} - -impl StartOfFile { - pub fn new(element: Arc) -> Self { - Self { element } - } - - pub fn append(self, c: char) -> Vec { - if '<' == c { - vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&self.element)))] - } else { - vec![] - } - } -} - -#[cfg(test)] -mod start_of_file_tests { - use super::*; - use std::sync::Arc; - use crate::schema; - - fn create_test_element() -> Arc { - let element = schema::XsElement { - name: "reasoning".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: Some("true".to_string()), - text: None, - xs_attribute: None, - xs_sequence: Some(schema::XsSequence { - text: None, - xs_any: schema::XsAny { - min_occurs: "0".to_string(), - max_occurs: "unbounded".to_string(), - process_contents: "skip".to_string(), - }, - }), - }, - }; - Arc::new(element) - } - - #[test] - fn test_start_of_file_less_than() { - let element = create_test_element(); - let token = StartOfFile::new(Arc::clone(&element)); - - // Test with < - let next_tokens = token.append('<'); - assert!(!next_tokens.is_empty(), "Should accept '<'"); - - match &next_tokens[0] { - Token::ElementOpenStart(_) => (), - _ => panic!("Expected ElementStart, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_start_of_file_invalid() { - let element = create_test_element(); - let token = StartOfFile::new(Arc::clone(&element)); - - // Test with invalid character - let next_tokens = token.append('a'); - assert!(next_tokens.is_empty(), "Should reject invalid character"); - } -} \ No newline at end of file +use crate::*; +use std::sync::Arc; + +/// Represents the initial state at the start of an XML file, containing a reference to the root element schema +#[derive(Clone, Debug)] +pub struct StartOfFile { + element: Arc, +} + +impl StartOfFile { + pub fn new(element: Arc) -> Self { + Self { element } + } + + pub fn append(self, c: char) -> Vec { + if '<' == c { + vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone( + &self.element, + )))] + } else { + vec![] + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_accept_open_start_token() { + let mut validator = crate::tests::example_validator(); + validator.append("<").unwrap(); + assert!(!validator.current_tokens.is_empty()); + assert!(validator + .current_tokens + .iter() + .filter(|token| matches!(token, Token::ElementOpenStart(_))) + .next() + .is_some()); + } + + #[test] + fn test_not_accept_other() { + let values = ["a", " ", "\t", ">", "/"]; + for value in values { + let mut validator = crate::tests::example_validator(); + validator.append(value).expect_err(value); + assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'"); + } + } +} diff --git a/lib/xml_schema_validator/src/token/text_content.rs b/lib/xml_schema_validator/src/token/text_content.rs index c22f0f9..401e766 100644 --- a/lib/xml_schema_validator/src/token/text_content.rs +++ b/lib/xml_schema_validator/src/token/text_content.rs @@ -1,84 +1,84 @@ -use std::sync::Arc; -use crate::*; - -/// Represents the text content within an XML element -#[derive(Clone, Debug)] -pub struct TextContent { - element: Arc, -} - -impl TextContent { - pub fn new(element: Arc) -> Self { - Self { - element, - } - } - - pub fn append(self, c: char) -> Vec { - if c == '<' { - vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))] - } else { - vec![Token::TextContent(TextContent::new( - Arc::clone(&self.element), - ))] - } - } -} - -#[cfg(test)] -mod text_content_tests { - use super::*; - use std::sync::Arc; - use crate::schema; - - fn create_test_element() -> Arc { - let element = schema::XsElement { - name: "reasoning".to_string(), - text: None, - xs_complex_type: schema::XsComplexType { - mixed: Some("true".to_string()), - text: None, - xs_attribute: None, - xs_sequence: Some(schema::XsSequence { - text: None, - xs_any: schema::XsAny { - min_occurs: "0".to_string(), - max_occurs: "unbounded".to_string(), - process_contents: "skip".to_string(), - }, - }), - }, - }; - Arc::new(element) - } - - #[test] - fn test_text_content_normal_char() { - let element = create_test_element(); - let token = TextContent::new(Arc::clone(&element)); - - // Test with normal character - let next_tokens = token.append('a'); - assert!(!next_tokens.is_empty(), "Should accept normal character"); - - match &next_tokens[0] { - Token::TextContent(_) => (), - _ => panic!("Expected TextContent, got {:?}", next_tokens[0]), - } - } - - #[test] - fn test_text_content_less_than() { - let element = create_test_element(); - let token = TextContent::new(Arc::clone(&element)); - - // Test with < (start of tag) - let next_tokens = token.append('<'); - assert!(!next_tokens.is_empty(), "Should accept '<'"); - - match &next_tokens[0] { - Token::ElementCloseStart(_) => (), - _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]), - } - } -} \ No newline at end of file +use crate::*; +use std::sync::Arc; + +/// Represents the text content within an XML element +#[derive(Clone, Debug)] +pub struct TextContent { + element: Arc, +} + +impl TextContent { + pub fn new(element: Arc) -> Self { + Self { element } + } + + pub fn append(self, c: char) -> Vec { + if c == '<' { + vec![Token::ElementCloseStart(ElementCloseStart::new( + Arc::clone(&self.element), + ))] + } else { + vec![Token::TextContent(TextContent::new(Arc::clone( + &self.element, + )))] + } + } +} + +#[cfg(test)] +mod text_content_tests { + use super::*; + use crate::schema; + use std::sync::Arc; + + fn create_test_element() -> Arc { + let element = schema::XsElement { + name: "reasoning".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }; + Arc::new(element) + } + + #[test] + fn test_text_content_normal_char() { + let element = create_test_element(); + let token = TextContent::new(Arc::clone(&element)); + + // Test with normal character + let next_tokens = token.append('a'); + assert!(!next_tokens.is_empty(), "Should accept normal character"); + + match &next_tokens[0] { + Token::TextContent(_) => (), + _ => panic!("Expected TextContent, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_text_content_less_than() { + let element = create_test_element(); + let token = TextContent::new(Arc::clone(&element)); + + // Test with < (start of tag) + let next_tokens = token.append('<'); + assert!(!next_tokens.is_empty(), "Should accept '<'"); + + match &next_tokens[0] { + Token::ElementCloseStart(_) => (), + _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]), + } + } +} diff --git a/lib/xml_schema_validator/tests/test_logits_processor.py b/lib/xml_schema_validator/tests/test_logits_processor.py index 57b586a..5ddcd18 100644 --- a/lib/xml_schema_validator/tests/test_logits_processor.py +++ b/lib/xml_schema_validator/tests/test_logits_processor.py @@ -1,13 +1,12 @@ from transformers import AutoTokenizer from xml_schema_validator import XmlLogitsProcessor import os -import time import torch model_path = "../../model" api_token = os.environ["SIA_HF_API_KEY"] -xml_schema_actions = open("../../action_schema.xsd").read() +xml_schema_actions = open("example_schema.xsd").read() xml_schema_only_root_node = """ @@ -59,60 +58,27 @@ def test_token_masking(): tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token) processor = XmlLogitsProcessor(tokenizer, xml_schema_actions) + + # Process empty input to set prompt length + input_ids = torch.tensor([tokenizer.encode("")]) + scores = torch.ones((1, len(tokenizer))) # All tokens have equal probability + processed_scores = processor(input_ids, scores) - # Create dummy input_ids and scores + # Process valid continuation input_ids = torch.tensor([tokenizer.encode("")]) scores = torch.ones((1, len(tokenizer))) # All tokens have equal probability - - # Process the scores processed_scores = processor(input_ids, scores) # Verify that valid continuations still have positive scores - assert processed_scores[0, tokenizer.encode(" ", add_special_tokens=False)[0]] > -float('inf') + space_token = tokenizer.encode(" ", add_special_tokens=False)[0] + assert processed_scores[0, space_token] > -float('inf') # Verify that invalid continuations are masked - assert processed_scores[0, tokenizer.encode("", add_special_tokens=False)[0]] == -float('inf') - -def test_performance(): - """Test performance with larger XML documents""" - - tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token) - processor = XmlLogitsProcessor(tokenizer, xml_schema_actions) - - # Generate a larger XML document - large_xml = "" + "Test content. " * 100 + "" - - # Measure time to process - start_time = time.time() - processor.core.append(large_xml) - processing_time = time.time() - start_time - - print(f"Processing time for large XML: {processing_time:.4f} seconds") - # You might want to assert that processing time is below a threshold - -def test_subword_tokens(): - """Test handling of subword tokens that might split XML tags""" - tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token) - schema_text = open("../../action_schema.xsd").read() - - processor = XmlLogitsProcessor(tokenizer, schema_text) - - # Find some XML tag that gets split into multiple tokens by your tokenizer - tag = "" - tokens = tokenizer.encode(tag, add_special_tokens=False) - - if len(tokens) > 1: - print(f"Tag '{tag}' is split into {len(tokens)} tokens") - - # Test that the processor can handle these split tokens - import torch - input_ids = torch.tensor([[tokens[0]]]) - scores = torch.ones((1, len(tokenizer))) - - processed_scores = processor(input_ids, scores) - - # The next token in the sequence should have a high score - assert processed_scores[0, tokens[1]] > -float('inf') + input_ids = torch.tensor([tokenizer.encode("")]) + scores = torch.ones((1, len(tokenizer))) # All tokens have equal probability + processed_scores = processor(input_ids, scores) + assert processed_scores[0, tokenizer.eos_token_id] == 1 + assert processed_scores[0, space_token] == -float('inf') if __name__ == "__main__": # Run individual tests for debugging @@ -120,5 +86,3 @@ if __name__ == "__main__": test_xml_schema_parsing() test_basic_xml_validation() test_token_masking() - test_performance() - test_subword_tokens()