Improved tests for tokens
This commit is contained in:
@@ -33,16 +33,19 @@ impl CharTrieNode {
|
||||
py: Python<'_>,
|
||||
xml_validator: &XmlSchemaValidator,
|
||||
) -> Vec<Py<PyAny>> {
|
||||
let res = self.children.iter().map(|(c, n)|{
|
||||
let mut node_validator = xml_validator.clone();
|
||||
if node_validator.append_char(*c).is_ok() {
|
||||
// If the node is valid, recursively check its children
|
||||
n.find_invalid_tokens(py, &node_validator)
|
||||
} else {
|
||||
// If the node is invalid, add it and its children to the list of invalid tokens
|
||||
n.all_tokens(py)
|
||||
}
|
||||
})
|
||||
let res = self
|
||||
.children
|
||||
.iter()
|
||||
.map(|(c, n)| {
|
||||
let mut node_validator = xml_validator.clone();
|
||||
if node_validator.append_char(*c).is_ok() {
|
||||
// If the node is valid, recursively check its children
|
||||
n.find_invalid_tokens(py, &node_validator)
|
||||
} else {
|
||||
// If the node is invalid, add it and its children to the list of invalid tokens
|
||||
n.all_tokens(py)
|
||||
}
|
||||
})
|
||||
.flatten()
|
||||
.collect();
|
||||
res
|
||||
@@ -109,4 +112,4 @@ impl CharTrie {
|
||||
root: self.root.clone_ref(py),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,160 +103,9 @@ mod tests {
|
||||
XmlSchemaValidator::new(&schema_text).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_element_open() {
|
||||
let mut validator = example_validator();
|
||||
let input = "<reasoning>";
|
||||
validator.append(input).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_element_open_whitespace() {
|
||||
let mut validator = example_validator();
|
||||
let input = "<reasoning >";
|
||||
validator.append(input).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_element_open_invalid_name() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
let input = "<rae";
|
||||
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
validator.append(input).expect_err("Should fail");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_element_close() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
let input = "<reasoning>test</reasoning>";
|
||||
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
validator.append(input).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_element_close_mismatched() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
let input = "<reasoning>test</wrong>";
|
||||
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
validator
|
||||
.append(input)
|
||||
.expect_err("Should fail with mismatched closing tag");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_with_attribute() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
let input = r#"<delete id="20250411_170104_361"/>"#;
|
||||
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
validator.append(input).unwrap();
|
||||
assert!(validator.eof());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_missing_required_attribute() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
// Create validator
|
||||
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
|
||||
// First, validate up to the opening tag
|
||||
assert!(
|
||||
validator.append("<delete>").is_err(),
|
||||
"Should fail with missing required attribute"
|
||||
);
|
||||
|
||||
// Create a new validator and try closing tag approach
|
||||
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
// Try to validate an incomplete delete tag that's missing the required id attribute
|
||||
let input = "<delete/>";
|
||||
let result = validator.append(input);
|
||||
assert!(result.is_err(), "Should fail without required id attribute");
|
||||
|
||||
// Check the error message - note the test now checks for either "required" OR "No valid continuations"
|
||||
if let Err(err) = result {
|
||||
println!("Error message: {}", err);
|
||||
let error_str = err.to_string().to_lowercase();
|
||||
assert!(
|
||||
error_str.contains("required") || error_str.contains("no valid continuations"),
|
||||
"Error should mention missing required attribute or invalid continuation"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_nesting() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
let validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
validator
|
||||
.clone()
|
||||
.append("<reasoning><reasoning>")
|
||||
.expect_err("Should fail because of nested tag");
|
||||
validator
|
||||
.clone()
|
||||
.append("<reasoning> <reasoning>")
|
||||
.expect_err("Should fail because of nested tag, even with whitespace");
|
||||
validator
|
||||
.clone()
|
||||
.append("<reasoning>foo<reasoning>")
|
||||
.expect_err("Should fail because of nested tag, even with text");
|
||||
validator
|
||||
.clone()
|
||||
.append("<reasoning> foo <reasoning>")
|
||||
.expect_err("Should fail because of nested tag, even with text and whitespace");
|
||||
validator
|
||||
.clone()
|
||||
.append("<reasoning> foo <reasoning/>")
|
||||
.expect_err("Should fail because of nested tag, even with self closing tag");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stop_self_closing() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
let input = "<stop/>";
|
||||
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
validator.append(input).unwrap();
|
||||
assert!(validator.eof());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_read_stdin_self_closing() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
let input = "<read_stdin/>";
|
||||
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
validator.append(input).unwrap();
|
||||
assert!(validator.eof());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_with_attributes() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
let input = r#"<single timeout="2.5" limit="2048">ls -la</single>"#;
|
||||
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
validator.append(input).unwrap();
|
||||
assert!(validator.eof());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reasoning_with_content() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
let input =
|
||||
"<reasoning>I should explore the file system for interesting files.</reasoning>";
|
||||
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
validator.append(input).unwrap();
|
||||
assert!(validator.eof());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_stdout_with_content() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
let input = "<write_stdout>Hello world!</write_stdout>";
|
||||
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
validator.append(input).unwrap();
|
||||
assert!(validator.eof());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_create_validator() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
let schema_text = std::fs::read_to_string("example_schema.xsd").unwrap();
|
||||
let validator = XmlSchemaValidator::new(&schema_text);
|
||||
assert!(
|
||||
validator.is_ok(),
|
||||
@@ -267,73 +116,121 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_get_element_names() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
let validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
let validator = example_validator();
|
||||
let names = validator.get_element_names();
|
||||
assert!(
|
||||
names.contains(&"reasoning".to_string()),
|
||||
"Element names should contain 'reasoning', got: {:?}",
|
||||
names
|
||||
);
|
||||
assert!(
|
||||
names.contains(&"delete".to_string()),
|
||||
"Element names should contain 'delete', got: {:?}",
|
||||
names
|
||||
);
|
||||
}
|
||||
|
||||
// Test basic XML fragment validation - debugging the "<reasoning>" failure
|
||||
#[test]
|
||||
fn test_append_reasoning_tag() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
let validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
// Test character by character for precise failure point
|
||||
assert!(validator.can_continue_with('<'), "Should accept '<'");
|
||||
// Clone at each step to debug exact point of failure
|
||||
let mut v1 = validator.clone();
|
||||
assert!(v1.append_char('<').is_ok(), "Failed to append '<'");
|
||||
let mut v2 = v1.clone();
|
||||
assert!(v2.append_char('r').is_ok(), "Failed to append 'r'");
|
||||
// Continue with each character in "reasoning>"
|
||||
let mut v3 = v2.clone();
|
||||
for c in "easoning>".chars() {
|
||||
assert!(v3.append_char(c).is_ok(), "Failed to append '{}'", c);
|
||||
fn test_valid_elements() {
|
||||
// Test each element type
|
||||
let valid_elements = [
|
||||
"<reasoning>Test content</reasoning>",
|
||||
"<delete id=\"12345\"/>",
|
||||
"<stop/>",
|
||||
"<single>ls -la</single>",
|
||||
"<repeat>ps -ef</repeat>",
|
||||
"<read_stdin/>",
|
||||
"<write_stdout>Hello world!</write_stdout>",
|
||||
];
|
||||
|
||||
for xml in valid_elements {
|
||||
let mut validator = example_validator();
|
||||
assert!(
|
||||
validator.append(xml).is_ok(),
|
||||
"Should accept valid XML: {}",
|
||||
xml
|
||||
);
|
||||
assert!(
|
||||
validator.eof(),
|
||||
"Should be at EOF after complete element: {}",
|
||||
xml
|
||||
);
|
||||
}
|
||||
// Try the whole string at once
|
||||
let mut v4 = validator.clone();
|
||||
let result = v4.append("<reasoning>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_elements() {
|
||||
// Test various invalid XML constructs
|
||||
let invalid_elements = [
|
||||
"<invalid>", // Unknown element
|
||||
"<reasoning></invalid>", // Mismatched tags
|
||||
"<delete/>", // Missing required attribute
|
||||
"<reasoning><reasoning></reasoning>", // Invalid nesting
|
||||
"<stop id=\"123\"/>", // Attribute not in schema
|
||||
];
|
||||
|
||||
for xml in invalid_elements {
|
||||
let mut validator = example_validator();
|
||||
assert!(
|
||||
validator.append(xml).is_err(),
|
||||
"Should reject invalid XML: {}",
|
||||
xml
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_attributes() {
|
||||
// Test elements with attributes
|
||||
let mut validator = example_validator();
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"Failed to append '<reasoning>': {:?}",
|
||||
result.err()
|
||||
validator.append("<delete id=\"12345\"/>").is_ok(),
|
||||
"Should accept element with required attribute"
|
||||
);
|
||||
|
||||
let mut validator = example_validator();
|
||||
assert!(
|
||||
validator
|
||||
.append("<single timeout=\"2.5\" limit=\"2048\">command</single>")
|
||||
.is_ok(),
|
||||
"Should accept element with multiple attributes"
|
||||
);
|
||||
|
||||
// Test with invalid attribute values
|
||||
let mut validator = example_validator();
|
||||
assert!(
|
||||
validator.append("<delete id=\"\"/>").is_ok(),
|
||||
"Should accept empty attribute values"
|
||||
);
|
||||
}
|
||||
|
||||
// Test validation of multiple fragments
|
||||
#[test]
|
||||
fn test_multiple_append() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
// Test sequential appends
|
||||
assert!(validator.append("<").is_ok(), "Failed to append '<'");
|
||||
fn test_partial_append() {
|
||||
let mut validator = example_validator();
|
||||
|
||||
// Test incremental appending
|
||||
assert!(validator.append("<r").is_ok(), "Should accept partial tag");
|
||||
assert!(
|
||||
validator.append("reasoning").is_ok(),
|
||||
"Failed to append 'reasoning'"
|
||||
validator.append("easoning>").is_ok(),
|
||||
"Should accept continuation"
|
||||
);
|
||||
assert!(validator.append(">").is_ok(), "Failed to append '>'");
|
||||
assert!(validator.append("Hello ").is_ok(), "Should accept content");
|
||||
assert!(
|
||||
validator.append("test content").is_ok(),
|
||||
"Failed to append content"
|
||||
validator.append("world!").is_ok(),
|
||||
"Should accept more content"
|
||||
);
|
||||
assert!(
|
||||
validator.append("</reasoning>").is_ok(),
|
||||
"Failed to append closing tag"
|
||||
"Should accept closing tag"
|
||||
);
|
||||
// Should be at EOF now
|
||||
assert!(validator.eof(), "Should be at EOF after complete XML");
|
||||
assert!(validator.eof(), "Should be at EOF after complete element");
|
||||
}
|
||||
|
||||
// Test token continuation - critical for the token masking issue
|
||||
#[test]
|
||||
fn test_can_continue_with() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
|
||||
// Test at start
|
||||
let validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
let validator = example_validator();
|
||||
|
||||
// At start, only '<' is valid
|
||||
assert!(
|
||||
validator.can_continue_with('<'),
|
||||
"Should accept '<' at start"
|
||||
@@ -342,398 +239,57 @@ mod tests {
|
||||
!validator.can_continue_with('a'),
|
||||
"Should reject 'a' at start"
|
||||
);
|
||||
// Test after opening tag
|
||||
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
|
||||
// After opening a tag
|
||||
let mut validator = example_validator();
|
||||
validator.append("<reasoning>").unwrap();
|
||||
// Any character should be valid inside a mixed content element
|
||||
|
||||
// In mixed content, many characters should be valid
|
||||
assert!(
|
||||
validator.can_continue_with('a'),
|
||||
"Should accept 'a' inside element"
|
||||
);
|
||||
assert!(
|
||||
validator.can_continue_with('1'),
|
||||
"Should accept '1' inside element"
|
||||
"Should accept 'a' in content"
|
||||
);
|
||||
assert!(
|
||||
validator.can_continue_with('<'),
|
||||
"Should accept '<' inside element"
|
||||
"Should accept '<' for closing"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod diagnostic_tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
|
||||
// Simple schema with just the reasoning tag for cleaner debugging
|
||||
fn get_simple_schema() -> String {
|
||||
r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
|
||||
<xs:element name="reasoning">
|
||||
<xs:complexType mixed="true">
|
||||
<xs:sequence>
|
||||
<xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
|
||||
</xs:sequence>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
</xs:schema>"#
|
||||
.to_string()
|
||||
}
|
||||
|
||||
// Test every character in an opening tag one by one
|
||||
#[test]
|
||||
fn test_char_by_char_open_tag() {
|
||||
let schema_text = get_simple_schema();
|
||||
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
|
||||
// Test each character individually
|
||||
assert!(validator.can_continue_with('<'), "Should accept '<'");
|
||||
assert!(validator.append_char('<').is_ok(), "Failed to append '<'");
|
||||
|
||||
assert!(validator.can_continue_with('r'), "Should accept 'r'");
|
||||
assert!(validator.append_char('r').is_ok(), "Failed to append 'r'");
|
||||
|
||||
for c in "easoning".chars() {
|
||||
assert!(validator.can_continue_with(c), "Should accept '{}'", c);
|
||||
assert!(validator.append_char(c).is_ok(), "Failed to append '{}'", c);
|
||||
}
|
||||
|
||||
// This is where many tests are failing - let's check the state before and after
|
||||
println!("State before '>': {:?}", validator.current_tokens);
|
||||
let result = validator.append_char('>');
|
||||
println!("Result of append '>': {:?}", result);
|
||||
if result.is_ok() {
|
||||
println!("State after '>': {:?}", validator.current_tokens);
|
||||
}
|
||||
assert!(result.is_ok(), "Failed to append '>'");
|
||||
}
|
||||
|
||||
// Test the token transitions specifically
|
||||
#[test]
|
||||
fn test_token_transitions() {
|
||||
// Create minimal elements for testing token transitions directly
|
||||
let test_element = Arc::new(schema::XsElement {
|
||||
name: "reasoning".to_string(),
|
||||
text: None,
|
||||
xs_complex_type: schema::XsComplexType {
|
||||
mixed: Some("true".to_string()),
|
||||
text: None,
|
||||
xs_attribute: None,
|
||||
xs_sequence: Some(schema::XsSequence {
|
||||
text: None,
|
||||
xs_any: schema::XsAny {
|
||||
min_occurs: "0".to_string(),
|
||||
max_occurs: "unbounded".to_string(),
|
||||
process_contents: "skip".to_string(),
|
||||
},
|
||||
}),
|
||||
},
|
||||
});
|
||||
|
||||
// Test the transition from ElementOpenName to ElementOpenEnd
|
||||
let element_name =
|
||||
token::ElementOpenName::new(Arc::clone(&test_element), "reasoning".to_string());
|
||||
let tokens = element_name.append('>');
|
||||
assert!(!tokens.is_empty(), "Should produce valid tokens after '>'");
|
||||
println!(
|
||||
"Token after appending '>' to ElementOpenName: {:?}",
|
||||
tokens[0]
|
||||
);
|
||||
|
||||
// Check what type the token is
|
||||
match &tokens[0] {
|
||||
Token::ElementOpenEnd(element_open) => {
|
||||
println!(
|
||||
"Successfully transitioned to ElementOpen: {:?}",
|
||||
element_open
|
||||
);
|
||||
}
|
||||
other => {
|
||||
panic!("Expected ElementOpen, got {:?}", other);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test element self-closing functionality
|
||||
#[test]
|
||||
fn test_element_self_close_transition() {
|
||||
let test_element = Arc::new(schema::XsElement {
|
||||
name: "stop".to_string(),
|
||||
text: None,
|
||||
xs_complex_type: schema::XsComplexType {
|
||||
mixed: None,
|
||||
text: None,
|
||||
xs_attribute: None,
|
||||
xs_sequence: None,
|
||||
},
|
||||
});
|
||||
|
||||
// Test transition from ElementName to ElementSelfClose
|
||||
let element_name =
|
||||
token::ElementOpenName::new(Arc::clone(&test_element), "stop".to_string());
|
||||
let tokens = element_name.append('/');
|
||||
assert!(!tokens.is_empty(), "Should handle '/' after element name");
|
||||
|
||||
match &tokens[0] {
|
||||
Token::ElementSelfClose(element_self_close) => {
|
||||
println!("Successfully transitioned to ElementSelfClose");
|
||||
// Now test transition to EndOfFile
|
||||
let next_tokens = element_self_close.clone().append('>');
|
||||
assert!(!next_tokens.is_empty(), "Should handle '>' after '/'");
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::EndOfFile(_) => println!("Successfully transitioned to EndOfFile"),
|
||||
other => panic!("Expected EndOfFile, got {:?}", other),
|
||||
}
|
||||
}
|
||||
other => panic!("Expected ElementSelfClose, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
// Test whitespace handling
|
||||
#[test]
|
||||
fn test_whitespace_handling() {
|
||||
let schema_text = get_simple_schema();
|
||||
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
|
||||
// Test opening tag with whitespace
|
||||
for c in "<reasoning ".chars() {
|
||||
println!("Appending character: '{}'", c);
|
||||
assert!(validator.append_char(c).is_ok(), "Failed to append '{}'", c);
|
||||
}
|
||||
|
||||
println!("State after '<reasoning ': {:?}", validator.current_tokens);
|
||||
|
||||
// Test completion with >
|
||||
assert!(
|
||||
validator.can_continue_with('>'),
|
||||
"Should accept '>' after whitespace"
|
||||
);
|
||||
assert!(
|
||||
validator.append_char('>').is_ok(),
|
||||
"Failed to append '>' after whitespace"
|
||||
);
|
||||
}
|
||||
|
||||
// Test attribute handling
|
||||
#[test]
|
||||
fn test_attribute_handling() {
|
||||
let schema_text = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
|
||||
<xs:element name="delete">
|
||||
<xs:complexType>
|
||||
<xs:attribute name="id" type="xs:string" use="required"/>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
</xs:schema>"#;
|
||||
|
||||
let mut validator = XmlSchemaValidator::new(schema_text).unwrap();
|
||||
|
||||
// Test opening tag with attribute
|
||||
for c in "<delete ".chars() {
|
||||
assert!(validator.append_char(c).is_ok(), "Failed to append '{}'", c);
|
||||
}
|
||||
|
||||
// Test attribute name
|
||||
for c in "id".chars() {
|
||||
assert!(
|
||||
validator.can_continue_with(c),
|
||||
"Should accept '{}' in attribute name",
|
||||
c
|
||||
);
|
||||
assert!(
|
||||
validator.append_char(c).is_ok(),
|
||||
"Failed to append '{}' in attribute name",
|
||||
c
|
||||
);
|
||||
}
|
||||
|
||||
// Test attribute equals and quotes
|
||||
for c in "=\"123\"".chars() {
|
||||
println!("Testing attribute character: '{}'", c);
|
||||
assert!(
|
||||
validator.can_continue_with(c),
|
||||
"Should accept '{}' in attribute value",
|
||||
c
|
||||
);
|
||||
assert!(
|
||||
validator.append_char(c).is_ok(),
|
||||
"Failed to append '{}' in attribute value",
|
||||
c
|
||||
);
|
||||
}
|
||||
|
||||
// Test closing the self-closing tag
|
||||
println!(
|
||||
"Current state before closing: {:?}",
|
||||
validator.current_tokens
|
||||
);
|
||||
for c in "/>".chars() {
|
||||
assert!(
|
||||
validator.can_continue_with(c),
|
||||
"Should accept '{}' to close tag",
|
||||
c
|
||||
);
|
||||
assert!(
|
||||
validator.append_char(c).is_ok(),
|
||||
"Failed to append '{}' to close tag",
|
||||
c
|
||||
);
|
||||
}
|
||||
|
||||
// Check if we've reached EOF
|
||||
assert!(validator.eof(), "Should be at EOF after self-closing tag");
|
||||
}
|
||||
|
||||
// Test for the specific error patterns seen in the test output
|
||||
#[test]
|
||||
fn test_specific_error_cases() {
|
||||
let schema_text = std::fs::read_to_string("../../action_schema.xsd")
|
||||
.unwrap_or_else(|_| get_simple_schema());
|
||||
|
||||
// Test case 1: Simple opening tag (noticed in `test_element_open`)
|
||||
let input1 = "<reasoning>";
|
||||
let mut validator1 = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
let result1 = validator1.append(input1);
|
||||
println!("Result appending '<reasoning>': {:?}", result1);
|
||||
assert!(result1.is_ok(), "Failed on '<reasoning>'");
|
||||
|
||||
// Test case 2: Element with whitespace (noticed in `test_element_open_whitespace`)
|
||||
let input2 = "<reasoning >";
|
||||
let mut validator2 = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
let result2 = validator2.append(input2);
|
||||
println!("Result appending '<reasoning >': {:?}", result2);
|
||||
assert!(result2.is_ok(), "Failed on '<reasoning >'");
|
||||
|
||||
// Test case 3: Self-closing tags (noticed in `test_read_stdin_self_closing`)
|
||||
let input3 = "<read_stdin/>";
|
||||
let mut validator3 = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
let result3 = validator3.append(input3);
|
||||
println!("Result appending '<read_stdin/>': {:?}", result3);
|
||||
assert!(result3.is_ok(), "Failed on '<read_stdin/>'");
|
||||
|
||||
// Test case 4: Attributes (noticed in `test_single_with_attributes`)
|
||||
let input4 = r#"<single timeout="2.5">"#;
|
||||
let mut validator4 = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
let result4 = validator4.append(input4);
|
||||
println!("Result appending attribute: {:?}", result4);
|
||||
assert!(result4.is_ok(), "Failed on attribute handling");
|
||||
}
|
||||
|
||||
// Test the basic token state machine with a minimal element
|
||||
#[test]
|
||||
fn test_basic_token_operations() {
|
||||
// Create a minimal element for testing
|
||||
let element = Arc::new(schema::XsElement {
|
||||
name: "test".to_string(),
|
||||
text: None,
|
||||
xs_complex_type: schema::XsComplexType {
|
||||
mixed: Some("true".to_string()),
|
||||
text: None,
|
||||
xs_attribute: None,
|
||||
xs_sequence: None,
|
||||
},
|
||||
});
|
||||
|
||||
// Create and test each token type
|
||||
let start_of_file = token::StartOfFile::new(Arc::clone(&element));
|
||||
let tokens = start_of_file.append('<');
|
||||
assert!(!tokens.is_empty(), "StartOfFile should accept '<'");
|
||||
|
||||
let element_start = match &tokens[0] {
|
||||
Token::ElementOpenStart(es) => es.clone(),
|
||||
_ => panic!("Expected ElementStart"),
|
||||
};
|
||||
|
||||
let tokens = element_start.append('t');
|
||||
assert!(!tokens.is_empty(), "ElementStart should accept 't'");
|
||||
|
||||
let element_name = match &tokens[0] {
|
||||
Token::ElementOpenName(en) => en.clone(),
|
||||
_ => panic!("Expected ElementName"),
|
||||
};
|
||||
|
||||
// This is often where things break
|
||||
println!("About to append 'e' to ElementName...");
|
||||
let tokens = element_name.append('e');
|
||||
assert!(!tokens.is_empty(), "ElementName should accept 'e'");
|
||||
|
||||
// Build the rest of the name
|
||||
let mut current_token = tokens[0].clone();
|
||||
for c in "st".chars() {
|
||||
let next_tokens = current_token.append(c);
|
||||
assert!(!next_tokens.is_empty(), "Token should accept '{}'", c);
|
||||
current_token = next_tokens[0].clone();
|
||||
}
|
||||
|
||||
// Now try to append '>'
|
||||
println!("About to append '>' to token: {:?}", current_token);
|
||||
let tokens = current_token.append('>');
|
||||
println!("Result of appending '>': {:?}", tokens);
|
||||
assert!(!tokens.is_empty(), "Token should accept '>'");
|
||||
|
||||
// Verify transition to ElementOpen
|
||||
match &tokens[0] {
|
||||
Token::ElementOpenEnd(_) => println!("Successfully transitioned to ElementOpen"),
|
||||
other => panic!("Expected ElementOpen, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
// This test will help identify issues with token creation for specific elements
|
||||
#[test]
|
||||
fn test_element_specific_issues() {
|
||||
if let Ok(schema_text) = std::fs::read_to_string("../../action_schema.xsd") {
|
||||
let validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
|
||||
// Get all element names in the schema
|
||||
let element_names = validator.get_element_names();
|
||||
println!("Elements in schema: {:?}", element_names);
|
||||
|
||||
// Test each element separately
|
||||
for name in &element_names {
|
||||
println!("\nTesting element: {}", name);
|
||||
|
||||
let mut element_validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
let open_tag = format!("<{}>", name);
|
||||
|
||||
// Test character by character
|
||||
println!("Testing character by character for <{}>", name);
|
||||
let mut progress = String::new();
|
||||
for c in open_tag.chars() {
|
||||
progress.push(c);
|
||||
let result = element_validator.can_continue_with(c);
|
||||
println!(
|
||||
" Can continue with '{}' after '{}': {}",
|
||||
c,
|
||||
&progress[..progress.len() - 1],
|
||||
result
|
||||
);
|
||||
|
||||
if !result {
|
||||
// This is helpful to diagnose where the process fails
|
||||
println!(" ❌ Failed on character '{}' for element '{}'", c, name);
|
||||
break;
|
||||
}
|
||||
|
||||
if let Err(e) = element_validator.append_char(c) {
|
||||
println!(
|
||||
" ❌ Failed to append '{}' for element '{}': {:?}",
|
||||
c, name, e
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Test complete tag at once
|
||||
let mut complete_validator = XmlSchemaValidator::new(&schema_text).unwrap();
|
||||
match complete_validator.append(&open_tag) {
|
||||
Ok(_) => println!(" ✅ Successfully appended <{}>", name),
|
||||
Err(e) => println!(" ❌ Failed to append <{}>: {:?}", name, e),
|
||||
}
|
||||
}
|
||||
} else {
|
||||
println!("Schema file not found, skipping element-specific tests");
|
||||
}
|
||||
|
||||
// Test for characters that form closing tag
|
||||
let mut validator = example_validator();
|
||||
validator.append("<reasoning>content<").unwrap();
|
||||
assert!(
|
||||
validator.can_continue_with('/'),
|
||||
"Should accept '/' for closing"
|
||||
);
|
||||
|
||||
// Invalid continuation after EOF
|
||||
let mut validator = example_validator();
|
||||
validator.append("<stop/>").unwrap();
|
||||
assert!(
|
||||
!validator.can_continue_with('a'),
|
||||
"Should reject chars after EOF"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cloning() {
|
||||
let mut validator = example_validator();
|
||||
validator.append("<reasoning>").unwrap();
|
||||
|
||||
// Clone the validator and make sure they behave independently
|
||||
let mut clone = validator.clone();
|
||||
|
||||
// Modify the clone
|
||||
clone.append("test").unwrap();
|
||||
clone.append("</reasoning>").unwrap();
|
||||
assert!(clone.eof(), "Clone should be at EOF");
|
||||
|
||||
// Original should be unchanged
|
||||
assert!(!validator.eof(), "Original should not be at EOF");
|
||||
|
||||
// Original should still accept more content
|
||||
assert!(validator.append("different").is_ok());
|
||||
assert!(validator.append("</reasoning>").is_ok());
|
||||
assert!(validator.eof(), "Original should now be at EOF");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,7 +112,9 @@ mod tests {
|
||||
#[test]
|
||||
fn test_delete_quote() {
|
||||
Python::with_gil(|py| {
|
||||
py.run(&std::ffi::CString::new(r#"
|
||||
py.run(
|
||||
&std::ffi::CString::new(
|
||||
r#"
|
||||
from transformers import AutoTokenizer
|
||||
from xml_schema_validator import XmlLogitsProcessor
|
||||
import os
|
||||
@@ -128,7 +130,13 @@ processed_scores = processor(input_ids, scores)
|
||||
input_ids = torch.tensor([tokenizer.encode("<delete id=\"")])
|
||||
scores = torch.ones((1, len(tokenizer))) # All tokens have equal probability
|
||||
processed_scores = processor(input_ids, scores)
|
||||
"#).unwrap(), None, None).unwrap();
|
||||
"#,
|
||||
)
|
||||
.unwrap(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,6 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(
|
||||
Clone,
|
||||
Debug,
|
||||
Deserialize,
|
||||
Eq,
|
||||
Hash,
|
||||
PartialEq,
|
||||
Serialize,
|
||||
)]
|
||||
#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
|
||||
pub struct XsSchema {
|
||||
#[serde(rename = "@xmlns:xs")]
|
||||
pub xmlns_xs: String,
|
||||
@@ -20,15 +12,7 @@ pub struct XsSchema {
|
||||
pub xs_element: Vec<XsElement>,
|
||||
}
|
||||
|
||||
#[derive(
|
||||
Clone,
|
||||
Debug,
|
||||
Deserialize,
|
||||
Eq,
|
||||
Hash,
|
||||
PartialEq,
|
||||
Serialize,
|
||||
)]
|
||||
#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
|
||||
pub struct XsElement {
|
||||
#[serde(rename = "@name")]
|
||||
pub name: String,
|
||||
@@ -38,15 +22,7 @@ pub struct XsElement {
|
||||
pub xs_complex_type: XsComplexType,
|
||||
}
|
||||
|
||||
#[derive(
|
||||
Clone,
|
||||
Debug,
|
||||
Deserialize,
|
||||
Eq,
|
||||
Hash,
|
||||
PartialEq,
|
||||
Serialize,
|
||||
)]
|
||||
#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
|
||||
pub struct XsComplexType {
|
||||
#[serde(rename = "@mixed")]
|
||||
pub mixed: Option<String>,
|
||||
@@ -58,15 +34,7 @@ pub struct XsComplexType {
|
||||
pub xs_sequence: Option<XsSequence>,
|
||||
}
|
||||
|
||||
#[derive(
|
||||
Clone,
|
||||
Debug,
|
||||
Deserialize,
|
||||
Eq,
|
||||
Hash,
|
||||
PartialEq,
|
||||
Serialize,
|
||||
)]
|
||||
#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
|
||||
pub struct XsAttribute {
|
||||
#[serde(rename = "@name")]
|
||||
pub name: String,
|
||||
@@ -76,15 +44,7 @@ pub struct XsAttribute {
|
||||
pub xs_attribute_use: String,
|
||||
}
|
||||
|
||||
#[derive(
|
||||
Clone,
|
||||
Debug,
|
||||
Deserialize,
|
||||
Eq,
|
||||
Hash,
|
||||
PartialEq,
|
||||
Serialize,
|
||||
)]
|
||||
#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
|
||||
pub struct XsSequence {
|
||||
#[serde(rename = "$text")]
|
||||
pub text: Option<String>,
|
||||
@@ -92,15 +52,7 @@ pub struct XsSequence {
|
||||
pub xs_any: XsAny,
|
||||
}
|
||||
|
||||
#[derive(
|
||||
Clone,
|
||||
Debug,
|
||||
Deserialize,
|
||||
Eq,
|
||||
Hash,
|
||||
PartialEq,
|
||||
Serialize,
|
||||
)]
|
||||
#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
|
||||
pub struct XsAny {
|
||||
#[serde(rename = "@minOccurs")]
|
||||
pub min_occurs: String,
|
||||
|
||||
@@ -1,102 +1,79 @@
|
||||
use crate::*;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Represents the equals sign after an XML attribute name
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AttributeEquals {
|
||||
element: Arc<schema::XsElement>,
|
||||
used_attributes: Vec<schema::XsAttribute>,
|
||||
}
|
||||
|
||||
impl AttributeEquals {
|
||||
pub fn new(
|
||||
element: Arc<schema::XsElement>,
|
||||
used_attributes: Vec<schema::XsAttribute>,
|
||||
) -> Self {
|
||||
Self {
|
||||
element,
|
||||
used_attributes,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn append(self, c: char) -> Vec<Token> {
|
||||
if c == '"' {
|
||||
// Opening quote starts the attribute value
|
||||
return vec![Token::AttributeValue(AttributeValue::new(
|
||||
Arc::clone(&self.element),
|
||||
self.used_attributes,
|
||||
))];
|
||||
} else if c.is_whitespace() {
|
||||
// Allow whitespace between equals sign and opening quote
|
||||
return vec![Token::AttributeEquals(self)];
|
||||
} else {
|
||||
// Any other character is invalid
|
||||
return vec![];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_attribute_equals_quote() {
|
||||
let values = [
|
||||
"<delete id=\"",
|
||||
"<single timeout=\"",
|
||||
"<single limit=\"",
|
||||
];
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.filter(|token| matches!(token, Token::AttributeValue(_)))
|
||||
.next()
|
||||
.is_some(),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_attribute_equals_with_whitespace() {
|
||||
let values = [
|
||||
"<delete id= \"",
|
||||
"<single timeout = \"",
|
||||
"<single limit \t = \n \"",
|
||||
];
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.filter(|token| matches!(token, Token::AttributeValue(_)))
|
||||
.next()
|
||||
.is_some(),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_after_equals() {
|
||||
let values = [
|
||||
"<delete id=a", // Only quote should be accepted after equals
|
||||
"<single timeout=<", // Invalid character after equals
|
||||
"<single timeout=1", // Unquoted numbers not supported yet
|
||||
];
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect_err(value);
|
||||
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
|
||||
}
|
||||
}
|
||||
}
|
||||
use crate::*;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Represents the equals sign after an XML attribute name
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AttributeEquals {
|
||||
element: Arc<schema::XsElement>,
|
||||
used_attributes: Vec<schema::XsAttribute>,
|
||||
}
|
||||
|
||||
impl AttributeEquals {
|
||||
pub fn new(element: Arc<schema::XsElement>, used_attributes: Vec<schema::XsAttribute>) -> Self {
|
||||
Self {
|
||||
element,
|
||||
used_attributes,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn append(self, c: char) -> Vec<Token> {
|
||||
if c == '"' {
|
||||
// Opening quote starts the attribute value
|
||||
return vec![Token::AttributeValue(AttributeValue::new(
|
||||
Arc::clone(&self.element),
|
||||
self.used_attributes,
|
||||
))];
|
||||
} else if c.is_whitespace() {
|
||||
// Allow whitespace between equals sign and opening quote
|
||||
return vec![Token::AttributeEquals(self)];
|
||||
} else {
|
||||
// Any other character is invalid
|
||||
return vec![];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_valid_attribute_equals() {
|
||||
let values = [
|
||||
"<delete id=\"",
|
||||
"<single timeout=\"",
|
||||
"<single limit=\"",
|
||||
"<delete id= \"", // with space
|
||||
"<single timeout = \"", // with spaces around equals
|
||||
"<single limit \t = \n \"", // with lots of whitespace
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::AttributeValue(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_attribute_equals() {
|
||||
let values = [
|
||||
"<delete id=a", // Only quote should be accepted after equals
|
||||
"<single timeout=<", // Invalid character after equals
|
||||
"<single timeout=1", // Unquoted numbers not supported
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
let result = validator.append(value);
|
||||
assert!(result.is_err(), "Should reject '{value}'");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,11 @@ impl AttributeName {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_with_used(first_char: char, element: Arc<schema::XsElement>, used_attributes: Vec<schema::XsAttribute>) -> Self {
|
||||
pub fn new_with_used(
|
||||
first_char: char,
|
||||
element: Arc<schema::XsElement>,
|
||||
used_attributes: Vec<schema::XsAttribute>,
|
||||
) -> Self {
|
||||
Self {
|
||||
buffer: first_char.to_string(),
|
||||
element,
|
||||
@@ -26,7 +30,33 @@ impl AttributeName {
|
||||
}
|
||||
}
|
||||
|
||||
// Helper method to check if the current buffer matches any attribute that's already used
|
||||
fn is_duplicate_attribute(&self) -> bool {
|
||||
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
|
||||
// Find any attribute that starts with our current buffer
|
||||
for attr in attributes
|
||||
.iter()
|
||||
.filter(|a| a.name.starts_with(&self.buffer))
|
||||
{
|
||||
// Check if this attribute or any prefix of it has already been used
|
||||
if self
|
||||
.used_attributes
|
||||
.iter()
|
||||
.any(|used| used.name == attr.name)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub fn append(mut self, c: char) -> Vec<Token> {
|
||||
// First check if we're already working with a duplicate attribute
|
||||
if self.is_duplicate_attribute() {
|
||||
return vec![]; // Return empty vector to indicate invalid state
|
||||
}
|
||||
|
||||
if c == '=' {
|
||||
// Check if we've matched a valid attribute name
|
||||
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
|
||||
@@ -34,14 +64,9 @@ impl AttributeName {
|
||||
.iter()
|
||||
.find(|attribute| attribute.name == self.buffer)
|
||||
{
|
||||
// Check if this attribute has already been used
|
||||
if self.used_attributes.contains(&attribute) {
|
||||
return vec![];
|
||||
}
|
||||
|
||||
// Add this attribute to the used set
|
||||
self.used_attributes.push(attribute.clone());
|
||||
|
||||
|
||||
return vec![Token::AttributeEquals(AttributeEquals::new(
|
||||
Arc::clone(&self.element),
|
||||
self.used_attributes,
|
||||
@@ -54,13 +79,9 @@ impl AttributeName {
|
||||
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
|
||||
if attributes
|
||||
.iter()
|
||||
.any(|attribute| attribute.name == self.buffer)
|
||||
.find(|attribute| attribute.name == self.buffer)
|
||||
.is_some()
|
||||
{
|
||||
// Check if this attribute has already been used
|
||||
if self.used_attributes.iter().filter(|attribute| attribute.name == self.buffer).next().is_some() {
|
||||
return vec![];
|
||||
}
|
||||
|
||||
return vec![Token::AttributeName(self)];
|
||||
}
|
||||
}
|
||||
@@ -68,21 +89,29 @@ impl AttributeName {
|
||||
} else {
|
||||
// Add character to buffer and check if we're still building a valid attribute name
|
||||
let new_buffer = self.buffer.clone() + &c.to_string();
|
||||
|
||||
|
||||
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
|
||||
// Check if any attribute name starts with our buffer
|
||||
if attributes
|
||||
.iter()
|
||||
.any(|attribute| attribute.name.starts_with(&new_buffer))
|
||||
{
|
||||
return vec![Token::AttributeName(AttributeName {
|
||||
// Create a new token with the updated buffer
|
||||
let new_token = AttributeName {
|
||||
buffer: new_buffer,
|
||||
element: Arc::clone(&self.element),
|
||||
used_attributes: self.used_attributes,
|
||||
})];
|
||||
used_attributes: self.used_attributes.clone(),
|
||||
};
|
||||
|
||||
// Check if this would be a duplicate
|
||||
if new_token.is_duplicate_attribute() {
|
||||
return vec![];
|
||||
}
|
||||
|
||||
return vec![Token::AttributeName(new_token)];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Character doesn't match what we expect for this attribute name
|
||||
return vec![];
|
||||
}
|
||||
@@ -96,33 +125,41 @@ mod tests {
|
||||
#[test]
|
||||
fn test_valid_attribute_name() {
|
||||
let values = [
|
||||
"<delete id=",
|
||||
"<single timeout=",
|
||||
"<single limit=",
|
||||
"<delete id",
|
||||
"<single timeout",
|
||||
"<single limit",
|
||||
"<delete id ", // with trailing space
|
||||
"<single timeout ", // with trailing spaces
|
||||
"<single limit\t", // with tab
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
|
||||
// Either still building the attribute name or reached equals sign
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.filter(|token| matches!(token, Token::AttributeEquals(_)))
|
||||
.next()
|
||||
.is_some(),
|
||||
validator.current_tokens.iter().any(|token| matches!(
|
||||
token,
|
||||
Token::AttributeName(_) | Token::AttributeEquals(_)
|
||||
)),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_attribute_name_with_whitespace() {
|
||||
fn test_valid_attribute_name_with_equals() {
|
||||
let values = [
|
||||
"<delete id =",
|
||||
"<single timeout =",
|
||||
"<single limit\t=",
|
||||
"<delete id=",
|
||||
"<single timeout=",
|
||||
"<single limit=",
|
||||
"<delete id =", // with space before equals
|
||||
"<single timeout =", // with spaces before equals
|
||||
"<single limit\t=", // with tab before equals
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
@@ -131,9 +168,7 @@ mod tests {
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.filter(|token| matches!(token, Token::AttributeEquals(_)))
|
||||
.next()
|
||||
.is_some(),
|
||||
.any(|token| matches!(token, Token::AttributeEquals(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
@@ -144,22 +179,32 @@ mod tests {
|
||||
let values = [
|
||||
"<delete invalid=",
|
||||
"<single wrongattr=",
|
||||
"<delete xyz",
|
||||
"<single badname",
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect_err(value);
|
||||
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
|
||||
let result = validator.append(value);
|
||||
assert!(result.is_err(), "Should reject '{value}'");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_duplicate_attribute() {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
// First attribute is valid
|
||||
validator.append("<delete id=\"123\" ").expect("First attribute should be accepted");
|
||||
|
||||
// Try to add the same attribute again
|
||||
let result = validator.append("id=");
|
||||
assert!(result.is_err(), "Duplicate attribute should be rejected");
|
||||
let values = [
|
||||
"<delete id=\"123\" id",
|
||||
"<single timeout=\"2.5\" timeout",
|
||||
"<single limit=\"100\" limit",
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
let result = validator.append(value);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"Should reject duplicate attribute in '{value}'"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,10 +13,7 @@ pub struct AttributeValue {
|
||||
|
||||
impl AttributeValue {
|
||||
/// Create a new AttributeValue with a list of attributes already set
|
||||
pub fn new(
|
||||
element: Arc<schema::XsElement>,
|
||||
attributes_set: Vec<schema::XsAttribute>,
|
||||
) -> Self {
|
||||
pub fn new(element: Arc<schema::XsElement>, attributes_set: Vec<schema::XsAttribute>) -> Self {
|
||||
Self {
|
||||
element,
|
||||
buffer: "\"".to_string(),
|
||||
@@ -102,7 +99,9 @@ impl AttributeValue {
|
||||
if '/' == c {
|
||||
tokens.push(Token::ElementSelfClose(ElementSelfClose::new()));
|
||||
} else if '>' == c {
|
||||
tokens.push(Token::ElementOpenEnd(ElementOpenEnd::new(self.element.clone())));
|
||||
tokens.push(Token::ElementOpenEnd(ElementOpenEnd::new(
|
||||
self.element.clone(),
|
||||
)));
|
||||
}
|
||||
}
|
||||
if self.buffer.ends_with(char::is_whitespace) && !all_attributes_set {
|
||||
@@ -121,6 +120,36 @@ impl AttributeValue {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_valid_attribute_value() {
|
||||
let values = [
|
||||
"<delete id=\"123\"",
|
||||
"<delete id=\"abc_123\"",
|
||||
"<single timeout=\"2.5\"",
|
||||
"<single limit=\"100\"",
|
||||
"<delete id=\" \"", // space as value
|
||||
"<delete id=\" \"", // multiple spaces as value
|
||||
"<delete id=\"\"", // empty value
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
|
||||
// Should be in AttributeValue state or ready for next attribute/closing
|
||||
assert!(
|
||||
validator.current_tokens.iter().any(|token| matches!(
|
||||
token,
|
||||
Token::AttributeValue(_)
|
||||
| Token::ElementSelfClose(_)
|
||||
| Token::ElementOpenEnd(_)
|
||||
)),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unfinished_attribute_value() {
|
||||
let values = [
|
||||
@@ -130,12 +159,12 @@ mod tests {
|
||||
"<single timeout=\"1.",
|
||||
"<single limit=\"50",
|
||||
];
|
||||
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
|
||||
|
||||
// Should still be in AttributeValue state
|
||||
assert!(
|
||||
validator
|
||||
@@ -147,28 +176,13 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_type_validation() {
|
||||
// Test integer validation
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append("<single limit=\"123\"").expect("Valid integer");
|
||||
|
||||
// Test float validation
|
||||
validator = crate::tests::example_validator();
|
||||
validator.append("<single timeout=\"1.5\"").expect("Valid float");
|
||||
|
||||
// Test string validation
|
||||
validator = crate::tests::example_validator();
|
||||
validator.append("<delete id=\"abc123\"").expect("Valid string");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_attributes() {
|
||||
let values = [
|
||||
"<single limit=\"100\" timeout=\"1.5\"",
|
||||
"<single timeout=\"0.5\" limit=\"200\"",
|
||||
];
|
||||
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
@@ -177,17 +191,16 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_attribute_value_escape_chars() {
|
||||
// Test handling of escaped characters in attribute values
|
||||
fn test_attribute_value_with_escape_chars() {
|
||||
let values = [
|
||||
"<delete id=\"escaped"quote\"",
|
||||
"<delete id=\"escaped&amp\"",
|
||||
];
|
||||
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,111 +39,88 @@ impl ElementCloseName {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::schema;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn create_test_element() -> Arc<schema::XsElement> {
|
||||
let element = schema::XsElement {
|
||||
name: "reasoning".to_string(),
|
||||
text: None,
|
||||
xs_complex_type: schema::XsComplexType {
|
||||
mixed: Some("true".to_string()),
|
||||
text: None,
|
||||
xs_attribute: None,
|
||||
xs_sequence: Some(schema::XsSequence {
|
||||
text: None,
|
||||
xs_any: schema::XsAny {
|
||||
min_occurs: "0".to_string(),
|
||||
max_occurs: "unbounded".to_string(),
|
||||
process_contents: "skip".to_string(),
|
||||
},
|
||||
}),
|
||||
},
|
||||
};
|
||||
Arc::new(element)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_close_name_partial() {
|
||||
let element = create_test_element();
|
||||
let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
|
||||
fn test_valid_element_close_name() {
|
||||
let values = [
|
||||
"<reasoning>content</r",
|
||||
"<reasoning>content</re",
|
||||
"<reasoning>content</rea",
|
||||
"<reasoning>content</reasoning",
|
||||
"<single>script</single",
|
||||
"<write_stdout>text</write_stdout",
|
||||
];
|
||||
|
||||
// Test with next character in name
|
||||
let next_tokens = token.append('e');
|
||||
assert!(
|
||||
!next_tokens.is_empty(),
|
||||
"Should accept next character in name"
|
||||
);
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::ElementCloseName(name) => {
|
||||
assert_eq!(name.buffer, "re", "Buffer should contain 're'");
|
||||
}
|
||||
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::ElementCloseName(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_close_name_complete() {
|
||||
let element = create_test_element();
|
||||
let token = ElementCloseName::new(Arc::clone(&element), "reasonin".to_string());
|
||||
fn test_element_close_name_with_whitespace() {
|
||||
let values = [
|
||||
"<reasoning>content</reasoning ",
|
||||
"<single>script</single\t",
|
||||
"<write_stdout>text</write_stdout ",
|
||||
];
|
||||
|
||||
// Test with last character of name
|
||||
let next_tokens = token.append('g');
|
||||
assert!(
|
||||
!next_tokens.is_empty(),
|
||||
"Should accept last character of name"
|
||||
);
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::ElementCloseName(name) => {
|
||||
assert_eq!(
|
||||
name.buffer, "reasoning",
|
||||
"Buffer should contain 'reasoning'"
|
||||
);
|
||||
}
|
||||
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
|
||||
}
|
||||
|
||||
// Test with > after complete name
|
||||
let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
|
||||
let next_tokens = token.append('>');
|
||||
assert!(
|
||||
!next_tokens.is_empty(),
|
||||
"Should accept '>' after complete name"
|
||||
);
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::EndOfFile(_) => (),
|
||||
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::ElementCloseName(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_close_name_whitespace() {
|
||||
let element = create_test_element();
|
||||
let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
|
||||
fn test_complete_element_close() {
|
||||
let values = [
|
||||
"<reasoning>content</reasoning>",
|
||||
"<single>script</single>",
|
||||
"<write_stdout>text</write_stdout>",
|
||||
];
|
||||
|
||||
// Test with whitespace after complete name
|
||||
let next_tokens = token.append(' ');
|
||||
assert!(
|
||||
!next_tokens.is_empty(),
|
||||
"Should accept whitespace after complete name"
|
||||
);
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::ElementCloseName(_) => (),
|
||||
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::EndOfFile(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_close_name_invalid() {
|
||||
let element = create_test_element();
|
||||
let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
|
||||
fn test_invalid_element_close_name() {
|
||||
let values = [
|
||||
"<reasoning>content</invalid",
|
||||
"<single>script</wrong",
|
||||
"<reasoning>text</reasoning_wrong",
|
||||
];
|
||||
|
||||
// Test with invalid next character
|
||||
let next_tokens = token.append('x');
|
||||
assert!(next_tokens.is_empty(), "Should reject invalid character");
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
let result = validator.append(value);
|
||||
assert!(result.is_err(), "Should reject '{value}'");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,74 +37,85 @@ impl ElementCloseStart {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::schema;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn create_test_element() -> Arc<schema::XsElement> {
|
||||
let element = schema::XsElement {
|
||||
name: "reasoning".to_string(),
|
||||
text: None,
|
||||
xs_complex_type: schema::XsComplexType {
|
||||
mixed: Some("true".to_string()),
|
||||
text: None,
|
||||
xs_attribute: None,
|
||||
xs_sequence: Some(schema::XsSequence {
|
||||
text: None,
|
||||
xs_any: schema::XsAny {
|
||||
min_occurs: "0".to_string(),
|
||||
max_occurs: "unbounded".to_string(),
|
||||
process_contents: "skip".to_string(),
|
||||
},
|
||||
}),
|
||||
},
|
||||
};
|
||||
Arc::new(element)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_close_start_slash() {
|
||||
let element = create_test_element();
|
||||
let token = ElementCloseStart::new(Arc::clone(&element));
|
||||
fn test_valid_element_close_start() {
|
||||
let values = [
|
||||
"<reasoning>content<",
|
||||
"<single>script<",
|
||||
"<write_stdout>text<",
|
||||
];
|
||||
|
||||
// Test with slash
|
||||
let next_tokens = token.append('/');
|
||||
assert!(!next_tokens.is_empty(), "Should accept '/'");
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::ElementCloseStart(start) => {
|
||||
assert!(start.has_slash, "Should mark as having slash");
|
||||
}
|
||||
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::ElementCloseStart(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_close_start_element_name() {
|
||||
let element = create_test_element();
|
||||
let mut token = ElementCloseStart::new(Arc::clone(&element));
|
||||
token.has_slash = true;
|
||||
fn test_valid_element_close_start_with_slash() {
|
||||
let values = [
|
||||
"<reasoning>content</",
|
||||
"<single>script</",
|
||||
"<write_stdout>text</",
|
||||
];
|
||||
|
||||
// Test with first character of element name
|
||||
let next_tokens = token.append('r');
|
||||
assert!(
|
||||
!next_tokens.is_empty(),
|
||||
"Should accept first letter of element name"
|
||||
);
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::ElementCloseName(_) => (),
|
||||
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::ElementCloseStart(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_close_start_invalid_name() {
|
||||
let element = create_test_element();
|
||||
let mut token = ElementCloseStart::new(Arc::clone(&element));
|
||||
token.has_slash = true;
|
||||
fn test_valid_element_name_start_after_slash() {
|
||||
let values = [
|
||||
"<reasoning>content</r",
|
||||
"<single>script</s",
|
||||
"<write_stdout>text</w",
|
||||
];
|
||||
|
||||
// Test with invalid first character of element name
|
||||
let next_tokens = token.append('x');
|
||||
assert!(next_tokens.is_empty(), "Should reject invalid first letter");
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::ElementCloseName(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_element_close_start() {
|
||||
let values = [
|
||||
"<reasoning>content<x", // Invalid character after
|
||||
"<single>script<>", // Missing slash
|
||||
"<write_stdout>text</x", // Invalid character after </
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
let result = validator.append(value);
|
||||
assert!(result.is_err(), "Should reject '{value}'");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,21 +9,24 @@ pub struct ElementOpenEnd {
|
||||
|
||||
impl ElementOpenEnd {
|
||||
pub fn new(element: Arc<schema::XsElement>) -> Self {
|
||||
Self {
|
||||
element,
|
||||
}
|
||||
Self { element }
|
||||
}
|
||||
|
||||
pub fn append(self, c: char) -> Vec<Token> {
|
||||
if '<' == c {
|
||||
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
|
||||
vec![Token::ElementCloseStart(ElementCloseStart::new(
|
||||
Arc::clone(&self.element),
|
||||
))]
|
||||
} else {
|
||||
// Check if this element allows mixed content
|
||||
let is_mixed = self.element.xs_complex_type.mixed
|
||||
let is_mixed = self
|
||||
.element
|
||||
.xs_complex_type
|
||||
.mixed
|
||||
.as_ref()
|
||||
.map(|val| val == "true")
|
||||
.unwrap_or(false);
|
||||
|
||||
|
||||
if is_mixed || c.is_whitespace() {
|
||||
// Allow text content for mixed elements, or whitespace for any element
|
||||
vec![Token::TextContent(TextContent::new(self.element))]
|
||||
@@ -38,126 +41,87 @@ impl ElementOpenEnd {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::schema;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn create_test_element() -> Arc<schema::XsElement> {
|
||||
let element = schema::XsElement {
|
||||
name: "reasoning".to_string(),
|
||||
text: None,
|
||||
xs_complex_type: schema::XsComplexType {
|
||||
mixed: Some("true".to_string()),
|
||||
text: None,
|
||||
xs_attribute: None,
|
||||
xs_sequence: Some(schema::XsSequence {
|
||||
text: None,
|
||||
xs_any: schema::XsAny {
|
||||
min_occurs: "0".to_string(),
|
||||
max_occurs: "unbounded".to_string(),
|
||||
process_contents: "skip".to_string(),
|
||||
},
|
||||
}),
|
||||
},
|
||||
};
|
||||
Arc::new(element)
|
||||
}
|
||||
|
||||
fn create_element_non_mixed() -> Arc<schema::XsElement> {
|
||||
let element = schema::XsElement {
|
||||
name: "element".to_string(),
|
||||
text: None,
|
||||
xs_complex_type: schema::XsComplexType {
|
||||
mixed: None, // Not mixed content
|
||||
text: None,
|
||||
xs_attribute: None,
|
||||
xs_sequence: Some(schema::XsSequence {
|
||||
text: None,
|
||||
xs_any: schema::XsAny {
|
||||
min_occurs: "0".to_string(),
|
||||
max_occurs: "unbounded".to_string(),
|
||||
process_contents: "skip".to_string(),
|
||||
},
|
||||
}),
|
||||
},
|
||||
};
|
||||
Arc::new(element)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_append_text_in_mixed_content() {
|
||||
let element = create_test_element();
|
||||
let token = ElementOpenEnd::new(Arc::clone(&element));
|
||||
fn test_valid_element_open_end() {
|
||||
let values = [
|
||||
"<reasoning>",
|
||||
"<write_stdout>",
|
||||
"<single>",
|
||||
"<reasoning >", // with space
|
||||
"<single >", // with multiple spaces
|
||||
];
|
||||
|
||||
// Test appending text in mixed content element
|
||||
let next_tokens = token.append('a');
|
||||
assert!(
|
||||
!next_tokens.is_empty(),
|
||||
"Should allow text content in mixed element"
|
||||
);
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::TextContent(_) => (),
|
||||
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::ElementOpenEnd(_) | Token::TextContent(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_append_closing_tag_start() {
|
||||
let element = create_test_element();
|
||||
let token = ElementOpenEnd::new(Arc::clone(&element));
|
||||
fn test_valid_text_content_after_open() {
|
||||
let values = [
|
||||
"<reasoning>text",
|
||||
"<write_stdout>Hello world",
|
||||
"<single>ls -la",
|
||||
"<reasoning> text with spaces ",
|
||||
];
|
||||
|
||||
// Test starting a closing tag
|
||||
let next_tokens = token.append('<');
|
||||
assert!(!next_tokens.is_empty(), "Should allow closing tag start");
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::ElementCloseStart(_) => (),
|
||||
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::TextContent(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_append_whitespace_in_mixed() {
|
||||
let element = create_test_element();
|
||||
let token = ElementOpenEnd::new(Arc::clone(&element));
|
||||
fn test_valid_close_start_after_open() {
|
||||
let values = ["<reasoning><", "<write_stdout><", "<single><"];
|
||||
|
||||
// Test whitespace in mixed content
|
||||
let next_tokens = token.append(' ');
|
||||
assert!(
|
||||
!next_tokens.is_empty(),
|
||||
"Should allow whitespace in mixed content"
|
||||
);
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::TextContent(_) => (),
|
||||
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::ElementCloseStart(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_append_in_non_mixed() {
|
||||
let element = create_element_non_mixed();
|
||||
let token = ElementOpenEnd::new(Arc::clone(&element));
|
||||
fn test_invalid_nested_elements() {
|
||||
let values = [
|
||||
"<reasoning><reasoning>",
|
||||
"<write_stdout><single>",
|
||||
"<single><reasoning>",
|
||||
];
|
||||
|
||||
// Test appending text in non-mixed content (should reject)
|
||||
let next_tokens = token.clone().append('a');
|
||||
assert!(
|
||||
next_tokens.is_empty(),
|
||||
"Should reject text in non-mixed element"
|
||||
);
|
||||
|
||||
// Test whitespace (should accept leading to closing tag)
|
||||
let next_tokens = token.clone().append(' ');
|
||||
assert!(
|
||||
!next_tokens.is_empty(),
|
||||
"Should allow whitespace in non-mixed"
|
||||
);
|
||||
|
||||
// Test closing tag start
|
||||
let next_tokens = token.append('<');
|
||||
assert!(
|
||||
!next_tokens.is_empty(),
|
||||
"Should allow closing tag start in non-mixed"
|
||||
);
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
let result = validator.append(value);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"Should reject nested elements in '{value}'"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,22 +44,31 @@ impl ElementOpenName {
|
||||
if !self.buffer.contains(&self.element.name) {
|
||||
return vec![];
|
||||
}
|
||||
// Check if the element has attributes
|
||||
// Check if this element type allows self-closing
|
||||
// Elements with mixed content or those that expect content should not be self-closing
|
||||
let is_mixed = self
|
||||
.element
|
||||
.xs_complex_type
|
||||
.mixed
|
||||
.as_ref()
|
||||
.map(|val| val == "true")
|
||||
.unwrap_or(false);
|
||||
// Elements with mixed content should not be self-closing
|
||||
if is_mixed {
|
||||
return vec![];
|
||||
}
|
||||
// Check if the element has required attributes
|
||||
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
|
||||
// Check if they are required
|
||||
// Check if any required attributes are missing
|
||||
if attributes
|
||||
.iter()
|
||||
.filter(|attribute| attribute.xs_attribute_use == "required")
|
||||
.next()
|
||||
.is_some()
|
||||
.any(|attribute| attribute.xs_attribute_use == "required")
|
||||
{
|
||||
return vec![];
|
||||
} else {
|
||||
return vec![Token::ElementSelfClose(ElementSelfClose::new())];
|
||||
}
|
||||
} else {
|
||||
return vec![Token::ElementSelfClose(ElementSelfClose::new())];
|
||||
}
|
||||
// If we get here, self-closing is allowed
|
||||
return vec![Token::ElementSelfClose(ElementSelfClose::new())];
|
||||
} else if c.is_whitespace() {
|
||||
// Check if we've matched the full element name
|
||||
if self.buffer.contains(&self.element.name) {
|
||||
@@ -114,12 +123,20 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_attribute_name() {
|
||||
fn test_valid_element_open_name() {
|
||||
let values = [
|
||||
"<delete i", // id
|
||||
"<single t", // timeout
|
||||
"<single l", // limit
|
||||
"<r", // partial reasoning
|
||||
"<re", // partial reasoning
|
||||
"<reasoning", // complete name
|
||||
"<s", // partial stop/single
|
||||
"<si", // partial single
|
||||
"<sin", // partial single
|
||||
"<single", // complete name
|
||||
"<d", // partial delete
|
||||
"<de", // partial delete
|
||||
"<delete", // complete name
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
@@ -128,17 +145,23 @@ mod tests {
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.filter(|token| matches!(token, Token::AttributeName(_)))
|
||||
.next()
|
||||
.is_some(),
|
||||
.any(|token| matches!(token, Token::ElementOpenName(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_valid_self_closing() {
|
||||
let values = ["<stop/", "<stop /"];
|
||||
fn test_valid_element_name_with_closing() {
|
||||
let values = [
|
||||
"<reasoning>",
|
||||
"<stop>",
|
||||
"<single>",
|
||||
"<delete id=\"123\">",
|
||||
"<reasoning >", // with space
|
||||
"<stop >", // with spaces
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
@@ -147,17 +170,21 @@ mod tests {
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.filter(|token| matches!(token, Token::ElementSelfClose(_)))
|
||||
.next()
|
||||
.is_some(),
|
||||
.any(|token| matches!(token, Token::ElementOpenEnd(_) | Token::TextContent(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_valid_closing() {
|
||||
let values = ["<stop>", "<stop >", "<reasoning>", "<reasoning >"];
|
||||
fn test_valid_element_name_with_self_closing() {
|
||||
let values = [
|
||||
"<stop/>",
|
||||
"<read_stdin/>",
|
||||
"<stop />", // with space
|
||||
"<read_stdin />", // with spaces
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
@@ -166,21 +193,71 @@ mod tests {
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.filter(|token| matches!(token, Token::ElementOpenEnd(_)))
|
||||
.next()
|
||||
.is_some(),
|
||||
.any(|token| matches!(token, Token::EndOfFile(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_closing() {
|
||||
let values = ["<delete/", "<delete /", "<delete>", "<delete >"];
|
||||
fn test_valid_element_name_with_attribute() {
|
||||
let values = [
|
||||
"<delete id",
|
||||
"<single timeout",
|
||||
"<single limit",
|
||||
"<delete id=",
|
||||
"<delete id=\"123\"",
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect_err(value);
|
||||
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
|
||||
// Should be in AttributeName, AttributeEquals, or AttributeValue state
|
||||
assert!(
|
||||
validator.current_tokens.iter().any(|token| matches!(
|
||||
token,
|
||||
Token::AttributeName(_) | Token::AttributeEquals(_) | Token::AttributeValue(_)
|
||||
)),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_element_name() {
|
||||
let values = [
|
||||
"<invalid", // Non-existent element
|
||||
"<wrongelement", // Non-existent element
|
||||
"<Reasoning", // Wrong case
|
||||
"<DELETE", // Wrong case
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
let result = validator.append(value);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"Should reject invalid element name in '{value}'"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_element_self_closing() {
|
||||
let values = [
|
||||
"<reasoning/>", // Mixed element can't self close
|
||||
"<delete/>", // Missing required attribute
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
let result = validator.append(value);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"Should reject invalid self-closing in '{value}'"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,13 +29,14 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_accept_valid_element_names() {
|
||||
fn test_valid_element_open_start() {
|
||||
let values = [
|
||||
"<d", // delete
|
||||
"<s", // stop / single
|
||||
"<r", // repeat / reasoning / read_stdin
|
||||
"<w", // write_stdout
|
||||
"<d", // start of delete
|
||||
"<s", // start of stop/single
|
||||
"<r", // start of reasoning/repeat/read_stdin
|
||||
"<w", // start of write_stdout
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
@@ -44,21 +45,56 @@ mod tests {
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.filter(|token| matches!(token, Token::ElementOpenName(_)))
|
||||
.next()
|
||||
.is_some(),
|
||||
.any(|token| matches!(token, Token::ElementOpenName(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_not_accept_other() {
|
||||
let values = ["< ", "<a", "</"];
|
||||
fn test_valid_full_element_names() {
|
||||
let values = [
|
||||
"<delete",
|
||||
"<stop",
|
||||
"<single",
|
||||
"<repeat",
|
||||
"<reasoning",
|
||||
"<read_stdin",
|
||||
"<write_stdout",
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect_err(value);
|
||||
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::ElementOpenName(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_element_open_start() {
|
||||
let values = [
|
||||
"< ", // Space after
|
||||
"<a", // Invalid element name start
|
||||
"<1", // Number after
|
||||
"<_", // Underscore after
|
||||
"</", // Closing tag not allowed at start
|
||||
"<.", // Invalid character after
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
let result = validator.append(value);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"Should reject invalid element start in '{value}'"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,8 +24,14 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_valid() {
|
||||
let values = ["<stop/>"];
|
||||
fn test_valid_element_self_close() {
|
||||
let values = [
|
||||
"<stop/>",
|
||||
"<read_stdin/>",
|
||||
"<stop />", // with space before /
|
||||
"<read_stdin />", // with space before /
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
@@ -34,21 +40,52 @@ mod tests {
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.filter(|token| matches!(token, Token::EndOfFile(_)))
|
||||
.next()
|
||||
.is_some(),
|
||||
.any(|token| matches!(token, Token::EndOfFile(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid() {
|
||||
let values = ["<stop/?", "<stop/a", "<stop/ "];
|
||||
fn test_valid_element_with_attribute_self_close() {
|
||||
let values = [
|
||||
"<delete id=\"123\"/>",
|
||||
"<single timeout=\"1.5\"/>",
|
||||
"<single limit=\"100\"/>",
|
||||
"<delete id=\"abc\" />", // with space before /
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect_err(value);
|
||||
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::EndOfFile(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_element_self_close() {
|
||||
let values = [
|
||||
"<stop/a", // Invalid character after /
|
||||
"<stop/?", // Invalid character after /
|
||||
"<stop/ >", // Space between / and >
|
||||
"<delete/>", // Missing required attribute
|
||||
"<read_stdin>/>", // > in wrong position
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
let result = validator.append(value);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"Should reject invalid self-closing in '{value}'"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,11 +19,54 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_end_of_file_any_char() {
|
||||
let token = EndOfFile::new();
|
||||
fn test_complete_valid_elements_reach_eof() {
|
||||
let values = [
|
||||
"<stop/>",
|
||||
"<read_stdin/>",
|
||||
"<reasoning>test</reasoning>",
|
||||
"<single>command</single>",
|
||||
"<delete id=\"123\"/>",
|
||||
"<write_stdout>Hello world</write_stdout>",
|
||||
];
|
||||
|
||||
// Test with any character (should reject)
|
||||
let next_tokens = token.append('a');
|
||||
assert!(next_tokens.is_empty(), "Should reject any character at EOF");
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::EndOfFile(_))),
|
||||
"{value}"
|
||||
);
|
||||
|
||||
// Also verify that eof() method returns true
|
||||
assert!(validator.eof(), "eof() should return true for '{value}'");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_continuation_after_eof() {
|
||||
let valid_values = [
|
||||
"<stop/>",
|
||||
"<reasoning>test</reasoning>",
|
||||
"<delete id=\"123\"/>",
|
||||
];
|
||||
|
||||
for value in valid_values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
|
||||
// Try to append more characters after EOF
|
||||
let continuations = ["a", " ", "<", ">"];
|
||||
for cont in continuations {
|
||||
let result = validator.append(cont);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"Should reject '{cont}' after EOF in '{value}'"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,220 +66,3 @@ impl Token {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::schema;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub fn create_test_element() -> Arc<schema::XsElement> {
|
||||
let element = schema::XsElement {
|
||||
name: "reasoning".to_string(),
|
||||
text: None,
|
||||
xs_complex_type: schema::XsComplexType {
|
||||
mixed: Some("true".to_string()),
|
||||
text: None,
|
||||
xs_attribute: None,
|
||||
xs_sequence: Some(schema::XsSequence {
|
||||
text: None,
|
||||
xs_any: schema::XsAny {
|
||||
min_occurs: "0".to_string(),
|
||||
max_occurs: "unbounded".to_string(),
|
||||
process_contents: "skip".to_string(),
|
||||
},
|
||||
}),
|
||||
},
|
||||
};
|
||||
Arc::new(element)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_element_start_transition() {
|
||||
let element = create_test_element();
|
||||
let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
|
||||
|
||||
// Should accept first char of element name
|
||||
let next_tokens = token.append('r');
|
||||
assert_eq!(next_tokens.len(), 1, "Should have one transition");
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::ElementOpenName(_) => (),
|
||||
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
|
||||
}
|
||||
|
||||
// Should reject invalid char
|
||||
let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
|
||||
let next_tokens = token.append('x');
|
||||
assert_eq!(next_tokens.len(), 0, "Should reject 'x'");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_element_name_transition() {
|
||||
let element = create_test_element();
|
||||
let token =
|
||||
Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()));
|
||||
|
||||
// Continue building name
|
||||
let next_tokens = token.append('e');
|
||||
assert_eq!(next_tokens.len(), 1, "Should have one transition");
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::ElementOpenName(_) => (),
|
||||
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
|
||||
}
|
||||
|
||||
// Build complete name and end with >
|
||||
// Remove the unused buffer
|
||||
let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(
|
||||
&element,
|
||||
)))];
|
||||
|
||||
for c in "reasoning>".chars() {
|
||||
let mut new_tokens = Vec::new();
|
||||
for token in current_tokens {
|
||||
new_tokens.append(&mut token.append(c));
|
||||
}
|
||||
current_tokens = new_tokens;
|
||||
|
||||
assert!(
|
||||
!current_tokens.is_empty(),
|
||||
"Should have valid transition after appending '{}'",
|
||||
c
|
||||
);
|
||||
}
|
||||
|
||||
// Check final state
|
||||
assert_eq!(current_tokens.len(), 1, "Should have one final transition");
|
||||
match ¤t_tokens[0] {
|
||||
Token::ElementOpenEnd(_) => (),
|
||||
_ => panic!("Expected ElementOpen, got {:?}", current_tokens[0]),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_element_open_end_transition() {
|
||||
let element = create_test_element();
|
||||
let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
|
||||
|
||||
// Should accept text content in mixed element
|
||||
let next_tokens = token.append('t');
|
||||
assert_eq!(next_tokens.len(), 1, "Should accept text");
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::TextContent(_) => (),
|
||||
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
|
||||
}
|
||||
|
||||
// Should accept closing tag start
|
||||
let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
|
||||
let next_tokens = token.append('<');
|
||||
assert_eq!(next_tokens.len(), 1, "Should accept '<'");
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::ElementCloseStart(_) => (),
|
||||
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_text_content_transition() {
|
||||
let element = create_test_element();
|
||||
let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
|
||||
|
||||
// Should accept more text
|
||||
let next_tokens = token.append('a');
|
||||
assert_eq!(next_tokens.len(), 1, "Should accept more text");
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::TextContent(_) => (),
|
||||
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
|
||||
}
|
||||
|
||||
// Should accept closing tag start
|
||||
let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
|
||||
let next_tokens = token.append('<');
|
||||
assert_eq!(next_tokens.len(), 1, "Should accept '<'");
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::ElementCloseStart(_) => (),
|
||||
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_element_close_start_transition() {
|
||||
let element = create_test_element();
|
||||
let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
|
||||
|
||||
// Should accept /
|
||||
let next_tokens = token.append('/');
|
||||
assert_eq!(next_tokens.len(), 1, "Should accept '/'");
|
||||
|
||||
let token = next_tokens[0].clone();
|
||||
match token {
|
||||
Token::ElementCloseStart(_) => (),
|
||||
_ => panic!("Expected ElementCloseStart with slash, got {:?}", token),
|
||||
}
|
||||
|
||||
// After /, should accept first char of element name
|
||||
let next_tokens = token.append('r');
|
||||
assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::ElementCloseName(_) => (),
|
||||
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_element_close_name_transition() {
|
||||
let element = create_test_element();
|
||||
let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
|
||||
|
||||
// Add / first
|
||||
let next_tokens = token.append('/');
|
||||
assert_eq!(next_tokens.len(), 1, "Should accept '/'");
|
||||
|
||||
// Start building name - clone the token to avoid moving out of the vector
|
||||
let next_tokens = next_tokens[0].clone().append('r');
|
||||
assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
|
||||
|
||||
// Continue with each character
|
||||
let mut current_tokens = next_tokens;
|
||||
for c in "easoning".chars() {
|
||||
let mut new_tokens = Vec::new();
|
||||
for token in current_tokens {
|
||||
new_tokens.append(&mut token.append(c));
|
||||
}
|
||||
current_tokens = new_tokens;
|
||||
|
||||
assert!(
|
||||
!current_tokens.is_empty(),
|
||||
"Should have valid transition after appending '{}'",
|
||||
c
|
||||
);
|
||||
}
|
||||
|
||||
// Final > to close - clone the token to avoid moving out of the vector
|
||||
let next_tokens = current_tokens[0].clone().append('>');
|
||||
assert_eq!(next_tokens.len(), 1, "Should accept '>'");
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::EndOfFile(_) => (),
|
||||
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_end_of_file_properties() {
|
||||
let token = Token::EndOfFile(EndOfFile::new());
|
||||
|
||||
// Should be identified as EOF
|
||||
assert!(token.is_eof(), "EndOfFile should be recognized as EOF");
|
||||
|
||||
// Should not accept any more input
|
||||
let next_tokens = token.append('a');
|
||||
assert_eq!(next_tokens.len(), 0, "Should not accept any more input");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,25 +28,64 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_accept_open_start_token() {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append("<").unwrap();
|
||||
assert!(!validator.current_tokens.is_empty());
|
||||
assert!(validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.filter(|token| matches!(token, Token::ElementOpenStart(_)))
|
||||
.next()
|
||||
.is_some());
|
||||
fn test_valid_start_of_file() {
|
||||
let values = [
|
||||
"<", // Just the opening angle bracket
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::ElementOpenStart(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_not_accept_other() {
|
||||
let values = ["a", " ", "\t", ">", "/"];
|
||||
fn test_valid_element_starts() {
|
||||
let values = [
|
||||
"<d", // delete
|
||||
"<s", // stop/single
|
||||
"<r", // reasoning/repeat/read_stdin
|
||||
"<w", // write_stdout
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect_err(value);
|
||||
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::ElementOpenName(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_start_of_file() {
|
||||
let values = [
|
||||
"a", // Text before tag
|
||||
" ", // Whitespace before tag
|
||||
"\t", // Tab before tag
|
||||
">", // Closing bracket before opening
|
||||
"/", // Slash before opening bracket
|
||||
"<!", // XML declaration not supported
|
||||
"<?", // XML processing instruction not supported
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
let result = validator.append(value);
|
||||
assert!(result.is_err(), "Should reject '{value}' at start of file");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,59 +26,104 @@ impl TextContent {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod text_content_tests {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::schema;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn create_test_element() -> Arc<schema::XsElement> {
|
||||
let element = schema::XsElement {
|
||||
name: "reasoning".to_string(),
|
||||
text: None,
|
||||
xs_complex_type: schema::XsComplexType {
|
||||
mixed: Some("true".to_string()),
|
||||
text: None,
|
||||
xs_attribute: None,
|
||||
xs_sequence: Some(schema::XsSequence {
|
||||
text: None,
|
||||
xs_any: schema::XsAny {
|
||||
min_occurs: "0".to_string(),
|
||||
max_occurs: "unbounded".to_string(),
|
||||
process_contents: "skip".to_string(),
|
||||
},
|
||||
}),
|
||||
},
|
||||
};
|
||||
Arc::new(element)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_text_content_normal_char() {
|
||||
let element = create_test_element();
|
||||
let token = TextContent::new(Arc::clone(&element));
|
||||
fn test_valid_text_content() {
|
||||
let values = [
|
||||
"<reasoning>text",
|
||||
"<reasoning>Hello world",
|
||||
"<reasoning>Special chars: !@#$%^&*()",
|
||||
"<reasoning>Numbers: 12345",
|
||||
"<reasoning>Mixed content with spaces",
|
||||
"<write_stdout>Output message",
|
||||
"<single>ls -la /tmp",
|
||||
];
|
||||
|
||||
// Test with normal character
|
||||
let next_tokens = token.append('a');
|
||||
assert!(!next_tokens.is_empty(), "Should accept normal character");
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::TextContent(_) => (),
|
||||
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::TextContent(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_text_content_less_than() {
|
||||
let element = create_test_element();
|
||||
let token = TextContent::new(Arc::clone(&element));
|
||||
fn test_whitespace_text_content() {
|
||||
let values = [
|
||||
"<reasoning> ",
|
||||
"<reasoning> ",
|
||||
"<reasoning>\t",
|
||||
"<reasoning>\n",
|
||||
"<reasoning> \t\n ",
|
||||
"<write_stdout> ",
|
||||
"<single> ",
|
||||
];
|
||||
|
||||
// Test with < (start of tag)
|
||||
let next_tokens = token.append('<');
|
||||
assert!(!next_tokens.is_empty(), "Should accept '<'");
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::TextContent(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
match &next_tokens[0] {
|
||||
Token::ElementCloseStart(_) => (),
|
||||
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
|
||||
#[test]
|
||||
fn test_valid_element_close_start_after_text() {
|
||||
let values = [
|
||||
"<reasoning>text<",
|
||||
"<reasoning>Hello world<",
|
||||
"<write_stdout>message<",
|
||||
"<single>command<",
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::ElementCloseStart(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_complete_element_with_text() {
|
||||
let values = [
|
||||
"<reasoning>text</reasoning>",
|
||||
"<reasoning>Hello world</reasoning>",
|
||||
"<write_stdout>message</write_stdout>",
|
||||
"<single>command</single>",
|
||||
];
|
||||
|
||||
for value in values {
|
||||
let mut validator = crate::tests::example_validator();
|
||||
validator.append(value).expect(value);
|
||||
assert!(!validator.current_tokens.is_empty(), "{value}");
|
||||
assert!(
|
||||
validator
|
||||
.current_tokens
|
||||
.iter()
|
||||
.any(|token| matches!(token, Token::EndOfFile(_))),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user