llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improved tests for tokens

Browse Source
This commit is contained in:
Geens
2025-04-13 19:05:10 +02:00
parent ac2c1ae08b
commit 773134e65b
17 changed files with 968 additions and 1402 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
lib/xml_schema_validator/src/char_trie.rs
View File

@@ -33,16 +33,19 @@ impl CharTrieNode {
py: Python<'_>,
xml_validator: &XmlSchemaValidator,
) -> Vec<Py<PyAny>> {
let res = self.children.iter().map(|(c, n)|{
let mut node_validator = xml_validator.clone();
if node_validator.append_char(*c).is_ok() {
// If the node is valid, recursively check its children
n.find_invalid_tokens(py, &node_validator)
} else {
// If the node is invalid, add it and its children to the list of invalid tokens
n.all_tokens(py)
}
})
let res = self
.children
.iter()
.map(|(c, n)| {
let mut node_validator = xml_validator.clone();
if node_validator.append_char(*c).is_ok() {
// If the node is valid, recursively check its children
n.find_invalid_tokens(py, &node_validator)
} else {
// If the node is invalid, add it and its children to the list of invalid tokens
n.all_tokens(py)
}
})
.flatten()
.collect();
res

714
lib/xml_schema_validator/src/lib.rs
View File

@@ -103,160 +103,9 @@ mod tests {
XmlSchemaValidator::new(&schema_text).unwrap()
}
#[test]
fn test_element_open() {
let mut validator = example_validator();
let input = "<reasoning>";
validator.append(input).unwrap();
}
#[test]
fn test_element_open_whitespace() {
let mut validator = example_validator();
let input = "<reasoning >";
validator.append(input).unwrap();
}
#[test]
fn test_element_open_invalid_name() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let input = "<rae";
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator.append(input).expect_err("Should fail");
}
#[test]
fn test_element_close() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let input = "<reasoning>test</reasoning>";
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator.append(input).unwrap();
}
#[test]
fn test_element_close_mismatched() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let input = "<reasoning>test</wrong>";
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator
.append(input)
.expect_err("Should fail with mismatched closing tag");
}
#[test]
fn test_delete_with_attribute() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let input = r#"<delete id="20250411_170104_361"/>"#;
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator.append(input).unwrap();
assert!(validator.eof());
}
#[test]
fn test_delete_missing_required_attribute() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
// Create validator
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
// First, validate up to the opening tag
assert!(
validator.append("<delete>").is_err(),
"Should fail with missing required attribute"
);
// Create a new validator and try closing tag approach
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
// Try to validate an incomplete delete tag that's missing the required id attribute
let input = "<delete/>";
let result = validator.append(input);
assert!(result.is_err(), "Should fail without required id attribute");
// Check the error message - note the test now checks for either "required" OR "No valid continuations"
if let Err(err) = result {
println!("Error message: {}", err);
let error_str = err.to_string().to_lowercase();
assert!(
error_str.contains("required") || error_str.contains("no valid continuations"),
"Error should mention missing required attribute or invalid continuation"
);
}
}
#[test]
fn test_invalid_nesting() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator
.clone()
.append("<reasoning><reasoning>")
.expect_err("Should fail because of nested tag");
validator
.clone()
.append("<reasoning> <reasoning>")
.expect_err("Should fail because of nested tag, even with whitespace");
validator
.clone()
.append("<reasoning>foo<reasoning>")
.expect_err("Should fail because of nested tag, even with text");
validator
.clone()
.append("<reasoning> foo <reasoning>")
.expect_err("Should fail because of nested tag, even with text and whitespace");
validator
.clone()
.append("<reasoning> foo <reasoning/>")
.expect_err("Should fail because of nested tag, even with self closing tag");
}
#[test]
fn test_stop_self_closing() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let input = "<stop/>";
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator.append(input).unwrap();
assert!(validator.eof());
}
#[test]
fn test_read_stdin_self_closing() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let input = "<read_stdin/>";
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator.append(input).unwrap();
assert!(validator.eof());
}
#[test]
fn test_single_with_attributes() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let input = r#"<single timeout="2.5" limit="2048">ls -la</single>"#;
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator.append(input).unwrap();
assert!(validator.eof());
}
#[test]
fn test_reasoning_with_content() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let input =
"<reasoning>I should explore the file system for interesting files.</reasoning>";
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator.append(input).unwrap();
assert!(validator.eof());
}
#[test]
fn test_write_stdout_with_content() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let input = "<write_stdout>Hello world!</write_stdout>";
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
validator.append(input).unwrap();
assert!(validator.eof());
}
#[test]
fn test_create_validator() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let schema_text = std::fs::read_to_string("example_schema.xsd").unwrap();
let validator = XmlSchemaValidator::new(&schema_text);
assert!(
validator.is_ok(),
@@ -267,73 +116,121 @@ mod tests {
#[test]
fn test_get_element_names() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let validator = XmlSchemaValidator::new(&schema_text).unwrap();
let validator = example_validator();
let names = validator.get_element_names();
assert!(
names.contains(&"reasoning".to_string()),
"Element names should contain 'reasoning', got: {:?}",
names
);
assert!(
names.contains(&"delete".to_string()),
"Element names should contain 'delete', got: {:?}",
names
);
}
// Test basic XML fragment validation - debugging the "<reasoning>" failure
#[test]
fn test_append_reasoning_tag() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let validator = XmlSchemaValidator::new(&schema_text).unwrap();
// Test character by character for precise failure point
assert!(validator.can_continue_with('<'), "Should accept '<'");
// Clone at each step to debug exact point of failure
let mut v1 = validator.clone();
assert!(v1.append_char('<').is_ok(), "Failed to append '<'");
let mut v2 = v1.clone();
assert!(v2.append_char('r').is_ok(), "Failed to append 'r'");
// Continue with each character in "reasoning>"
let mut v3 = v2.clone();
for c in "easoning>".chars() {
assert!(v3.append_char(c).is_ok(), "Failed to append '{}'", c);
fn test_valid_elements() {
// Test each element type
let valid_elements = [
"<reasoning>Test content</reasoning>",
"<delete id=\"12345\"/>",
"<stop/>",
"<single>ls -la</single>",
"<repeat>ps -ef</repeat>",
"<read_stdin/>",
"<write_stdout>Hello world!</write_stdout>",
];
for xml in valid_elements {
let mut validator = example_validator();
assert!(
validator.append(xml).is_ok(),
"Should accept valid XML: {}",
xml
);
assert!(
validator.eof(),
"Should be at EOF after complete element: {}",
xml
);
}
// Try the whole string at once
let mut v4 = validator.clone();
let result = v4.append("<reasoning>");
}
#[test]
fn test_invalid_elements() {
// Test various invalid XML constructs
let invalid_elements = [
"<invalid>", // Unknown element
"<reasoning></invalid>", // Mismatched tags
"<delete/>", // Missing required attribute
"<reasoning><reasoning></reasoning>", // Invalid nesting
"<stop id=\"123\"/>", // Attribute not in schema
];
for xml in invalid_elements {
let mut validator = example_validator();
assert!(
validator.append(xml).is_err(),
"Should reject invalid XML: {}",
xml
);
}
}
#[test]
fn test_attributes() {
// Test elements with attributes
let mut validator = example_validator();
assert!(
result.is_ok(),
"Failed to append '<reasoning>': {:?}",
result.err()
validator.append("<delete id=\"12345\"/>").is_ok(),
"Should accept element with required attribute"
);
let mut validator = example_validator();
assert!(
validator
.append("<single timeout=\"2.5\" limit=\"2048\">command</single>")
.is_ok(),
"Should accept element with multiple attributes"
);
// Test with invalid attribute values
let mut validator = example_validator();
assert!(
validator.append("<delete id=\"\"/>").is_ok(),
"Should accept empty attribute values"
);
}
// Test validation of multiple fragments
#[test]
fn test_multiple_append() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
// Test sequential appends
assert!(validator.append("<").is_ok(), "Failed to append '<'");
fn test_partial_append() {
let mut validator = example_validator();
// Test incremental appending
assert!(validator.append("<r").is_ok(), "Should accept partial tag");
assert!(
validator.append("reasoning").is_ok(),
"Failed to append 'reasoning'"
validator.append("easoning>").is_ok(),
"Should accept continuation"
);
assert!(validator.append(">").is_ok(), "Failed to append '>'");
assert!(validator.append("Hello ").is_ok(), "Should accept content");
assert!(
validator.append("test content").is_ok(),
"Failed to append content"
validator.append("world!").is_ok(),
"Should accept more content"
);
assert!(
validator.append("</reasoning>").is_ok(),
"Failed to append closing tag"
"Should accept closing tag"
);
// Should be at EOF now
assert!(validator.eof(), "Should be at EOF after complete XML");
assert!(validator.eof(), "Should be at EOF after complete element");
}
// Test token continuation - critical for the token masking issue
#[test]
fn test_can_continue_with() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
// Test at start
let validator = XmlSchemaValidator::new(&schema_text).unwrap();
let validator = example_validator();
// At start, only '<' is valid
assert!(
validator.can_continue_with('<'),
"Should accept '<' at start"
@@ -342,398 +239,57 @@ mod tests {
!validator.can_continue_with('a'),
"Should reject 'a' at start"
);
// Test after opening tag
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
// After opening a tag
let mut validator = example_validator();
validator.append("<reasoning>").unwrap();
// Any character should be valid inside a mixed content element
// In mixed content, many characters should be valid
assert!(
validator.can_continue_with('a'),
"Should accept 'a' inside element"
);
assert!(
validator.can_continue_with('1'),
"Should accept '1' inside element"
"Should accept 'a' in content"
);
assert!(
validator.can_continue_with('<'),
"Should accept '<' inside element"
"Should accept '<' for closing"
);
}
}
#[cfg(test)]
mod diagnostic_tests {
use super::*;
use std::sync::Arc;
// Simple schema with just the reasoning tag for cleaner debugging
fn get_simple_schema() -> String {
r#"<?xml version="1.0" encoding="UTF-8"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
<xs:element name="reasoning">
<xs:complexType mixed="true">
<xs:sequence>
<xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:schema>"#
.to_string()
}
// Test every character in an opening tag one by one
#[test]
fn test_char_by_char_open_tag() {
let schema_text = get_simple_schema();
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
// Test each character individually
assert!(validator.can_continue_with('<'), "Should accept '<'");
assert!(validator.append_char('<').is_ok(), "Failed to append '<'");
assert!(validator.can_continue_with('r'), "Should accept 'r'");
assert!(validator.append_char('r').is_ok(), "Failed to append 'r'");
for c in "easoning".chars() {
assert!(validator.can_continue_with(c), "Should accept '{}'", c);
assert!(validator.append_char(c).is_ok(), "Failed to append '{}'", c);
}
// This is where many tests are failing - let's check the state before and after
println!("State before '>': {:?}", validator.current_tokens);
let result = validator.append_char('>');
println!("Result of append '>': {:?}", result);
if result.is_ok() {
println!("State after '>': {:?}", validator.current_tokens);
}
assert!(result.is_ok(), "Failed to append '>'");
}
// Test the token transitions specifically
#[test]
fn test_token_transitions() {
// Create minimal elements for testing token transitions directly
let test_element = Arc::new(schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
});
// Test the transition from ElementOpenName to ElementOpenEnd
let element_name =
token::ElementOpenName::new(Arc::clone(&test_element), "reasoning".to_string());
let tokens = element_name.append('>');
assert!(!tokens.is_empty(), "Should produce valid tokens after '>'");
println!(
"Token after appending '>' to ElementOpenName: {:?}",
tokens[0]
);
// Check what type the token is
match &tokens[0] {
Token::ElementOpenEnd(element_open) => {
println!(
"Successfully transitioned to ElementOpen: {:?}",
element_open
);
}
other => {
panic!("Expected ElementOpen, got {:?}", other);
}
}
}
// Test element self-closing functionality
#[test]
fn test_element_self_close_transition() {
let test_element = Arc::new(schema::XsElement {
name: "stop".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None,
text: None,
xs_attribute: None,
xs_sequence: None,
},
});
// Test transition from ElementName to ElementSelfClose
let element_name =
token::ElementOpenName::new(Arc::clone(&test_element), "stop".to_string());
let tokens = element_name.append('/');
assert!(!tokens.is_empty(), "Should handle '/' after element name");
match &tokens[0] {
Token::ElementSelfClose(element_self_close) => {
println!("Successfully transitioned to ElementSelfClose");
// Now test transition to EndOfFile
let next_tokens = element_self_close.clone().append('>');
assert!(!next_tokens.is_empty(), "Should handle '>' after '/'");
match &next_tokens[0] {
Token::EndOfFile(_) => println!("Successfully transitioned to EndOfFile"),
other => panic!("Expected EndOfFile, got {:?}", other),
}
}
other => panic!("Expected ElementSelfClose, got {:?}", other),
}
}
// Test whitespace handling
#[test]
fn test_whitespace_handling() {
let schema_text = get_simple_schema();
let mut validator = XmlSchemaValidator::new(&schema_text).unwrap();
// Test opening tag with whitespace
for c in "<reasoning ".chars() {
println!("Appending character: '{}'", c);
assert!(validator.append_char(c).is_ok(), "Failed to append '{}'", c);
}
println!("State after '<reasoning ': {:?}", validator.current_tokens);
// Test completion with >
assert!(
validator.can_continue_with('>'),
"Should accept '>' after whitespace"
);
assert!(
validator.append_char('>').is_ok(),
"Failed to append '>' after whitespace"
);
}
// Test attribute handling
#[test]
fn test_attribute_handling() {
let schema_text = r#"<?xml version="1.0" encoding="UTF-8"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
<xs:element name="delete">
<xs:complexType>
<xs:attribute name="id" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>
</xs:schema>"#;
let mut validator = XmlSchemaValidator::new(schema_text).unwrap();
// Test opening tag with attribute
for c in "<delete ".chars() {
assert!(validator.append_char(c).is_ok(), "Failed to append '{}'", c);
}
// Test attribute name
for c in "id".chars() {
assert!(
validator.can_continue_with(c),
"Should accept '{}' in attribute name",
c
);
assert!(
validator.append_char(c).is_ok(),
"Failed to append '{}' in attribute name",
c
);
}
// Test attribute equals and quotes
for c in "=\"123\"".chars() {
println!("Testing attribute character: '{}'", c);
assert!(
validator.can_continue_with(c),
"Should accept '{}' in attribute value",
c
);
assert!(
validator.append_char(c).is_ok(),
"Failed to append '{}' in attribute value",
c
);
}
// Test closing the self-closing tag
println!(
"Current state before closing: {:?}",
validator.current_tokens
);
for c in "/>".chars() {
assert!(
validator.can_continue_with(c),
"Should accept '{}' to close tag",
c
);
assert!(
validator.append_char(c).is_ok(),
"Failed to append '{}' to close tag",
c
);
}
// Check if we've reached EOF
assert!(validator.eof(), "Should be at EOF after self-closing tag");
}
// Test for the specific error patterns seen in the test output
#[test]
fn test_specific_error_cases() {
let schema_text = std::fs::read_to_string("../../action_schema.xsd")
.unwrap_or_else(|_| get_simple_schema());
// Test case 1: Simple opening tag (noticed in `test_element_open`)
let input1 = "<reasoning>";
let mut validator1 = XmlSchemaValidator::new(&schema_text).unwrap();
let result1 = validator1.append(input1);
println!("Result appending '<reasoning>': {:?}", result1);
assert!(result1.is_ok(), "Failed on '<reasoning>'");
// Test case 2: Element with whitespace (noticed in `test_element_open_whitespace`)
let input2 = "<reasoning >";
let mut validator2 = XmlSchemaValidator::new(&schema_text).unwrap();
let result2 = validator2.append(input2);
println!("Result appending '<reasoning >': {:?}", result2);
assert!(result2.is_ok(), "Failed on '<reasoning >'");
// Test case 3: Self-closing tags (noticed in `test_read_stdin_self_closing`)
let input3 = "<read_stdin/>";
let mut validator3 = XmlSchemaValidator::new(&schema_text).unwrap();
let result3 = validator3.append(input3);
println!("Result appending '<read_stdin/>': {:?}", result3);
assert!(result3.is_ok(), "Failed on '<read_stdin/>'");
// Test case 4: Attributes (noticed in `test_single_with_attributes`)
let input4 = r#"<single timeout="2.5">"#;
let mut validator4 = XmlSchemaValidator::new(&schema_text).unwrap();
let result4 = validator4.append(input4);
println!("Result appending attribute: {:?}", result4);
assert!(result4.is_ok(), "Failed on attribute handling");
}
// Test the basic token state machine with a minimal element
#[test]
fn test_basic_token_operations() {
// Create a minimal element for testing
let element = Arc::new(schema::XsElement {
name: "test".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: None,
},
});
// Create and test each token type
let start_of_file = token::StartOfFile::new(Arc::clone(&element));
let tokens = start_of_file.append('<');
assert!(!tokens.is_empty(), "StartOfFile should accept '<'");
let element_start = match &tokens[0] {
Token::ElementOpenStart(es) => es.clone(),
_ => panic!("Expected ElementStart"),
};
let tokens = element_start.append('t');
assert!(!tokens.is_empty(), "ElementStart should accept 't'");
let element_name = match &tokens[0] {
Token::ElementOpenName(en) => en.clone(),
_ => panic!("Expected ElementName"),
};
// This is often where things break
println!("About to append 'e' to ElementName...");
let tokens = element_name.append('e');
assert!(!tokens.is_empty(), "ElementName should accept 'e'");
// Build the rest of the name
let mut current_token = tokens[0].clone();
for c in "st".chars() {
let next_tokens = current_token.append(c);
assert!(!next_tokens.is_empty(), "Token should accept '{}'", c);
current_token = next_tokens[0].clone();
}
// Now try to append '>'
println!("About to append '>' to token: {:?}", current_token);
let tokens = current_token.append('>');
println!("Result of appending '>': {:?}", tokens);
assert!(!tokens.is_empty(), "Token should accept '>'");
// Verify transition to ElementOpen
match &tokens[0] {
Token::ElementOpenEnd(_) => println!("Successfully transitioned to ElementOpen"),
other => panic!("Expected ElementOpen, got {:?}", other),
}
}
// This test will help identify issues with token creation for specific elements
#[test]
fn test_element_specific_issues() {
if let Ok(schema_text) = std::fs::read_to_string("../../action_schema.xsd") {
let validator = XmlSchemaValidator::new(&schema_text).unwrap();
// Get all element names in the schema
let element_names = validator.get_element_names();
println!("Elements in schema: {:?}", element_names);
// Test each element separately
for name in &element_names {
println!("\nTesting element: {}", name);
let mut element_validator = XmlSchemaValidator::new(&schema_text).unwrap();
let open_tag = format!("<{}>", name);
// Test character by character
println!("Testing character by character for <{}>", name);
let mut progress = String::new();
for c in open_tag.chars() {
progress.push(c);
let result = element_validator.can_continue_with(c);
println!(
" Can continue with '{}' after '{}': {}",
c,
&progress[..progress.len() - 1],
result
);
if !result {
// This is helpful to diagnose where the process fails
println!(" ❌ Failed on character '{}' for element '{}'", c, name);
break;
}
if let Err(e) = element_validator.append_char(c) {
println!(
" ❌ Failed to append '{}' for element '{}': {:?}",
c, name, e
);
break;
}
}
// Test complete tag at once
let mut complete_validator = XmlSchemaValidator::new(&schema_text).unwrap();
match complete_validator.append(&open_tag) {
Ok(_) => println!(" ✅ Successfully appended <{}>", name),
Err(e) => println!(" ❌ Failed to append <{}>: {:?}", name, e),
}
}
} else {
println!("Schema file not found, skipping element-specific tests");
}
// Test for characters that form closing tag
let mut validator = example_validator();
validator.append("<reasoning>content<").unwrap();
assert!(
validator.can_continue_with('/'),
"Should accept '/' for closing"
);
// Invalid continuation after EOF
let mut validator = example_validator();
validator.append("<stop/>").unwrap();
assert!(
!validator.can_continue_with('a'),
"Should reject chars after EOF"
);
}
#[test]
fn test_cloning() {
let mut validator = example_validator();
validator.append("<reasoning>").unwrap();
// Clone the validator and make sure they behave independently
let mut clone = validator.clone();
// Modify the clone
clone.append("test").unwrap();
clone.append("</reasoning>").unwrap();
assert!(clone.eof(), "Clone should be at EOF");
// Original should be unchanged
assert!(!validator.eof(), "Original should not be at EOF");
// Original should still accept more content
assert!(validator.append("different").is_ok());
assert!(validator.append("</reasoning>").is_ok());
assert!(validator.eof(), "Original should now be at EOF");
}
}

12
lib/xml_schema_validator/src/py_xml_logits_processor_core.rs
View File

@@ -112,7 +112,9 @@ mod tests {
#[test]
fn test_delete_quote() {
Python::with_gil(|py| {
py.run(&std::ffi::CString::new(r#"
py.run(
&std::ffi::CString::new(
r#"
from transformers import AutoTokenizer
from xml_schema_validator import XmlLogitsProcessor
import os
@@ -128,7 +130,13 @@ processed_scores = processor(input_ids, scores)
input_ids = torch.tensor([tokenizer.encode("<delete id=\"")])
scores = torch.ones((1, len(tokenizer))) # All tokens have equal probability
processed_scores = processor(input_ids, scores)
"#).unwrap(), None, None).unwrap();
"#,
)
.unwrap(),
None,
None,
)
.unwrap();
})
}

60
lib/xml_schema_validator/src/schema.rs
View File

@@ -1,14 +1,6 @@
use serde::{Deserialize, Serialize};
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
pub struct XsSchema {
#[serde(rename = "@xmlns:xs")]
pub xmlns_xs: String,
@@ -20,15 +12,7 @@ pub struct XsSchema {
pub xs_element: Vec<XsElement>,
}
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
pub struct XsElement {
#[serde(rename = "@name")]
pub name: String,
@@ -38,15 +22,7 @@ pub struct XsElement {
pub xs_complex_type: XsComplexType,
}
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
pub struct XsComplexType {
#[serde(rename = "@mixed")]
pub mixed: Option<String>,
@@ -58,15 +34,7 @@ pub struct XsComplexType {
pub xs_sequence: Option<XsSequence>,
}
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
pub struct XsAttribute {
#[serde(rename = "@name")]
pub name: String,
@@ -76,15 +44,7 @@ pub struct XsAttribute {
pub xs_attribute_use: String,
}
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
pub struct XsSequence {
#[serde(rename = "$text")]
pub text: Option<String>,
@@ -92,15 +52,7 @@ pub struct XsSequence {
pub xs_any: XsAny,
}
#[derive(
Clone,
Debug,
Deserialize,
Eq,
Hash,
PartialEq,
Serialize,
)]
#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
pub struct XsAny {
#[serde(rename = "@minOccurs")]
pub min_occurs: String,

49
lib/xml_schema_validator/src/token/attribute_equals.rs
View File

@@ -9,10 +9,7 @@ pub struct AttributeEquals {
}
impl AttributeEquals {
pub fn new(
element: Arc<schema::XsElement>,
used_attributes: Vec<schema::XsAttribute>,
) -> Self {
pub fn new(element: Arc<schema::XsElement>, used_attributes: Vec<schema::XsAttribute>) -> Self {
Self {
element,
used_attributes,
@@ -41,12 +38,16 @@ mod tests {
use super::*;
#[test]
fn test_attribute_equals_quote() {
fn test_valid_attribute_equals() {
let values = [
"<delete id=\"",
"<single timeout=\"",
"<single limit=\"",
"<delete id= \"", // with space
"<single timeout = \"", // with spaces around equals
"<single limit \t = \n \"", // with lots of whitespace
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
@@ -55,48 +56,24 @@ mod tests {
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeValue(_)))
.next()
.is_some(),
.any(|token| matches!(token, Token::AttributeValue(_))),
"{value}"
);
}
}
#[test]
fn test_attribute_equals_with_whitespace() {
fn test_invalid_attribute_equals() {
let values = [
"<delete id= \"",
"<single timeout = \"",
"<single limit \t = \n \"",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeValue(_)))
.next()
.is_some(),
"{value}"
);
}
}
#[test]
fn test_invalid_after_equals() {
let values = [
"<delete id=a", // Only quote should be accepted after equals
"<delete id=a", // Only quote should be accepted after equals
"<single timeout=<", // Invalid character after equals
"<single timeout=1", // Unquoted numbers not supported yet
"<single timeout=1", // Unquoted numbers not supported
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
let result = validator.append(value);
assert!(result.is_err(), "Should reject '{value}'");
}
}
}

123
lib/xml_schema_validator/src/token/attribute_name.rs
View File

@@ -18,7 +18,11 @@ impl AttributeName {
}
}
pub fn new_with_used(first_char: char, element: Arc<schema::XsElement>, used_attributes: Vec<schema::XsAttribute>) -> Self {
pub fn new_with_used(
first_char: char,
element: Arc<schema::XsElement>,
used_attributes: Vec<schema::XsAttribute>,
) -> Self {
Self {
buffer: first_char.to_string(),
element,
@@ -26,7 +30,33 @@ impl AttributeName {
}
}
// Helper method to check if the current buffer matches any attribute that's already used
fn is_duplicate_attribute(&self) -> bool {
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
// Find any attribute that starts with our current buffer
for attr in attributes
.iter()
.filter(|a| a.name.starts_with(&self.buffer))
{
// Check if this attribute or any prefix of it has already been used
if self
.used_attributes
.iter()
.any(|used| used.name == attr.name)
{
return true;
}
}
}
false
}
pub fn append(mut self, c: char) -> Vec<Token> {
// First check if we're already working with a duplicate attribute
if self.is_duplicate_attribute() {
return vec![]; // Return empty vector to indicate invalid state
}
if c == '=' {
// Check if we've matched a valid attribute name
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
@@ -34,11 +64,6 @@ impl AttributeName {
.iter()
.find(|attribute| attribute.name == self.buffer)
{
// Check if this attribute has already been used
if self.used_attributes.contains(&attribute) {
return vec![];
}
// Add this attribute to the used set
self.used_attributes.push(attribute.clone());
@@ -54,13 +79,9 @@ impl AttributeName {
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
if attributes
.iter()
.any(|attribute| attribute.name == self.buffer)
.find(|attribute| attribute.name == self.buffer)
.is_some()
{
// Check if this attribute has already been used
if self.used_attributes.iter().filter(|attribute| attribute.name == self.buffer).next().is_some() {
return vec![];
}
return vec![Token::AttributeName(self)];
}
}
@@ -75,11 +96,19 @@ impl AttributeName {
.iter()
.any(|attribute| attribute.name.starts_with(&new_buffer))
{
return vec![Token::AttributeName(AttributeName {
// Create a new token with the updated buffer
let new_token = AttributeName {
buffer: new_buffer,
element: Arc::clone(&self.element),
used_attributes: self.used_attributes,
})];
used_attributes: self.used_attributes.clone(),
};
// Check if this would be a duplicate
if new_token.is_duplicate_attribute() {
return vec![];
}
return vec![Token::AttributeName(new_token)];
}
}
@@ -96,33 +125,41 @@ mod tests {
#[test]
fn test_valid_attribute_name() {
let values = [
"<delete id=",
"<single timeout=",
"<single limit=",
"<delete id",
"<single timeout",
"<single limit",
"<delete id ", // with trailing space
"<single timeout ", // with trailing spaces
"<single limit\t", // with tab
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
// Either still building the attribute name or reached equals sign
assert!(
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeEquals(_)))
.next()
.is_some(),
validator.current_tokens.iter().any(|token| matches!(
token,
Token::AttributeName(_) | Token::AttributeEquals(_)
)),
"{value}"
);
}
}
#[test]
fn test_attribute_name_with_whitespace() {
fn test_valid_attribute_name_with_equals() {
let values = [
"<delete id =",
"<single timeout =",
"<single limit\t=",
"<delete id=",
"<single timeout=",
"<single limit=",
"<delete id =", // with space before equals
"<single timeout =", // with spaces before equals
"<single limit\t=", // with tab before equals
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
@@ -131,9 +168,7 @@ mod tests {
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeEquals(_)))
.next()
.is_some(),
.any(|token| matches!(token, Token::AttributeEquals(_))),
"{value}"
);
}
@@ -144,22 +179,32 @@ mod tests {
let values = [
"<delete invalid=",
"<single wrongattr=",
"<delete xyz",
"<single badname",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
let result = validator.append(value);
assert!(result.is_err(), "Should reject '{value}'");
}
}
#[test]
fn test_duplicate_attribute() {
let mut validator = crate::tests::example_validator();
// First attribute is valid
validator.append("<delete id=\"123\" ").expect("First attribute should be accepted");
let values = [
"<delete id=\"123\" id",
"<single timeout=\"2.5\" timeout",
"<single limit=\"100\" limit",
];
// Try to add the same attribute again
let result = validator.append("id=");
assert!(result.is_err(), "Duplicate attribute should be rejected");
for value in values {
let mut validator = crate::tests::example_validator();
let result = validator.append(value);
assert!(
result.is_err(),
"Should reject duplicate attribute in '{value}'"
);
}
}
}

57
lib/xml_schema_validator/src/token/attribute_value.rs
View File

@@ -13,10 +13,7 @@ pub struct AttributeValue {
impl AttributeValue {
/// Create a new AttributeValue with a list of attributes already set
pub fn new(
element: Arc<schema::XsElement>,
attributes_set: Vec<schema::XsAttribute>,
) -> Self {
pub fn new(element: Arc<schema::XsElement>, attributes_set: Vec<schema::XsAttribute>) -> Self {
Self {
element,
buffer: "\"".to_string(),
@@ -102,7 +99,9 @@ impl AttributeValue {
if '/' == c {
tokens.push(Token::ElementSelfClose(ElementSelfClose::new()));
} else if '>' == c {
tokens.push(Token::ElementOpenEnd(ElementOpenEnd::new(self.element.clone())));
tokens.push(Token::ElementOpenEnd(ElementOpenEnd::new(
self.element.clone(),
)));
}
}
if self.buffer.ends_with(char::is_whitespace) && !all_attributes_set {
@@ -121,6 +120,36 @@ impl AttributeValue {
mod tests {
use super::*;
#[test]
fn test_valid_attribute_value() {
let values = [
"<delete id=\"123\"",
"<delete id=\"abc_123\"",
"<single timeout=\"2.5\"",
"<single limit=\"100\"",
"<delete id=\" \"", // space as value
"<delete id=\" \"", // multiple spaces as value
"<delete id=\"\"", // empty value
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
// Should be in AttributeValue state or ready for next attribute/closing
assert!(
validator.current_tokens.iter().any(|token| matches!(
token,
Token::AttributeValue(_)
| Token::ElementSelfClose(_)
| Token::ElementOpenEnd(_)
)),
"{value}"
);
}
}
#[test]
fn test_unfinished_attribute_value() {
let values = [
@@ -147,21 +176,6 @@ mod tests {
}
}
#[test]
fn test_type_validation() {
// Test integer validation
let mut validator = crate::tests::example_validator();
validator.append("<single limit=\"123\"").expect("Valid integer");
// Test float validation
validator = crate::tests::example_validator();
validator.append("<single timeout=\"1.5\"").expect("Valid float");
// Test string validation
validator = crate::tests::example_validator();
validator.append("<delete id=\"abc123\"").expect("Valid string");
}
#[test]
fn test_multiple_attributes() {
let values = [
@@ -177,8 +191,7 @@ mod tests {
}
#[test]
fn test_attribute_value_escape_chars() {
// Test handling of escaped characters in attribute values
fn test_attribute_value_with_escape_chars() {
let values = [
"<delete id=\"escaped&quot;quote\"",
"<delete id=\"escaped&amp;amp\"",

153
lib/xml_schema_validator/src/token/element_close_name.rs
View File

@@ -39,111 +39,88 @@ impl ElementCloseName {
#[cfg(test)]
mod tests {
use super::*;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_close_name_partial() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
fn test_valid_element_close_name() {
let values = [
"<reasoning>content</r",
"<reasoning>content</re",
"<reasoning>content</rea",
"<reasoning>content</reasoning",
"<single>script</single",
"<write_stdout>text</write_stdout",
];
// Test with next character in name
let next_tokens = token.append('e');
assert!(
!next_tokens.is_empty(),
"Should accept next character in name"
);
match &next_tokens[0] {
Token::ElementCloseName(name) => {
assert_eq!(name.buffer, "re", "Buffer should contain 're'");
}
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::ElementCloseName(_))),
"{value}"
);
}
}
#[test]
fn test_close_name_complete() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "reasonin".to_string());
fn test_element_close_name_with_whitespace() {
let values = [
"<reasoning>content</reasoning ",
"<single>script</single\t",
"<write_stdout>text</write_stdout ",
];
// Test with last character of name
let next_tokens = token.append('g');
assert!(
!next_tokens.is_empty(),
"Should accept last character of name"
);
match &next_tokens[0] {
Token::ElementCloseName(name) => {
assert_eq!(
name.buffer, "reasoning",
"Buffer should contain 'reasoning'"
);
}
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
// Test with > after complete name
let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
let next_tokens = token.append('>');
assert!(
!next_tokens.is_empty(),
"Should accept '>' after complete name"
);
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::ElementCloseName(_))),
"{value}"
);
}
}
#[test]
fn test_close_name_whitespace() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string());
fn test_complete_element_close() {
let values = [
"<reasoning>content</reasoning>",
"<single>script</single>",
"<write_stdout>text</write_stdout>",
];
// Test with whitespace after complete name
let next_tokens = token.append(' ');
assert!(
!next_tokens.is_empty(),
"Should accept whitespace after complete name"
);
match &next_tokens[0] {
Token::ElementCloseName(_) => (),
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::EndOfFile(_))),
"{value}"
);
}
}
#[test]
fn test_close_name_invalid() {
let element = create_test_element();
let token = ElementCloseName::new(Arc::clone(&element), "r".to_string());
fn test_invalid_element_close_name() {
let values = [
"<reasoning>content</invalid",
"<single>script</wrong",
"<reasoning>text</reasoning_wrong",
];
// Test with invalid next character
let next_tokens = token.append('x');
assert!(next_tokens.is_empty(), "Should reject invalid character");
for value in values {
let mut validator = crate::tests::example_validator();
let result = validator.append(value);
assert!(result.is_err(), "Should reject '{value}'");
}
}
}

123
lib/xml_schema_validator/src/token/element_close_start.rs
View File

@@ -37,74 +37,85 @@ impl ElementCloseStart {
#[cfg(test)]
mod tests {
use super::*;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_close_start_slash() {
let element = create_test_element();
let token = ElementCloseStart::new(Arc::clone(&element));
fn test_valid_element_close_start() {
let values = [
"<reasoning>content<",
"<single>script<",
"<write_stdout>text<",
];
// Test with slash
let next_tokens = token.append('/');
assert!(!next_tokens.is_empty(), "Should accept '/'");
match &next_tokens[0] {
Token::ElementCloseStart(start) => {
assert!(start.has_slash, "Should mark as having slash");
}
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::ElementCloseStart(_))),
"{value}"
);
}
}
#[test]
fn test_close_start_element_name() {
let element = create_test_element();
let mut token = ElementCloseStart::new(Arc::clone(&element));
token.has_slash = true;
fn test_valid_element_close_start_with_slash() {
let values = [
"<reasoning>content</",
"<single>script</",
"<write_stdout>text</",
];
// Test with first character of element name
let next_tokens = token.append('r');
assert!(
!next_tokens.is_empty(),
"Should accept first letter of element name"
);
match &next_tokens[0] {
Token::ElementCloseName(_) => (),
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::ElementCloseStart(_))),
"{value}"
);
}
}
#[test]
fn test_close_start_invalid_name() {
let element = create_test_element();
let mut token = ElementCloseStart::new(Arc::clone(&element));
token.has_slash = true;
fn test_valid_element_name_start_after_slash() {
let values = [
"<reasoning>content</r",
"<single>script</s",
"<write_stdout>text</w",
];
// Test with invalid first character of element name
let next_tokens = token.append('x');
assert!(next_tokens.is_empty(), "Should reject invalid first letter");
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::ElementCloseName(_))),
"{value}"
);
}
}
#[test]
fn test_invalid_element_close_start() {
let values = [
"<reasoning>content<x", // Invalid character after
"<single>script<>", // Missing slash
"<write_stdout>text</x", // Invalid character after </
];
for value in values {
let mut validator = crate::tests::example_validator();
let result = validator.append(value);
assert!(result.is_err(), "Should reject '{value}'");
}
}
}

180
lib/xml_schema_validator/src/token/element_open_end.rs
View File

@@ -9,17 +9,20 @@ pub struct ElementOpenEnd {
impl ElementOpenEnd {
pub fn new(element: Arc<schema::XsElement>) -> Self {
Self {
element,
}
Self { element }
}
pub fn append(self, c: char) -> Vec<Token> {
if '<' == c {
vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))]
vec![Token::ElementCloseStart(ElementCloseStart::new(
Arc::clone(&self.element),
))]
} else {
// Check if this element allows mixed content
let is_mixed = self.element.xs_complex_type.mixed
let is_mixed = self
.element
.xs_complex_type
.mixed
.as_ref()
.map(|val| val == "true")
.unwrap_or(false);
@@ -38,126 +41,87 @@ impl ElementOpenEnd {
#[cfg(test)]
mod tests {
use super::*;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
fn create_element_non_mixed() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "element".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: None, // Not mixed content
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_append_text_in_mixed_content() {
let element = create_test_element();
let token = ElementOpenEnd::new(Arc::clone(&element));
fn test_valid_element_open_end() {
let values = [
"<reasoning>",
"<write_stdout>",
"<single>",
"<reasoning >", // with space
"<single >", // with multiple spaces
];
// Test appending text in mixed content element
let next_tokens = token.append('a');
assert!(
!next_tokens.is_empty(),
"Should allow text content in mixed element"
);
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::ElementOpenEnd(_) | Token::TextContent(_))),
"{value}"
);
}
}
#[test]
fn test_append_closing_tag_start() {
let element = create_test_element();
let token = ElementOpenEnd::new(Arc::clone(&element));
fn test_valid_text_content_after_open() {
let values = [
"<reasoning>text",
"<write_stdout>Hello world",
"<single>ls -la",
"<reasoning> text with spaces ",
];
// Test starting a closing tag
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should allow closing tag start");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::TextContent(_))),
"{value}"
);
}
}
#[test]
fn test_append_whitespace_in_mixed() {
let element = create_test_element();
let token = ElementOpenEnd::new(Arc::clone(&element));
fn test_valid_close_start_after_open() {
let values = ["<reasoning><", "<write_stdout><", "<single><"];
// Test whitespace in mixed content
let next_tokens = token.append(' ');
assert!(
!next_tokens.is_empty(),
"Should allow whitespace in mixed content"
);
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::ElementCloseStart(_))),
"{value}"
);
}
}
#[test]
fn test_append_in_non_mixed() {
let element = create_element_non_mixed();
let token = ElementOpenEnd::new(Arc::clone(&element));
fn test_invalid_nested_elements() {
let values = [
"<reasoning><reasoning>",
"<write_stdout><single>",
"<single><reasoning>",
];
// Test appending text in non-mixed content (should reject)
let next_tokens = token.clone().append('a');
assert!(
next_tokens.is_empty(),
"Should reject text in non-mixed element"
);
// Test whitespace (should accept leading to closing tag)
let next_tokens = token.clone().append(' ');
assert!(
!next_tokens.is_empty(),
"Should allow whitespace in non-mixed"
);
// Test closing tag start
let next_tokens = token.append('<');
assert!(
!next_tokens.is_empty(),
"Should allow closing tag start in non-mixed"
);
for value in values {
let mut validator = crate::tests::example_validator();
let result = validator.append(value);
assert!(
result.is_err(),
"Should reject nested elements in '{value}'"
);
}
}
}

137
lib/xml_schema_validator/src/token/element_open_name.rs
View File

@@ -44,22 +44,31 @@ impl ElementOpenName {
if !self.buffer.contains(&self.element.name) {
return vec![];
}
// Check if the element has attributes
// Check if this element type allows self-closing
// Elements with mixed content or those that expect content should not be self-closing
let is_mixed = self
.element
.xs_complex_type
.mixed
.as_ref()
.map(|val| val == "true")
.unwrap_or(false);
// Elements with mixed content should not be self-closing
if is_mixed {
return vec![];
}
// Check if the element has required attributes
if let Some(attributes) = &self.element.xs_complex_type.xs_attribute {
// Check if they are required
// Check if any required attributes are missing
if attributes
.iter()
.filter(|attribute| attribute.xs_attribute_use == "required")
.next()
.is_some()
.any(|attribute| attribute.xs_attribute_use == "required")
{
return vec![];
} else {
return vec![Token::ElementSelfClose(ElementSelfClose::new())];
}
} else {
return vec![Token::ElementSelfClose(ElementSelfClose::new())];
}
// If we get here, self-closing is allowed
return vec![Token::ElementSelfClose(ElementSelfClose::new())];
} else if c.is_whitespace() {
// Check if we've matched the full element name
if self.buffer.contains(&self.element.name) {
@@ -114,12 +123,20 @@ mod tests {
use super::*;
#[test]
fn test_attribute_name() {
fn test_valid_element_open_name() {
let values = [
"<delete i", // id
"<single t", // timeout
"<single l", // limit
"<r", // partial reasoning
"<re", // partial reasoning
"<reasoning", // complete name
"<s", // partial stop/single
"<si", // partial single
"<sin", // partial single
"<single", // complete name
"<d", // partial delete
"<de", // partial delete
"<delete", // complete name
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
@@ -128,17 +145,23 @@ mod tests {
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::AttributeName(_)))
.next()
.is_some(),
.any(|token| matches!(token, Token::ElementOpenName(_))),
"{value}"
);
}
}
#[test]
fn test_valid_self_closing() {
let values = ["<stop/", "<stop /"];
fn test_valid_element_name_with_closing() {
let values = [
"<reasoning>",
"<stop>",
"<single>",
"<delete id=\"123\">",
"<reasoning >", // with space
"<stop >", // with spaces
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
@@ -147,17 +170,21 @@ mod tests {
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementSelfClose(_)))
.next()
.is_some(),
.any(|token| matches!(token, Token::ElementOpenEnd(_) | Token::TextContent(_))),
"{value}"
);
}
}
#[test]
fn test_valid_closing() {
let values = ["<stop>", "<stop >", "<reasoning>", "<reasoning >"];
fn test_valid_element_name_with_self_closing() {
let values = [
"<stop/>",
"<read_stdin/>",
"<stop />", // with space
"<read_stdin />", // with spaces
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
@@ -166,21 +193,71 @@ mod tests {
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementOpenEnd(_)))
.next()
.is_some(),
.any(|token| matches!(token, Token::EndOfFile(_))),
"{value}"
);
}
}
#[test]
fn test_invalid_closing() {
let values = ["<delete/", "<delete /", "<delete>", "<delete >"];
fn test_valid_element_name_with_attribute() {
let values = [
"<delete id",
"<single timeout",
"<single limit",
"<delete id=",
"<delete id=\"123\"",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
// Should be in AttributeName, AttributeEquals, or AttributeValue state
assert!(
validator.current_tokens.iter().any(|token| matches!(
token,
Token::AttributeName(_) | Token::AttributeEquals(_) | Token::AttributeValue(_)
)),
"{value}"
);
}
}
#[test]
fn test_invalid_element_name() {
let values = [
"<invalid", // Non-existent element
"<wrongelement", // Non-existent element
"<Reasoning", // Wrong case
"<DELETE", // Wrong case
];
for value in values {
let mut validator = crate::tests::example_validator();
let result = validator.append(value);
assert!(
result.is_err(),
"Should reject invalid element name in '{value}'"
);
}
}
#[test]
fn test_invalid_element_self_closing() {
let values = [
"<reasoning/>", // Mixed element can't self close
"<delete/>", // Missing required attribute
];
for value in values {
let mut validator = crate::tests::example_validator();
let result = validator.append(value);
assert!(
result.is_err(),
"Should reject invalid self-closing in '{value}'"
);
}
}
}

60
lib/xml_schema_validator/src/token/element_open_start.rs
View File

@@ -29,13 +29,14 @@ mod tests {
use super::*;
#[test]
fn test_accept_valid_element_names() {
fn test_valid_element_open_start() {
let values = [
"<d", // delete
"<s", // stop / single
"<r", // repeat / reasoning / read_stdin
"<w", // write_stdout
"<d", // start of delete
"<s", // start of stop/single
"<r", // start of reasoning/repeat/read_stdin
"<w", // start of write_stdout
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
@@ -44,21 +45,56 @@ mod tests {
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementOpenName(_)))
.next()
.is_some(),
.any(|token| matches!(token, Token::ElementOpenName(_))),
"{value}"
);
}
}
#[test]
fn test_not_accept_other() {
let values = ["< ", "<a", "</"];
fn test_valid_full_element_names() {
let values = [
"<delete",
"<stop",
"<single",
"<repeat",
"<reasoning",
"<read_stdin",
"<write_stdout",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::ElementOpenName(_))),
"{value}"
);
}
}
#[test]
fn test_invalid_element_open_start() {
let values = [
"< ", // Space after
"<a", // Invalid element name start
"<1", // Number after
"<_", // Underscore after
"</", // Closing tag not allowed at start
"<.", // Invalid character after
];
for value in values {
let mut validator = crate::tests::example_validator();
let result = validator.append(value);
assert!(
result.is_err(),
"Should reject invalid element start in '{value}'"
);
}
}
}

55
lib/xml_schema_validator/src/token/element_self_close.rs
View File

@@ -24,8 +24,14 @@ mod tests {
use super::*;
#[test]
fn test_valid() {
let values = ["<stop/>"];
fn test_valid_element_self_close() {
let values = [
"<stop/>",
"<read_stdin/>",
"<stop />", // with space before /
"<read_stdin />", // with space before /
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
@@ -34,21 +40,52 @@ mod tests {
validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::EndOfFile(_)))
.next()
.is_some(),
.any(|token| matches!(token, Token::EndOfFile(_))),
"{value}"
);
}
}
#[test]
fn test_invalid() {
let values = ["<stop/?", "<stop/a", "<stop/ "];
fn test_valid_element_with_attribute_self_close() {
let values = [
"<delete id=\"123\"/>",
"<single timeout=\"1.5\"/>",
"<single limit=\"100\"/>",
"<delete id=\"abc\" />", // with space before /
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::EndOfFile(_))),
"{value}"
);
}
}
#[test]
fn test_invalid_element_self_close() {
let values = [
"<stop/a", // Invalid character after /
"<stop/?", // Invalid character after /
"<stop/ >", // Space between / and >
"<delete/>", // Missing required attribute
"<read_stdin>/>", // > in wrong position
];
for value in values {
let mut validator = crate::tests::example_validator();
let result = validator.append(value);
assert!(
result.is_err(),
"Should reject invalid self-closing in '{value}'"
);
}
}
}

53
lib/xml_schema_validator/src/token/end_of_file.rs
View File

@@ -19,11 +19,54 @@ mod tests {
use super::*;
#[test]
fn test_end_of_file_any_char() {
let token = EndOfFile::new();
fn test_complete_valid_elements_reach_eof() {
let values = [
"<stop/>",
"<read_stdin/>",
"<reasoning>test</reasoning>",
"<single>command</single>",
"<delete id=\"123\"/>",
"<write_stdout>Hello world</write_stdout>",
];
// Test with any character (should reject)
let next_tokens = token.append('a');
assert!(next_tokens.is_empty(), "Should reject any character at EOF");
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::EndOfFile(_))),
"{value}"
);
// Also verify that eof() method returns true
assert!(validator.eof(), "eof() should return true for '{value}'");
}
}
#[test]
fn test_no_continuation_after_eof() {
let valid_values = [
"<stop/>",
"<reasoning>test</reasoning>",
"<delete id=\"123\"/>",
];
for value in valid_values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
// Try to append more characters after EOF
let continuations = ["a", " ", "<", ">"];
for cont in continuations {
let result = validator.append(cont);
assert!(
result.is_err(),
"Should reject '{cont}' after EOF in '{value}'"
);
}
}
}
}

217
lib/xml_schema_validator/src/token/mod.rs
View File

@@ -66,220 +66,3 @@ impl Token {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::schema;
use std::sync::Arc;
pub fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_element_start_transition() {
let element = create_test_element();
let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
// Should accept first char of element name
let next_tokens = token.append('r');
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementOpenName(_) => (),
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
}
// Should reject invalid char
let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
let next_tokens = token.append('x');
assert_eq!(next_tokens.len(), 0, "Should reject 'x'");
}
#[test]
fn test_element_name_transition() {
let element = create_test_element();
let token =
Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()));
// Continue building name
let next_tokens = token.append('e');
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementOpenName(_) => (),
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
}
// Build complete name and end with >
// Remove the unused buffer
let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(
&element,
)))];
for c in "reasoning>".chars() {
let mut new_tokens = Vec::new();
for token in current_tokens {
new_tokens.append(&mut token.append(c));
}
current_tokens = new_tokens;
assert!(
!current_tokens.is_empty(),
"Should have valid transition after appending '{}'",
c
);
}
// Check final state
assert_eq!(current_tokens.len(), 1, "Should have one final transition");
match &current_tokens[0] {
Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", current_tokens[0]),
}
}
#[test]
fn test_element_open_end_transition() {
let element = create_test_element();
let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
// Should accept text content in mixed element
let next_tokens = token.append('t');
assert_eq!(next_tokens.len(), 1, "Should accept text");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
// Should accept closing tag start
let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
let next_tokens = token.append('<');
assert_eq!(next_tokens.len(), 1, "Should accept '<'");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_text_content_transition() {
let element = create_test_element();
let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
// Should accept more text
let next_tokens = token.append('a');
assert_eq!(next_tokens.len(), 1, "Should accept more text");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
}
// Should accept closing tag start
let token = Token::TextContent(TextContent::new(Arc::clone(&element)));
let next_tokens = token.append('<');
assert_eq!(next_tokens.len(), 1, "Should accept '<'");
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_element_close_start_transition() {
let element = create_test_element();
let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
// Should accept /
let next_tokens = token.append('/');
assert_eq!(next_tokens.len(), 1, "Should accept '/'");
let token = next_tokens[0].clone();
match token {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart with slash, got {:?}", token),
}
// After /, should accept first char of element name
let next_tokens = token.append('r');
assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
match &next_tokens[0] {
Token::ElementCloseName(_) => (),
_ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_element_close_name_transition() {
let element = create_test_element();
let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element)));
// Add / first
let next_tokens = token.append('/');
assert_eq!(next_tokens.len(), 1, "Should accept '/'");
// Start building name - clone the token to avoid moving out of the vector
let next_tokens = next_tokens[0].clone().append('r');
assert_eq!(next_tokens.len(), 1, "Should accept 'r'");
// Continue with each character
let mut current_tokens = next_tokens;
for c in "easoning".chars() {
let mut new_tokens = Vec::new();
for token in current_tokens {
new_tokens.append(&mut token.append(c));
}
current_tokens = new_tokens;
assert!(
!current_tokens.is_empty(),
"Should have valid transition after appending '{}'",
c
);
}
// Final > to close - clone the token to avoid moving out of the vector
let next_tokens = current_tokens[0].clone().append('>');
assert_eq!(next_tokens.len(), 1, "Should accept '>'");
match &next_tokens[0] {
Token::EndOfFile(_) => (),
_ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]),
}
}
#[test]
fn test_end_of_file_properties() {
let token = Token::EndOfFile(EndOfFile::new());
// Should be identified as EOF
assert!(token.is_eof(), "EndOfFile should be recognized as EOF");
// Should not accept any more input
let next_tokens = token.append('a');
assert_eq!(next_tokens.len(), 0, "Should not accept any more input");
}
}

67
lib/xml_schema_validator/src/token/start_of_file.rs
View File

@@ -28,25 +28,64 @@ mod tests {
use super::*;
#[test]
fn test_accept_open_start_token() {
let mut validator = crate::tests::example_validator();
validator.append("<").unwrap();
assert!(!validator.current_tokens.is_empty());
assert!(validator
.current_tokens
.iter()
.filter(|token| matches!(token, Token::ElementOpenStart(_)))
.next()
.is_some());
fn test_valid_start_of_file() {
let values = [
"<", // Just the opening angle bracket
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::ElementOpenStart(_))),
"{value}"
);
}
}
#[test]
fn test_not_accept_other() {
let values = ["a", " ", "\t", ">", "/"];
fn test_valid_element_starts() {
let values = [
"<d", // delete
"<s", // stop/single
"<r", // reasoning/repeat/read_stdin
"<w", // write_stdout
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect_err(value);
assert_eq!(validator.current_tokens.len(), 0, "Should reject '{value}'");
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::ElementOpenName(_))),
"{value}"
);
}
}
#[test]
fn test_invalid_start_of_file() {
let values = [
"a", // Text before tag
" ", // Whitespace before tag
"\t", // Tab before tag
">", // Closing bracket before opening
"/", // Slash before opening bracket
"<!", // XML declaration not supported
"<?", // XML processing instruction not supported
];
for value in values {
let mut validator = crate::tests::example_validator();
let result = validator.append(value);
assert!(result.is_err(), "Should reject '{value}' at start of file");
}
}
}

131
lib/xml_schema_validator/src/token/text_content.rs
View File

@@ -26,59 +26,104 @@ impl TextContent {
}
#[cfg(test)]
mod text_content_tests {
mod tests {
use super::*;
use crate::schema;
use std::sync::Arc;
fn create_test_element() -> Arc<schema::XsElement> {
let element = schema::XsElement {
name: "reasoning".to_string(),
text: None,
xs_complex_type: schema::XsComplexType {
mixed: Some("true".to_string()),
text: None,
xs_attribute: None,
xs_sequence: Some(schema::XsSequence {
text: None,
xs_any: schema::XsAny {
min_occurs: "0".to_string(),
max_occurs: "unbounded".to_string(),
process_contents: "skip".to_string(),
},
}),
},
};
Arc::new(element)
}
#[test]
fn test_text_content_normal_char() {
let element = create_test_element();
let token = TextContent::new(Arc::clone(&element));
fn test_valid_text_content() {
let values = [
"<reasoning>text",
"<reasoning>Hello world",
"<reasoning>Special chars: !@#$%^&*()",
"<reasoning>Numbers: 12345",
"<reasoning>Mixed content with spaces",
"<write_stdout>Output message",
"<single>ls -la /tmp",
];
// Test with normal character
let next_tokens = token.append('a');
assert!(!next_tokens.is_empty(), "Should accept normal character");
match &next_tokens[0] {
Token::TextContent(_) => (),
_ => panic!("Expected TextContent, got {:?}", next_tokens[0]),
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::TextContent(_))),
"{value}"
);
}
}
#[test]
fn test_text_content_less_than() {
let element = create_test_element();
let token = TextContent::new(Arc::clone(&element));
fn test_whitespace_text_content() {
let values = [
"<reasoning> ",
"<reasoning> ",
"<reasoning>\t",
"<reasoning>\n",
"<reasoning> \t\n ",
"<write_stdout> ",
"<single> ",
];
// Test with < (start of tag)
let next_tokens = token.append('<');
assert!(!next_tokens.is_empty(), "Should accept '<'");
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::TextContent(_))),
"{value}"
);
}
}
match &next_tokens[0] {
Token::ElementCloseStart(_) => (),
_ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]),
#[test]
fn test_valid_element_close_start_after_text() {
let values = [
"<reasoning>text<",
"<reasoning>Hello world<",
"<write_stdout>message<",
"<single>command<",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::ElementCloseStart(_))),
"{value}"
);
}
}
#[test]
fn test_complete_element_with_text() {
let values = [
"<reasoning>text</reasoning>",
"<reasoning>Hello world</reasoning>",
"<write_stdout>message</write_stdout>",
"<single>command</single>",
];
for value in values {
let mut validator = crate::tests::example_validator();
validator.append(value).expect(value);
assert!(!validator.current_tokens.is_empty(), "{value}");
assert!(
validator
.current_tokens
.iter()
.any(|token| matches!(token, Token::EndOfFile(_))),
"{value}"
);
}
}
}