WIP cdata and comment tokens

lib/xml_schema_validator/src/token/cdata.rs

@@ -0,0 +1,241 @@
+use crate::*;
+use std::sync::Arc;
+
+/// Represents a CDATA section that is in the process of being parsed
+#[derive(Clone, Debug)]
+pub struct CData {
+    /// The element that contains this CDATA section
+    element: Arc<schema::XsElement>,
+    /// Current parsing state
+    state: CDataState,
+}
+
+/// The different states of CDATA parsing
+#[derive(Clone, Debug, PartialEq)]
+enum CDataState {
+    /// Parsing "<", initial state
+    OpeningBracket1,
+    /// Parsing "!" after "<" in opening sequence
+    OpeningExclamationPoint,
+    /// Parsing "[" after "<!" in opening sequence
+    OpeningBracket2,
+    /// Parsing "C" after "<![" in opening sequence
+    OpeningC,
+    /// Parsing "D" after "<![C" in opening sequence
+    OpeningD,
+    /// Parsing "A" after "<![CD" in opening sequence
+    OpeningA1,
+    /// Parsing "T" after "<![CDA" in opening sequence
+    OpeningT,
+    /// Parsing "A" after "<![CDAT" in opening sequence
+    OpeningA2,
+    /// Parsing "[" after "<![CDATA" in opening sequence
+    OpeningBracket3,
+    /// Parsing the content of the CDATA section
+    Content,
+    /// Encountered first closing bracket "]"
+    ClosingBracket1,
+    /// Encountered second closing bracket "]]"
+    ClosingBracket2,
+    /// Encountered third closing bracket "]]>"
+    ClosingBracket3,
+}
+
+impl CData {
+    /// Create a new CDATA parsing state after seeing "<!"
+    pub fn new(element: Arc<schema::XsElement>) -> Self {
+        Self {
+            element,
+            state: CDataState::OpeningBracket1,
+        }
+    }
+
+    pub fn append(mut self, c: char) -> Vec<Token> {
+        match self.state {
+            CDataState::OpeningBracket1 if c == '!' => vec![Token::CData(Self{state: CDataState::OpeningExclamationPoint, ..self})],
+            CDataState::OpeningExclamationPoint if c == '[' => vec![Token::CData(Self{state: CDataState::OpeningBracket2, ..self})],
+            CDataState::OpeningBracket2 if c == 'C' => vec![Token::CData(Self{state: CDataState::OpeningC, ..self})],
+            CDataState::OpeningC if c == 'D' => vec![Token::CData(Self{state: CDataState::OpeningD, ..self})],
+            CDataState::OpeningD if c == 'A' => vec![Token::CData(Self{state: CDataState::OpeningA1, ..self})],
+            CDataState::OpeningA1 if c == 'T' => vec![Token::CData(Self{state: CDataState::OpeningT, ..self})],
+            CDataState::OpeningT if c == 'A' => vec![Token::CData(Self{state: CDataState::OpeningA2, ..self})],
+            CDataState::OpeningA2 if c == '[' => vec![Token::CData(Self{state: CDataState::OpeningBracket3, ..self})],
+            CDataState::OpeningBracket3 if c == ']' => vec![Token::CData(Self{state: CDataState::ClosingBracket1, ..self})],
+            CDataState::OpeningBracket3 if c != ']' => vec![Token::CData(Self{state: CDataState::Content, ..self})],
+            CDataState::ClosingBracket1 if c == ']' => vec![Token::CData(Self{state: CDataState::ClosingBracket2, ..self})],
+            CDataState::ClosingBracket2 if c == '>' => vec![Token::CData(Self{state: CDataState::ClosingBracket3, ..self})],
+            CDataState::ClosingBracket3 => self.handle_closing_bracket3(c),
+            CDataState::Content => self.handle_content(c),
+            _ => vec![],
+        }
+    }
+
+    fn handle_closing_bracket3(self, c: char) -> Vec<Token> {
+        if '<' == c {
+            let is_mixed = self
+                .element
+                .xs_complex_type
+                .mixed
+                .as_ref()
+                .map(|val| val == "true")
+                .unwrap_or(false);
+                
+            let mut tokens = vec![Token::ElementCloseStart(ElementCloseStart::new(
+                Arc::clone(&self.element),
+            ))];
+            
+            // If mixed content is allowed, also add Comment and CData as possible transitions
+            if is_mixed {
+                tokens.push(Token::Comment(Comment::new(Arc::clone(&self.element))));
+                tokens.push(Token::CData(CData::new(Arc::clone(&self.element))));
+            }
+            
+            tokens
+        } else {
+            // Check if this element allows mixed content
+            let is_mixed = self
+                .element
+                .xs_complex_type
+                .mixed
+                .as_ref()
+                .map(|val| val == "true")
+                .unwrap_or(false);
+
+            if c.is_whitespace() {
+                vec![Token::CData(self)]
+            } else if is_mixed {
+                vec![Token::TextContent(TextContent::new(self.element))]
+            } else {
+                vec![]
+            }
+        }
+    }
+
+    fn handle_content(self, c: char) -> Vec<Token> {
+        todo!()
+    }
+    
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_valid_cdata_sections() {
+        let values = [
+            "<reasoning><![CDATA[This is CDATA content]]></reasoning>",
+            "<reasoning>Text <![CDATA[CDATA section]]> more text</reasoning>",
+            "<reasoning><![CDATA[CDATA with XML-like content: <tag>text</tag>]]></reasoning>",
+            "<reasoning><![CDATA[\nMultiline\nCDATA\n]]></reasoning>",
+            "<reasoning><![CDATA[CDATA with special chars: <>&'\"]]></reasoning>",
+            "<reasoning><![CDATA[]]> Empty CDATA</reasoning>", // Empty CDATA is valid
+            "<reasoning><![CDATA[]]></reasoning>", // Empty CDATA
+        ];
+
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            assert!(
+                validator
+                    .current_tokens
+                    .iter()
+                    .any(|token| matches!(token, Token::EndOfFile(_))),
+                "{value}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_partial_cdata_sections() {
+        let values = [
+            "<reasoning><!",
+            "<reasoning><![",
+            "<reasoning><![C",
+            "<reasoning><![CD",
+            "<reasoning><![CDA",
+            "<reasoning><![CDAT",
+            "<reasoning><![CDATA",
+            "<reasoning><![CDATA[",
+            "<reasoning><![CDATA[Test",
+            "<reasoning><![CDATA[Test]",
+            "<reasoning><![CDATA[Test]]",
+        ];
+
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            
+            let is_valid = validator
+                .current_tokens
+                .iter()
+                .any(|token| matches!(token, Token::CData(_)));
+            
+            assert!(is_valid, "{value}");
+        }
+    }
+
+    #[test]
+    fn test_invalid_cdata_sections() {
+        let values = [
+            "<reasoning><![CDATA[Unclosed CDATA section</reasoning>",
+            "<reasoning><![CDATA Malformed CDATA start]]></reasoning>",
+            "<reasoning><![CDATA[Nested <![CDATA[section]]>]]></reasoning>", // Nesting not allowed
+            "<reasoning><![CDTA[Invalid tag name]]></reasoning>",
+            "<reasoning><![CDATA[section]> Missing closing bracket</reasoning>",
+        ];
+
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            let result = validator.append(value);
+            assert!(
+                result.is_err(),
+                "Should reject invalid CDATA syntax in '{value}'"
+            );
+        }
+    }
+
+    #[test]
+    fn test_cdata_in_mixed_content_only() {
+        // CDATA should only be allowed in mixed content elements
+        let non_mixed_elements = [
+            "<delete id=\"123\" <![C",
+            "<stop <![CDATA[data]]> />",
+            "<delete id=\"123\"><![C",
+        ];
+
+        for value in non_mixed_elements {
+            let mut validator = crate::tests::example_validator();
+            let result = validator.append(value);
+            assert!(
+                result.is_err(),
+                "CDATA should only be allowed in mixed content: '{value}'"
+            );
+        }
+    }
+
+    #[test]
+    fn test_cdata_with_special_characters() {
+        let values = [
+            "<reasoning><![CDATA[<tag>content</tag>]]></reasoning>", // XML tags
+            "<reasoning><![CDATA[& < > \" ']]></reasoning>", // Special characters
+            "<reasoning><![CDATA[Multiple\nLines\nOf\nContent]]></reasoning>", // Newlines
+            "<reasoning><![CDATA[]]></reasoning>", // Empty
+        ];
+
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            assert!(
+                validator
+                    .current_tokens
+                    .iter()
+                    .any(|token| matches!(token, Token::EndOfFile(_))),
+                "{value}"
+            );
+        }
+    }
+}

lib/xml_schema_validator/src/token/comment.rs

@@ -0,0 +1,196 @@
+use crate::*;
+use std::sync::Arc;
+
+/// Represents an XML comment that is in the process of being parsed
+#[derive(Clone, Debug)]
+pub struct Comment {
+    /// The element that contains this comment
+    element: Arc<schema::XsElement>,
+    /// Buffer containing the comment characters processed so far
+    buffer: String,
+    /// Tracks if we're at the end of the comment (checking for -->)
+    end_dashes: u8,
+}
+
+impl Comment {
+    /// Create a new Comment parsing state after seeing "<!"
+    pub fn new(element: Arc<schema::XsElement>) -> Self {
+        Self {
+            element,
+            buffer: "<!".to_string(),
+            end_dashes: 0,
+        }
+    }
+
+    pub fn append(mut self, c: char) -> Vec<Token> {
+        // Add character to buffer
+        self.buffer.push(c);
+        
+        let buffer_len = self.buffer.len();
+        
+        // Check for comment opening sequence "<!--"
+        if buffer_len <= 4 {
+            // We're still parsing the opening sequence
+            let expected_prefix = "<!--".get(0..buffer_len).unwrap_or("");
+            if self.buffer != expected_prefix {
+                // Invalid comment start
+                return vec![];
+            }
+            if buffer_len < 4 {
+                // Still building opening sequence
+                return vec![Token::Comment(self)];
+            }
+            // Complete opening sequence, continue to comment content
+            return vec![Token::Comment(self)];
+        }
+        
+        // Check for comment closing sequence "-->"
+        if self.end_dashes == 0 && c == '-' {
+            self.end_dashes = 1;
+            return vec![Token::Comment(self)];
+        } else if self.end_dashes == 1 {
+            if c == '-' {
+                self.end_dashes = 2;
+                return vec![Token::Comment(self)];
+            } else {
+                // Reset dash count if we didn't see a second dash
+                self.end_dashes = 0;
+                return vec![Token::Comment(self)];
+            }
+        } else if self.end_dashes == 2 {
+            if c == '>' {
+                // Comment is complete, transition back to text content
+                return vec![Token::TextContent(TextContent::new(self.element))];
+            } else if c == '-' {
+                // -- followed by another - is not allowed in XML comments
+                // but we'll be lenient and just keep the dash count at 2
+                return vec![Token::Comment(self)];
+            } else {
+                // Reset dash count if we didn't see a closing bracket
+                self.end_dashes = 0;
+                return vec![Token::Comment(self)];
+            }
+        }
+        
+        // Regular comment content
+        vec![Token::Comment(self)]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_valid_comments() {
+        let values = [
+            "<reasoning><!-- This is a comment --></reasoning>",
+            "<reasoning>Text <!-- Comment --> more text</reasoning>",
+            "<reasoning><!-- Multiple comments --> text <!-- another comment --></reasoning>",
+            "<reasoning><!--\nMultiline\ncomment\n--></reasoning>",
+            "<reasoning><!-- Comment with special chars: <>&'\" --></reasoning>",
+            "<reasoning><!-- -- Comment with double dash --></reasoning>",
+            "<reasoning><!--></reasoning>", // Empty comment
+        ];
+
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            assert!(
+                validator
+                    .current_tokens
+                    .iter()
+                    .any(|token| matches!(token, Token::EndOfFile(_))),
+                "{value}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_partial_comments() {
+        let values = [
+            "<reasoning><!",
+            "<reasoning><!-",
+            "<reasoning><!--",
+            "<reasoning><!-- Comment",
+            "<reasoning><!-- Comment -",
+            "<reasoning><!-- Comment --",
+        ];
+
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            
+            // Should be in Comment state or TextContent state for early prefixes
+            let is_valid = validator
+                .current_tokens
+                .iter()
+                .any(|token| matches!(token, Token::Comment(_) | Token::TextContent(_)));
+            
+            assert!(is_valid, "{value}");
+        }
+    }
+
+    #[test]
+    fn test_invalid_comments() {
+        let values = [
+            "<reasoning><!-- Unclosed comment</reasoning>",
+            "<reasoning><!-- Nested <!-- comment --> --></reasoning>", // Nesting not allowed
+            "<reasoning><!- Invalid comment syntax -></reasoning>",
+            "<reasoning><!--> Malformed comment</reasoning>",
+            "<reasoning><!----> Suspicious triple dash</reasoning>",
+        ];
+
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            let result = validator.append(value);
+            assert!(
+                result.is_err(),
+                "Should reject invalid comment syntax in '{value}'"
+            );
+        }
+    }
+
+    #[test]
+    fn test_comments_in_mixed_content_only() {
+        // Comments should only be allowed in mixed content elements
+        let non_mixed_elements = [
+            "<delete id=\"123\" <!-- comment --> />",
+            "<stop <!-- comment --> />",
+            "<delete id=\"123\"><!-- test --></delete>",
+        ];
+
+        for value in non_mixed_elements {
+            let mut validator = crate::tests::example_validator();
+            let result = validator.append(value);
+            assert!(
+                result.is_err(),
+                "Comments should only be allowed in mixed content: '{value}'"
+            );
+        }
+    }
+
+    #[test]
+    fn test_comments_with_special_characters() {
+        let values = [
+            "<reasoning><!-- <tag>not real xml</tag> --></reasoning>", // XML tags in comment
+            "<reasoning><!-- & < > \" ' --></reasoning>", // Special characters
+            "<reasoning><!-- Multiple\nLines\nOf\nComment --></reasoning>", // Newlines
+        ];
+
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            assert!(
+                validator
+                    .current_tokens
+                    .iter()
+                    .any(|token| matches!(token, Token::EndOfFile(_))),
+                "{value}"
+            );
+        }
+    }
+}

lib/xml_schema_validator/src/token/element_open_end.rs

@@ -14,9 +14,25 @@ impl ElementOpenEnd {
 
     pub fn append(self, c: char) -> Vec<Token> {
         if '<' == c {
-            vec![Token::ElementCloseStart(ElementCloseStart::new(
+            let is_mixed = self
+                .element
+                .xs_complex_type
+                .mixed
+                .as_ref()
+                .map(|val| val == "true")
+                .unwrap_or(false);
+                
+            let mut tokens = vec![Token::ElementCloseStart(ElementCloseStart::new(
                 Arc::clone(&self.element),
-            ))]
+            ))];
+            
+            // If mixed content is allowed, also add Comment and CData as possible transitions
+            if is_mixed {
+                tokens.push(Token::Comment(Comment::new(Arc::clone(&self.element))));
+                tokens.push(Token::CData(CData::new(Arc::clone(&self.element))));
+            }
+            
+            tokens
         } else {
             // Check if this element allows mixed content
             let is_mixed = self
@@ -124,4 +140,44 @@ mod tests {
             );
         }
     }
+    
+    #[test]
+    fn test_mixed_comments_cdata_after_open() {
+        let values = [
+            "<reasoning><!-- comment --><![CDATA[content]]></reasoning>",
+            "<reasoning><![CDATA[content]]><!-- comment --></reasoning>",
+        ];
+
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            assert!(
+                validator
+                    .current_tokens
+                    .iter()
+                    .any(|token| matches!(token, Token::EndOfFile(_))),
+                "{value}"
+            );
+        }
+    }
+    
+    #[test]
+    fn test_invalid_comment_cdata_in_non_mixed_elements() {
+        let values = [
+            "<stop><!-- comment --></stop>", 
+            "<delete id=\"123\"><!-- comment --></delete>",
+            "<stop><![CDATA[content]]></stop>",
+            "<delete id=\"123\"><![CDATA[content]]></delete>",
+        ];
+
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            let result = validator.append(value);
+            assert!(
+                result.is_err(),
+                "Comments and CDATA should not be allowed in non-mixed content elements: '{value}'"
+            );
+        }
+    }
 }

lib/xml_schema_validator/src/token/mod.rs

@@ -1,6 +1,8 @@
 mod attribute_equals;
 mod attribute_name;
 mod attribute_value;
+mod cdata;
+mod comment;
 mod element_close_name;
 mod element_close_start;
 mod element_open_end;
@@ -14,6 +16,8 @@ mod text_content;
 pub use attribute_equals::AttributeEquals;
 pub use attribute_name::AttributeName;
 pub use attribute_value::AttributeValue;
+pub use cdata::CData;
+pub use comment::Comment;
 pub use element_close_name::ElementCloseName;
 pub use element_close_start::ElementCloseStart;
 pub use element_open_end::ElementOpenEnd;
@@ -30,6 +34,8 @@ pub enum Token {
     AttributeEquals(AttributeEquals),
     AttributeName(AttributeName),
     AttributeValue(AttributeValue),
+    CData(CData),
+    Comment(Comment),
     ElementCloseName(ElementCloseName),
     ElementCloseStart(ElementCloseStart),
     ElementOpenEnd(ElementOpenEnd),
@@ -45,17 +51,19 @@ impl Token {
     pub fn append(self, c: char) -> Vec<Token> {
         match self {
             Token::AttributeEquals(attribute_equals) => attribute_equals.append(c),
+            Token::AttributeName(attribute_name) => attribute_name.append(c),
+            Token::AttributeValue(attribute_value) => attribute_value.append(c),
+            Token::CData(cdata) => cdata.append(c), // Handle CDATA token
+            Token::Comment(comment) => comment.append(c), // Handle Comment token
             Token::ElementCloseName(element_close_name) => element_close_name.append(c),
             Token::ElementCloseStart(element_close_start) => element_close_start.append(c),
             Token::ElementOpenName(element_name) => element_name.append(c),
             Token::ElementOpenEnd(element_open) => element_open.append(c),
             Token::ElementOpenStart(element_start) => element_start.append(c),
+            Token::ElementSelfClose(element_self_close) => element_self_close.append(c),
             Token::EndOfFile(end_of_file) => end_of_file.append(c),
             Token::StartOfFile(start_of_file) => start_of_file.append(c),
             Token::TextContent(text_content) => text_content.append(c),
-            Token::AttributeName(attribute_name) => attribute_name.append(c),
-            Token::AttributeValue(attribute_value) => attribute_value.append(c),
-            Token::ElementSelfClose(element_self_close) => element_self_close.append(c),
         }
     }
 

lib/xml_schema_validator/src/token/text_content.rs

@@ -14,9 +14,25 @@ impl TextContent {
 
     pub fn append(self, c: char) -> Vec<Token> {
         if c == '<' {
-            vec![Token::ElementCloseStart(ElementCloseStart::new(
+            let is_mixed = self
+                .element
+                .xs_complex_type
+                .mixed
+                .as_ref()
+                .map(|val| val == "true")
+                .unwrap_or(false);
+                
+            let mut tokens = vec![Token::ElementCloseStart(ElementCloseStart::new(
                 Arc::clone(&self.element),
-            ))]
+            ))];
+            
+            // If mixed content is allowed, also add Comment and CData as possible transitions
+            if is_mixed {
+                tokens.push(Token::Comment(Comment::new(Arc::clone(&self.element))));
+                tokens.push(Token::CData(CData::new(Arc::clone(&self.element))));
+            }
+            
+            tokens
         } else {
             vec![Token::TextContent(TextContent::new(Arc::clone(
                 &self.element,
@@ -126,4 +142,81 @@ mod tests {
             );
         }
     }
+    
+    // New tests for transitions to comment and CDATA
+    
+    #[test]
+    fn test_text_to_comment_transition() {
+        let values = [
+            "<reasoning>text<",
+            "<reasoning>text<!",
+            "<reasoning>text<!--",
+            "<reasoning>text<!-- comment",
+        ];
+
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            
+            let is_valid = validator
+                .current_tokens
+                .iter()
+                .any(|token| matches!(token, Token::Comment(_)));
+            
+            assert!(is_valid, "{value}");
+        }
+    }
+    
+    #[test]
+    fn test_text_to_cdata_transition() {
+        let values = [
+            "<reasoning>text<",
+            "<reasoning>text<!",
+            "<reasoning>text<![",
+            "<reasoning>text<![C",
+            "<reasoning>text<![CD",
+            "<reasoning>text<![CDA",
+            "<reasoning>text<![CDAT",
+            "<reasoning>text<![CDATA",
+            "<reasoning>text<![CDATA[",
+        ];
+
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            
+            // Should transition to CDATA state or remain in TextContent for partial prefix
+            let is_valid = validator
+                .current_tokens
+                .iter()
+                .any(|token| matches!(token, Token::CData(_)));
+            
+            assert!(is_valid, "{value}");
+        }
+    }
+    
+    #[test]
+    fn test_text_with_mixed_content() {
+        let values = [
+            "<reasoning>text<!-- comment -->more text</reasoning>",
+            "<reasoning>text<![CDATA[cdata content]]>more text</reasoning>",
+            "<reasoning>text<!-- comment --><![CDATA[cdata]]>more</reasoning>",
+            "<reasoning><!-- c1 -->text<![CDATA[cd]]><!-- c2 --></reasoning>",
+        ];
+
+        for value in values {
+            let mut validator = crate::tests::example_validator();
+            validator.append(value).expect(value);
+            assert!(!validator.current_tokens.is_empty(), "{value}");
+            assert!(
+                validator
+                    .current_tokens
+                    .iter()
+                    .any(|token| matches!(token, Token::EndOfFile(_))),
+                "{value}"
+            );
+        }
+    }
 }