Begin XML Schema Validator

lib/xml_schema_validator/.gitignore

@@ -0,0 +1,2 @@
+target/
+Cargo.lock

lib/xml_schema_validator/build.rs

@@ -0,0 +1,15 @@
+pub fn main() {
+    println!("cargo:rerun-if-changed=../../action_schema.xsd");
+    let output = std::process::Command::new("xml_schema_generator.exe")
+            .arg("--derive")
+            .arg("Clone, Debug, Deserialize, Serialize")
+            .arg("../../action_schema.xsd")
+            .arg("src/schema.rs")
+            .output()
+            .expect("Failed to execute xml_schema_generator");
+    
+        if !output.status.success() {
+            panic!("xml_schema_generator failed: {}", 
+                String::from_utf8_lossy(&output.stderr));
+        }
+}

lib/xml_schema_validator/cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "xml_schema_validator"
+version = "0.1.0"
+edition = "2021"
+authors = ["SIA Team"]
+description = "XML Schema validation library optimized for llm token validation"
+
+[lib]
+name = "xml_schema_validator"
+# Required for PyO3
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+# XML parsing utilities
+quick-xml = { version = "0.28.1", features = ["serde", "serialize"] }
+serde = { version = "1", features = ["derive"] }
+thiserror = "1.0.38"    # Better error handling
+
+# Python bindings
+pyo3 = { version = "0.18.0", features = ["extension-module"] }
+
+[build-dependencies]
+xml_schema_generator = "0.6.18"

lib/xml_schema_validator/pyproject.toml

@@ -0,0 +1,25 @@
+[build-system]
+requires = ["maturin>=1.0,<2.0"]
+build-backend = "maturin"
+
+[project]
+name = "xml_schema_validator"
+version = "0.1.0"
+description = "XML Schema validation library for SIA"
+readme = "README.md"
+requires-python = ">=3.10"
+classifiers = [
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Text Processing :: Markup :: XML",
+]
+dependencies = []
+
+[project.urls]
+repository = "https://github.com/sia/xml_schema_validator"
+
+[tool.maturin]
+python-source = "python"
+features = ["pyo3/extension-module"]

lib/xml_schema_validator/src/error.rs

@@ -0,0 +1,14 @@
+/// Error types that can occur during XML validation
+
+#[derive(Clone, Debug, thiserror::Error)]
+pub enum Error {
+    /// Indicates that the provided XSD schema is invalid
+    #[error("Invalid schema: {0}")]
+    InvalidSchema(#[from] quick_xml::DeError),
+    /// Indicates that the provided XML fragment is invalid
+    #[error("Invalid XML: {0}")]
+    InvalidXml(String),
+    /// Not implemented yet
+    #[error("Not implemented yet")]
+    NotImplemented,
+}

lib/xml_schema_validator/src/lib.rs

@@ -0,0 +1,104 @@
+#![warn(missing_docs)]
+#![warn(rust_2018_idioms)]
+
+//! A streaming XML validator that checks XML fragments against an XSD schema
+
+mod error;
+mod schema;
+mod token;
+
+use error::Error;
+use token::*;
+
+/// An XML validator that checks XML fragments against an XSD schema
+#[derive(Clone, Debug)]
+pub struct XmlValidator {
+    current_tokens: Vec<std::rc::Rc<Token>>,
+}
+
+/// The Result type used throughout this crate
+///
+/// This is a type alias for `std::result::Result` with our custom `Error` type
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl XmlValidator {
+    /// Create a new validator with the given schema text
+    pub fn new(schema_text: &str) -> Result<Self> {
+        let schema: schema::XsSchema = quick_xml::de::from_str(schema_text)?;
+        let current_tokens = schema.xs_element.iter().map(|element| {
+            std::rc::Rc::new(Token::StartOfFile(StartOfFile::new(std::rc::Rc::new(element.clone()))))
+        }).collect();
+        
+        //let current_tokens = vec![std::rc::Rc::new(Token::StartOfFile(StartOfFile))];
+        Ok(Self {
+            current_tokens,
+        })
+    }
+    
+    /// Append a fragment of XML to the validator
+    pub fn append(&mut self, fragment: &str) -> Result<()> {
+        for c in fragment.chars() {
+            self.append_char(c)?;
+        }
+        Ok(())
+    }
+
+    /// Append a single character to the validator
+    fn append_char(&mut self, c: char) -> Result<()> {
+        println!("Appending char: {}", c);
+        let mut new_tokens = vec![];
+        for token in self.current_tokens.clone() {
+            new_tokens.append(&mut token.append(c));
+        }
+        if new_tokens.is_empty() {
+            return Err(Error::InvalidXml("No continuations".to_string()));
+        }
+        self.current_tokens = new_tokens;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_element_open() {
+        let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
+        let input = "<reasoning>";
+        let mut validator = XmlValidator::new(&schema_text).unwrap();
+        validator.append(input).unwrap();
+    }
+
+    #[test]
+    fn test_element_open_whitespace() {
+        let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
+        let input = "<reasoning >";
+        let mut validator = XmlValidator::new(&schema_text).unwrap();
+        validator.append(input).unwrap();
+    }
+
+    #[test]
+    fn test_element_open_invalid_name() {
+        let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
+        let input = "<rae";
+        let mut validator = XmlValidator::new(&schema_text).unwrap();
+        validator.append(input).expect_err("Should fail");
+    }
+    
+    #[test]
+    fn test_element_close() {
+        let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
+        let input = "<reasoning>test</reasoning>";
+        let mut validator = XmlValidator::new(&schema_text).unwrap();
+        validator.append(input).unwrap();
+    }
+    
+    #[test]
+    fn test_element_close_mismatched() {
+        let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
+        let input = "<reasoning>test</wrong>";
+        let mut validator = XmlValidator::new(&schema_text).unwrap();
+        validator.append(input).expect_err("Should fail with mismatched closing tag");
+    }
+}

lib/xml_schema_validator/src/schema.rs

@@ -0,0 +1,64 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct XsSchema {
+    #[serde(rename = "@xmlns:xs")]
+    pub xmlns_xs: String,
+    #[serde(rename = "@elementFormDefault")]
+    pub element_form_default: String,
+    #[serde(rename = "$text")]
+    pub text: Option<String>,
+    #[serde(rename = "element")]
+    pub xs_element: Vec<XsElement>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct XsElement {
+    #[serde(rename = "@name")]
+    pub name: String,
+    #[serde(rename = "$text")]
+    pub text: Option<String>,
+    #[serde(rename = "complexType")]
+    pub xs_complex_type: XsComplexType,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct XsComplexType {
+    #[serde(rename = "@mixed")]
+    pub mixed: Option<String>,
+    #[serde(rename = "$text")]
+    pub text: Option<String>,
+    #[serde(rename = "attribute")]
+    pub xs_attribute: Option<Vec<XsAttribute>>,
+    #[serde(rename = "sequence")]
+    pub xs_sequence: Option<XsSequence>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct XsAttribute {
+    #[serde(rename = "@name")]
+    pub name: String,
+    #[serde(rename = "@type")]
+    pub xs_attribute_type: String,
+    #[serde(rename = "@use")]
+    pub xs_attribute_use: String,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct XsSequence {
+    #[serde(rename = "$text")]
+    pub text: Option<String>,
+    #[serde(rename = "any")]
+    pub xs_any: XsAny,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct XsAny {
+    #[serde(rename = "@minOccurs")]
+    pub min_occurs: String,
+    #[serde(rename = "@maxOccurs")]
+    pub max_occurs: String,
+    #[serde(rename = "@processContents")]
+    pub process_contents: String,
+}
+

lib/xml_schema_validator/src/token/element_close_end.rs

@@ -0,0 +1,21 @@
+use crate::*;
+
+/// Represents 
+#[derive(Clone, Debug)]
+pub struct ElementCloseEnd {
+}
+
+impl ElementCloseEnd {
+    pub fn new() -> Self {
+        Self {
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        if '>' == c {
+            vec![std::rc::Rc::new(Token::EndOfFile(EndOfFile::new()))]
+        } else {
+            vec![]
+        }
+    }
+}

lib/xml_schema_validator/src/token/element_close_name.rs

@@ -0,0 +1,51 @@
+use crate::*;
+
+/// Represents a closing element tag that is in the process of being parsed
+/// after the '</' and now parsing the element name
+#[derive(Clone, Debug)]
+pub struct ElementCloseName {
+    /// The element that was previously opened and is now being closed
+    element: std::rc::Rc<schema::XsElement>,
+    /// Current buffer of characters processed for the element name
+    buffer: String,
+}
+
+impl ElementCloseName {
+    /// Create a new instance with the given buffer
+    pub fn new(element: std::rc::Rc<schema::XsElement>, buffer: String) -> Self {
+        Self {
+            element,
+            buffer,
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        let new_buffer = self.buffer.clone() + &c.to_string();
+        
+        // If still building the element name
+        if self.element.name.starts_with(&new_buffer) {
+            // If we've matched the full name
+            if self.element.name == new_buffer {
+                // Now we're expecting '>' to close the tag
+                if c.is_whitespace() {
+                    // Handle whitespace after the element name
+                    vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new(
+                        vec![std::rc::Rc::new(Token::ElementCloseEnd(ElementCloseEnd::new()))]
+                    )))]
+                } else {
+                    // Directly transition to the closed state
+                    vec![std::rc::Rc::new(Token::ElementCloseEnd(ElementCloseEnd::new()))]
+                }
+            } else {
+                // Still building the element name
+                vec![std::rc::Rc::new(Token::ElementCloseName(ElementCloseName::new(
+                    self.element.clone(),
+                    new_buffer,
+                )))]
+            }
+        } else {
+            // The character doesn't match what we expect for this element name
+            vec![]
+        }
+    }
+}

lib/xml_schema_validator/src/token/element_close_start.rs

@@ -0,0 +1,36 @@
+use crate::*;
+
+/// Represents the start of an XML element closing tag (the '</' sequence)
+#[derive(Clone, Debug)]
+pub struct ElementCloseStart {
+    /// The element that was previously opened and is now being closed
+    element: std::rc::Rc<schema::XsElement>,
+    has_slash: bool,
+}
+
+impl ElementCloseStart {
+    pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
+        Self {
+            element,
+            has_slash: false,
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        if !self.has_slash && '/' == c {
+            vec![std::rc::Rc::new(Token::ElementCloseStart(Self {
+                element: self.element.clone(),
+                has_slash: true,
+            }))]
+        } else {
+            if self.element.name.starts_with(c) {
+                vec![std::rc::Rc::new(Token::ElementCloseName(ElementCloseName::new(
+                    self.element.clone(),
+                    c.to_string(),
+                )))]
+            } else {
+                vec![]
+            }
+        }
+    }
+}

lib/xml_schema_validator/src/token/element_open_end.rs

@@ -0,0 +1,35 @@
+use crate::*;
+
+/// Represents a fully opened XML element, containing the schema element reference
+#[derive(Clone, Debug)]
+pub struct ElementOpenEnd {
+    element: std::rc::Rc<schema::XsElement>,
+}
+
+impl ElementOpenEnd {
+    pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
+        Self {
+            element,
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        // Check if the element allows text content (mixed="true")
+        let allows_text = self.element.xs_complex_type.mixed
+            .as_ref()
+            .map(|mixed| mixed == "true")
+            .unwrap_or(false);
+
+        if allows_text {
+            if c.is_whitespace() {
+                vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new(
+                    vec![std::rc::Rc::new(Token::TextContent(TextContent::new(self.element.clone())))]
+                )))]
+            } else {
+                vec![std::rc::Rc::new(Token::TextContent(TextContent::new(self.element.clone())))]
+            }
+        } else {
+            vec![]
+        }
+    }
+}

lib/xml_schema_validator/src/token/element_open_name.rs

@@ -0,0 +1,39 @@
+use crate::*;
+
+/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
+#[derive(Clone, Debug)]
+pub struct ElementOpenName {
+    buffer: String,
+    element: std::rc::Rc<schema::XsElement>,
+}
+
+impl ElementOpenName {
+    pub fn new(element: std::rc::Rc<schema::XsElement>, buffer: String) -> Result<Self> {
+        if element.name.starts_with(buffer.as_str()) {
+            Ok(Self {
+                buffer,
+                element,
+            })
+        } else {
+            Err(Error::NotImplemented)
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        let buffer = self.buffer.clone() + &c.to_string();
+        if self.element.name.starts_with(buffer.as_str()) {
+            vec![std::rc::Rc::new(Token::ElementName(ElementOpenName {
+                buffer,
+                element: self.element.clone(),
+            }))]
+        } else if c.is_whitespace() {
+            vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new(
+                vec![std::rc::Rc::new(Token::ElementOpen(ElementOpenEnd::new(self.element.clone())))]
+            )))]
+        } else if '>' == c {
+            vec![std::rc::Rc::new(Token::ElementOpen(ElementOpenEnd::new(self.element.clone())))]
+        } else {
+            vec![]
+        }
+    }
+}

lib/xml_schema_validator/src/token/element_open_start.rs

@@ -0,0 +1,24 @@
+use crate::*;
+
+/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
+#[derive(Clone, Debug)]
+pub struct ElementOpenStart {
+    element: std::rc::Rc<schema::XsElement>,
+}
+
+impl ElementOpenStart {
+    pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
+        Self {
+            element,
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        let element_name = ElementOpenName::new(self.element.clone(), c.to_string());
+        if let Ok(element_name) = element_name {
+            vec![std::rc::Rc::from(Token::ElementName(element_name))]
+        } else {
+            vec![]
+        }
+    }
+}

lib/xml_schema_validator/src/token/end_of_file.rs

@@ -0,0 +1,16 @@
+use crate::*;
+
+/// Represents the final state at the end of an XML file, containing a reference to the root element schema
+#[derive(Clone, Debug)]
+pub struct EndOfFile {
+}
+
+impl EndOfFile {
+    pub fn new() -> Self {
+        Self {}
+    }
+
+    pub fn append(&self, _c: char) -> Vec<std::rc::Rc<Token>> {
+        vec![]
+    }
+}

lib/xml_schema_validator/src/token/mod.rs

@@ -0,0 +1,53 @@
+mod element_close_end;
+mod element_close_name;
+mod element_close_start;
+mod element_open_name;
+mod element_open_end;
+mod element_open_start;
+mod end_of_file;
+mod start_of_file;
+mod text_content;
+mod whitespace;
+
+pub use element_close_end::ElementCloseEnd;
+pub use element_close_name::ElementCloseName;
+pub use element_close_start::ElementCloseStart;
+pub use element_open_name::ElementOpenName;
+pub use element_open_end::ElementOpenEnd;
+pub use element_open_start::ElementOpenStart;
+pub use end_of_file::EndOfFile;
+pub use start_of_file::StartOfFile;
+pub use text_content::TextContent;
+pub use whitespace::Whitespace;
+
+/// Represents the different states of XML token parsing
+#[derive(Clone, Debug)]
+pub enum Token {
+    ElementCloseEnd(ElementCloseEnd),
+    ElementCloseName(ElementCloseName),
+    ElementCloseStart(ElementCloseStart),
+    ElementName(ElementOpenName),
+    ElementOpen(ElementOpenEnd),
+    ElementStart(ElementOpenStart),
+    EndOfFile(EndOfFile),
+    StartOfFile(StartOfFile),
+    TextContent(TextContent),
+    Whitespace(Whitespace),
+}
+
+impl Token {
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        match self {
+            Token::ElementCloseEnd(element_close_end) => element_close_end.append(c),
+            Token::ElementCloseName(element_close_name) => element_close_name.append(c),
+            Token::ElementCloseStart(element_close_start) => element_close_start.append(c),
+            Token::ElementName(element_name) => element_name.append(c),
+            Token::ElementOpen(element_open) => element_open.append(c),
+            Token::ElementStart(element_start) => element_start.append(c),
+            Token::EndOfFile(end_of_file) => end_of_file.append(c),
+            Token::StartOfFile(start_of_file) => start_of_file.append(c),
+            Token::TextContent(text_content) => text_content.append(c),
+            Token::Whitespace(whitespace) => whitespace.append(c),
+        }
+    }
+}

lib/xml_schema_validator/src/token/start_of_file.rs

@@ -0,0 +1,21 @@
+use crate::*;
+
+/// Represents the initial state at the start of an XML file, containing a reference to the root element schema
+#[derive(Clone, Debug)]
+pub struct StartOfFile {
+    element: std::rc::Rc<schema::XsElement>,
+}
+
+impl StartOfFile {
+    pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
+        Self { element }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        if '<' == c {
+            vec![std::rc::Rc::from(Token::ElementStart(ElementOpenStart::new(self.element.clone())))]
+        } else {
+            vec![]
+        }
+    }
+}

lib/xml_schema_validator/src/token/text_content.rs

@@ -0,0 +1,36 @@
+use crate::*;
+
+/// Represents the text content within an XML element
+#[derive(Clone, Debug)]
+pub struct TextContent {
+    element: std::rc::Rc<schema::XsElement>,
+    buffer: String,
+}
+
+impl TextContent {
+    pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
+        Self {
+            element,
+            buffer: String::new(),
+        }
+    }
+
+    /// Create a new TextContent with the given initial buffer
+    pub fn with_buffer(element: std::rc::Rc<schema::XsElement>, buffer: String) -> Self {
+        Self {
+            element,
+            buffer,
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        if c == '<' {
+            vec![std::rc::Rc::new(Token::ElementCloseStart(ElementCloseStart::new(self.element.clone())))]
+        } else {
+            vec![std::rc::Rc::new(Token::TextContent(TextContent::with_buffer(
+                self.element.clone(),
+                self.buffer.clone() + &c.to_string(),
+            )))]
+        }
+    }
+}

lib/xml_schema_validator/src/token/whitespace.rs

@@ -0,0 +1,29 @@
+use crate::*;
+
+/// Represents one or more whitespace characters
+#[derive(Clone, Debug)]
+pub struct Whitespace {
+    next: Vec<std::rc::Rc<Token>>
+}
+
+impl Whitespace {
+    pub fn new(next: Vec<std::rc::Rc<Token>>) -> Self {
+        Self {
+            next
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        if c.is_whitespace() {
+            vec![std::rc::Rc::new(Token::Whitespace(Self {
+                next: self.next.clone()
+            }))]
+        } else {
+            let mut tokens = vec![];
+            for token in self.next.iter() {
+                tokens.append(&mut token.append(c))
+            }
+            tokens
+        }
+    }
+}