Begin XML Schema Validator

Makefile

@@ -0,0 +1,186 @@
+# SIA Project Makefile
+# ==================
+
+# Default variables
+PYTHON_DIRS = sia tools test
+RUST_DIRS = lib/xml_schema_validator
+WEB_DIRS = web
+
+# Default target
+.PHONY: all
+all: help
+
+# ==================
+# Clean targets
+# ==================
+
+.PHONY: clean
+clean: clean-python clean-rust clean-docker clean-iterations clean-web clean-build clean-temp ## Clean all project artifacts
+
+.PHONY: clean-python
+clean-python: ## Clean Python artifacts
+	@echo "Cleaning Python artifacts..."
+	find $(PYTHON_DIRS) -type d -name "__pycache__" -exec rm -rf {} +
+	find $(PYTHON_DIRS) -type d -name "*.egg-info" -exec rm -rf {} +
+	find $(PYTHON_DIRS) -type f -name "*.pyc" -delete
+	find $(PYTHON_DIRS) -type f -name "*.pyo" -delete
+	find $(PYTHON_DIRS) -type f -name "*.pyd" -delete
+	find $(PYTHON_DIRS) -type f -name ".coverage" -delete
+	find $(PYTHON_DIRS) -type d -name "htmlcov" -exec rm -rf {} +
+	find $(PYTHON_DIRS) -type d -name ".pytest_cache" -exec rm -rf {} +
+	find $(PYTHON_DIRS) -type d -name ".coverage*" -exec rm -rf {} +
+	rm -rf .pytest_cache
+	rm -rf .coverage
+	rm -rf htmlcov
+	rm -rf .tox
+	rm -rf dist
+	rm -rf build
+
+.PHONY: clean-rust
+clean-rust: ## Clean Rust artifacts
+	@echo "Cleaning Rust artifacts..."
+	for dir in $(RUST_DIRS); do \
+		if [ -f "$$dir/Cargo.toml" ]; then \
+			(cd "$$dir" && cargo clean); \
+		fi; \
+	done
+
+.PHONY: clean-docker
+clean-docker: ## Clean Docker artifacts
+	@echo "Cleaning Docker artifacts..."
+	-docker system prune -f
+
+.PHONY: clean-iterations
+clean-iterations: ## Clean iteration files (WARNING: data loss)
+	@echo "WARNING: This will delete all iteration files in data/iterations"
+	@echo "Press Ctrl+C to cancel or Enter to continue"
+	@read
+	rm -rf data/iterations/*.xml
+
+.PHONY: clean-web
+clean-web: ## Clean web build artifacts
+	@echo "Cleaning web build artifacts..."
+	rm -rf web/dist
+	rm -rf web/node_modules
+	rm -rf static
+
+.PHONY: clean-build
+clean-build: ## Clean build artifacts
+	@echo "Cleaning build artifacts..."
+	rm -rf build/
+	rm -rf dist/
+	rm -rf *.egg-info/
+
+.PHONY: clean-temp
+clean-temp: ## Clean temporary files
+	@echo "Cleaning temporary files..."
+	find . -type f -name "*.log" -delete
+	find . -type f -name "*.tmp" -delete
+	find . -type f -name ".DS_Store" -delete
+	find . -type f -name "*.swp" -delete
+	find . -type f -name "*~" -delete
+
+# ==================
+# Format targets
+# ==================
+
+.PHONY: format
+format: format-python format-rust ## Format all code
+
+.PHONY: format-python
+format-python: ## Format Python code
+	@echo "Formatting Python code..."
+	black $(PYTHON_DIRS)
+
+.PHONY: format-rust
+format-rust: ## Format Rust code
+	@echo "Formatting Rust code..."
+	for dir in $(RUST_DIRS); do \
+		if [ -f "$$dir/Cargo.toml" ]; then \
+			(cd "$$dir" && cargo fmt); \
+		fi; \
+	done
+
+# ==================
+# Lint targets
+# ==================
+
+.PHONY: lint
+lint: lint-python lint-rust ## Lint all code
+
+.PHONY: lint-python
+lint-python: ## Lint Python code
+	@echo "Linting Python code..."
+	flake8 $(PYTHON_DIRS)
+
+.PHONY: lint-rust
+lint-rust: ## Lint Rust code
+	@echo "Linting Rust code..."
+	for dir in $(RUST_DIRS); do \
+		if [ -f "$$dir/Cargo.toml" ]; then \
+			(cd "$$dir" && cargo clippy -- -D warnings); \
+		fi; \
+	done
+
+# ==================
+# Test targets
+# ==================
+
+.PHONY: test
+test: test-python test-rust ## Run all tests
+
+.PHONY: test-python
+test-python: ## Run Python tests
+	@echo "Running Python tests..."
+	python -m pytest -xvs
+
+.PHONY: test-rust
+test-rust: ## Run Rust tests
+	@echo "Running Rust tests..."
+	for dir in $(RUST_DIRS); do \
+		if [ -f "$$dir/Cargo.toml" ]; then \
+			(cd "$$dir" && cargo test); \
+		fi; \
+	done
+
+# ==================
+# Build targets
+# ==================
+
+.PHONY: build
+build: build-rust build-web ## Build all components
+
+.PHONY: build-rust
+build-rust: ## Build Rust components
+	@echo "Building Rust components..."
+	for dir in $(RUST_DIRS); do \
+		if [ -f "$$dir/Cargo.toml" ]; then \
+			(cd "$$dir" && cargo build --release); \
+		fi; \
+	done
+
+.PHONY: build-web
+build-web: ## Build web interface
+	@echo "Building web interface..."
+	cd web && npm install && npm run build
+
+.PHONY: docker-build
+docker-build: ## Build Docker image
+	@echo "Building Docker image..."
+	docker build -t sia .
+
+# ==================
+# Utility targets
+# ==================
+
+.PHONY: help
+help: ## Show this help message
+	@echo "Usage: make [target]"
+	@echo ""
+	@echo "Targets:"
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: collect
+collect: ## Run collect.sh script
+	@echo "Running collect.sh script..."
+	./scripts/collect.sh -s core -s lib -s webui -o collect.txt

lib/xml_schema_validator/.gitignore

@@ -0,0 +1,2 @@
+target/
+Cargo.lock

lib/xml_schema_validator/build.rs

@@ -0,0 +1,15 @@
+pub fn main() {
+    println!("cargo:rerun-if-changed=../../action_schema.xsd");
+    let output = std::process::Command::new("xml_schema_generator.exe")
+            .arg("--derive")
+            .arg("Clone, Debug, Deserialize, Serialize")
+            .arg("../../action_schema.xsd")
+            .arg("src/schema.rs")
+            .output()
+            .expect("Failed to execute xml_schema_generator");
+    
+        if !output.status.success() {
+            panic!("xml_schema_generator failed: {}", 
+                String::from_utf8_lossy(&output.stderr));
+        }
+}

lib/xml_schema_validator/cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "xml_schema_validator"
+version = "0.1.0"
+edition = "2021"
+authors = ["SIA Team"]
+description = "XML Schema validation library optimized for llm token validation"
+
+[lib]
+name = "xml_schema_validator"
+# Required for PyO3
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+# XML parsing utilities
+quick-xml = { version = "0.28.1", features = ["serde", "serialize"] }
+serde = { version = "1", features = ["derive"] }
+thiserror = "1.0.38"    # Better error handling
+
+# Python bindings
+pyo3 = { version = "0.18.0", features = ["extension-module"] }
+
+[build-dependencies]
+xml_schema_generator = "0.6.18"

lib/xml_schema_validator/pyproject.toml

@@ -0,0 +1,25 @@
+[build-system]
+requires = ["maturin>=1.0,<2.0"]
+build-backend = "maturin"
+
+[project]
+name = "xml_schema_validator"
+version = "0.1.0"
+description = "XML Schema validation library for SIA"
+readme = "README.md"
+requires-python = ">=3.10"
+classifiers = [
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Text Processing :: Markup :: XML",
+]
+dependencies = []
+
+[project.urls]
+repository = "https://github.com/sia/xml_schema_validator"
+
+[tool.maturin]
+python-source = "python"
+features = ["pyo3/extension-module"]

lib/xml_schema_validator/src/error.rs

@@ -0,0 +1,14 @@
+/// Error types that can occur during XML validation
+
+#[derive(Clone, Debug, thiserror::Error)]
+pub enum Error {
+    /// Indicates that the provided XSD schema is invalid
+    #[error("Invalid schema: {0}")]
+    InvalidSchema(#[from] quick_xml::DeError),
+    /// Indicates that the provided XML fragment is invalid
+    #[error("Invalid XML: {0}")]
+    InvalidXml(String),
+    /// Not implemented yet
+    #[error("Not implemented yet")]
+    NotImplemented,
+}

lib/xml_schema_validator/src/lib.rs

@@ -0,0 +1,104 @@
+#![warn(missing_docs)]
+#![warn(rust_2018_idioms)]
+
+//! A streaming XML validator that checks XML fragments against an XSD schema
+
+mod error;
+mod schema;
+mod token;
+
+use error::Error;
+use token::*;
+
+/// An XML validator that checks XML fragments against an XSD schema
+#[derive(Clone, Debug)]
+pub struct XmlValidator {
+    current_tokens: Vec<std::rc::Rc<Token>>,
+}
+
+/// The Result type used throughout this crate
+///
+/// This is a type alias for `std::result::Result` with our custom `Error` type
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl XmlValidator {
+    /// Create a new validator with the given schema text
+    pub fn new(schema_text: &str) -> Result<Self> {
+        let schema: schema::XsSchema = quick_xml::de::from_str(schema_text)?;
+        let current_tokens = schema.xs_element.iter().map(|element| {
+            std::rc::Rc::new(Token::StartOfFile(StartOfFile::new(std::rc::Rc::new(element.clone()))))
+        }).collect();
+        
+        //let current_tokens = vec![std::rc::Rc::new(Token::StartOfFile(StartOfFile))];
+        Ok(Self {
+            current_tokens,
+        })
+    }
+    
+    /// Append a fragment of XML to the validator
+    pub fn append(&mut self, fragment: &str) -> Result<()> {
+        for c in fragment.chars() {
+            self.append_char(c)?;
+        }
+        Ok(())
+    }
+
+    /// Append a single character to the validator
+    fn append_char(&mut self, c: char) -> Result<()> {
+        println!("Appending char: {}", c);
+        let mut new_tokens = vec![];
+        for token in self.current_tokens.clone() {
+            new_tokens.append(&mut token.append(c));
+        }
+        if new_tokens.is_empty() {
+            return Err(Error::InvalidXml("No continuations".to_string()));
+        }
+        self.current_tokens = new_tokens;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_element_open() {
+        let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
+        let input = "<reasoning>";
+        let mut validator = XmlValidator::new(&schema_text).unwrap();
+        validator.append(input).unwrap();
+    }
+
+    #[test]
+    fn test_element_open_whitespace() {
+        let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
+        let input = "<reasoning >";
+        let mut validator = XmlValidator::new(&schema_text).unwrap();
+        validator.append(input).unwrap();
+    }
+
+    #[test]
+    fn test_element_open_invalid_name() {
+        let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
+        let input = "<rae";
+        let mut validator = XmlValidator::new(&schema_text).unwrap();
+        validator.append(input).expect_err("Should fail");
+    }
+    
+    #[test]
+    fn test_element_close() {
+        let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
+        let input = "<reasoning>test</reasoning>";
+        let mut validator = XmlValidator::new(&schema_text).unwrap();
+        validator.append(input).unwrap();
+    }
+    
+    #[test]
+    fn test_element_close_mismatched() {
+        let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap();
+        let input = "<reasoning>test</wrong>";
+        let mut validator = XmlValidator::new(&schema_text).unwrap();
+        validator.append(input).expect_err("Should fail with mismatched closing tag");
+    }
+}

lib/xml_schema_validator/src/schema.rs

@@ -0,0 +1,64 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct XsSchema {
+    #[serde(rename = "@xmlns:xs")]
+    pub xmlns_xs: String,
+    #[serde(rename = "@elementFormDefault")]
+    pub element_form_default: String,
+    #[serde(rename = "$text")]
+    pub text: Option<String>,
+    #[serde(rename = "element")]
+    pub xs_element: Vec<XsElement>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct XsElement {
+    #[serde(rename = "@name")]
+    pub name: String,
+    #[serde(rename = "$text")]
+    pub text: Option<String>,
+    #[serde(rename = "complexType")]
+    pub xs_complex_type: XsComplexType,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct XsComplexType {
+    #[serde(rename = "@mixed")]
+    pub mixed: Option<String>,
+    #[serde(rename = "$text")]
+    pub text: Option<String>,
+    #[serde(rename = "attribute")]
+    pub xs_attribute: Option<Vec<XsAttribute>>,
+    #[serde(rename = "sequence")]
+    pub xs_sequence: Option<XsSequence>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct XsAttribute {
+    #[serde(rename = "@name")]
+    pub name: String,
+    #[serde(rename = "@type")]
+    pub xs_attribute_type: String,
+    #[serde(rename = "@use")]
+    pub xs_attribute_use: String,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct XsSequence {
+    #[serde(rename = "$text")]
+    pub text: Option<String>,
+    #[serde(rename = "any")]
+    pub xs_any: XsAny,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct XsAny {
+    #[serde(rename = "@minOccurs")]
+    pub min_occurs: String,
+    #[serde(rename = "@maxOccurs")]
+    pub max_occurs: String,
+    #[serde(rename = "@processContents")]
+    pub process_contents: String,
+}
+

lib/xml_schema_validator/src/token/element_close_end.rs

@@ -0,0 +1,21 @@
+use crate::*;
+
+/// Represents 
+#[derive(Clone, Debug)]
+pub struct ElementCloseEnd {
+}
+
+impl ElementCloseEnd {
+    pub fn new() -> Self {
+        Self {
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        if '>' == c {
+            vec![std::rc::Rc::new(Token::EndOfFile(EndOfFile::new()))]
+        } else {
+            vec![]
+        }
+    }
+}

lib/xml_schema_validator/src/token/element_close_name.rs

@@ -0,0 +1,51 @@
+use crate::*;
+
+/// Represents a closing element tag that is in the process of being parsed
+/// after the '</' and now parsing the element name
+#[derive(Clone, Debug)]
+pub struct ElementCloseName {
+    /// The element that was previously opened and is now being closed
+    element: std::rc::Rc<schema::XsElement>,
+    /// Current buffer of characters processed for the element name
+    buffer: String,
+}
+
+impl ElementCloseName {
+    /// Create a new instance with the given buffer
+    pub fn new(element: std::rc::Rc<schema::XsElement>, buffer: String) -> Self {
+        Self {
+            element,
+            buffer,
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        let new_buffer = self.buffer.clone() + &c.to_string();
+        
+        // If still building the element name
+        if self.element.name.starts_with(&new_buffer) {
+            // If we've matched the full name
+            if self.element.name == new_buffer {
+                // Now we're expecting '>' to close the tag
+                if c.is_whitespace() {
+                    // Handle whitespace after the element name
+                    vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new(
+                        vec![std::rc::Rc::new(Token::ElementCloseEnd(ElementCloseEnd::new()))]
+                    )))]
+                } else {
+                    // Directly transition to the closed state
+                    vec![std::rc::Rc::new(Token::ElementCloseEnd(ElementCloseEnd::new()))]
+                }
+            } else {
+                // Still building the element name
+                vec![std::rc::Rc::new(Token::ElementCloseName(ElementCloseName::new(
+                    self.element.clone(),
+                    new_buffer,
+                )))]
+            }
+        } else {
+            // The character doesn't match what we expect for this element name
+            vec![]
+        }
+    }
+}

lib/xml_schema_validator/src/token/element_close_start.rs

@@ -0,0 +1,36 @@
+use crate::*;
+
+/// Represents the start of an XML element closing tag (the '</' sequence)
+#[derive(Clone, Debug)]
+pub struct ElementCloseStart {
+    /// The element that was previously opened and is now being closed
+    element: std::rc::Rc<schema::XsElement>,
+    has_slash: bool,
+}
+
+impl ElementCloseStart {
+    pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
+        Self {
+            element,
+            has_slash: false,
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        if !self.has_slash && '/' == c {
+            vec![std::rc::Rc::new(Token::ElementCloseStart(Self {
+                element: self.element.clone(),
+                has_slash: true,
+            }))]
+        } else {
+            if self.element.name.starts_with(c) {
+                vec![std::rc::Rc::new(Token::ElementCloseName(ElementCloseName::new(
+                    self.element.clone(),
+                    c.to_string(),
+                )))]
+            } else {
+                vec![]
+            }
+        }
+    }
+}

lib/xml_schema_validator/src/token/element_open_end.rs

@@ -0,0 +1,35 @@
+use crate::*;
+
+/// Represents a fully opened XML element, containing the schema element reference
+#[derive(Clone, Debug)]
+pub struct ElementOpenEnd {
+    element: std::rc::Rc<schema::XsElement>,
+}
+
+impl ElementOpenEnd {
+    pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
+        Self {
+            element,
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        // Check if the element allows text content (mixed="true")
+        let allows_text = self.element.xs_complex_type.mixed
+            .as_ref()
+            .map(|mixed| mixed == "true")
+            .unwrap_or(false);
+
+        if allows_text {
+            if c.is_whitespace() {
+                vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new(
+                    vec![std::rc::Rc::new(Token::TextContent(TextContent::new(self.element.clone())))]
+                )))]
+            } else {
+                vec![std::rc::Rc::new(Token::TextContent(TextContent::new(self.element.clone())))]
+            }
+        } else {
+            vec![]
+        }
+    }
+}

lib/xml_schema_validator/src/token/element_open_name.rs

@@ -0,0 +1,39 @@
+use crate::*;
+
+/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
+#[derive(Clone, Debug)]
+pub struct ElementOpenName {
+    buffer: String,
+    element: std::rc::Rc<schema::XsElement>,
+}
+
+impl ElementOpenName {
+    pub fn new(element: std::rc::Rc<schema::XsElement>, buffer: String) -> Result<Self> {
+        if element.name.starts_with(buffer.as_str()) {
+            Ok(Self {
+                buffer,
+                element,
+            })
+        } else {
+            Err(Error::NotImplemented)
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        let buffer = self.buffer.clone() + &c.to_string();
+        if self.element.name.starts_with(buffer.as_str()) {
+            vec![std::rc::Rc::new(Token::ElementName(ElementOpenName {
+                buffer,
+                element: self.element.clone(),
+            }))]
+        } else if c.is_whitespace() {
+            vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new(
+                vec![std::rc::Rc::new(Token::ElementOpen(ElementOpenEnd::new(self.element.clone())))]
+            )))]
+        } else if '>' == c {
+            vec![std::rc::Rc::new(Token::ElementOpen(ElementOpenEnd::new(self.element.clone())))]
+        } else {
+            vec![]
+        }
+    }
+}

lib/xml_schema_validator/src/token/element_open_start.rs

@@ -0,0 +1,24 @@
+use crate::*;
+
+/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
+#[derive(Clone, Debug)]
+pub struct ElementOpenStart {
+    element: std::rc::Rc<schema::XsElement>,
+}
+
+impl ElementOpenStart {
+    pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
+        Self {
+            element,
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        let element_name = ElementOpenName::new(self.element.clone(), c.to_string());
+        if let Ok(element_name) = element_name {
+            vec![std::rc::Rc::from(Token::ElementName(element_name))]
+        } else {
+            vec![]
+        }
+    }
+}

lib/xml_schema_validator/src/token/end_of_file.rs

@@ -0,0 +1,16 @@
+use crate::*;
+
+/// Represents the final state at the end of an XML file, containing a reference to the root element schema
+#[derive(Clone, Debug)]
+pub struct EndOfFile {
+}
+
+impl EndOfFile {
+    pub fn new() -> Self {
+        Self {}
+    }
+
+    pub fn append(&self, _c: char) -> Vec<std::rc::Rc<Token>> {
+        vec![]
+    }
+}

lib/xml_schema_validator/src/token/mod.rs

@@ -0,0 +1,53 @@
+mod element_close_end;
+mod element_close_name;
+mod element_close_start;
+mod element_open_name;
+mod element_open_end;
+mod element_open_start;
+mod end_of_file;
+mod start_of_file;
+mod text_content;
+mod whitespace;
+
+pub use element_close_end::ElementCloseEnd;
+pub use element_close_name::ElementCloseName;
+pub use element_close_start::ElementCloseStart;
+pub use element_open_name::ElementOpenName;
+pub use element_open_end::ElementOpenEnd;
+pub use element_open_start::ElementOpenStart;
+pub use end_of_file::EndOfFile;
+pub use start_of_file::StartOfFile;
+pub use text_content::TextContent;
+pub use whitespace::Whitespace;
+
+/// Represents the different states of XML token parsing
+#[derive(Clone, Debug)]
+pub enum Token {
+    ElementCloseEnd(ElementCloseEnd),
+    ElementCloseName(ElementCloseName),
+    ElementCloseStart(ElementCloseStart),
+    ElementName(ElementOpenName),
+    ElementOpen(ElementOpenEnd),
+    ElementStart(ElementOpenStart),
+    EndOfFile(EndOfFile),
+    StartOfFile(StartOfFile),
+    TextContent(TextContent),
+    Whitespace(Whitespace),
+}
+
+impl Token {
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        match self {
+            Token::ElementCloseEnd(element_close_end) => element_close_end.append(c),
+            Token::ElementCloseName(element_close_name) => element_close_name.append(c),
+            Token::ElementCloseStart(element_close_start) => element_close_start.append(c),
+            Token::ElementName(element_name) => element_name.append(c),
+            Token::ElementOpen(element_open) => element_open.append(c),
+            Token::ElementStart(element_start) => element_start.append(c),
+            Token::EndOfFile(end_of_file) => end_of_file.append(c),
+            Token::StartOfFile(start_of_file) => start_of_file.append(c),
+            Token::TextContent(text_content) => text_content.append(c),
+            Token::Whitespace(whitespace) => whitespace.append(c),
+        }
+    }
+}

lib/xml_schema_validator/src/token/start_of_file.rs

@@ -0,0 +1,21 @@
+use crate::*;
+
+/// Represents the initial state at the start of an XML file, containing a reference to the root element schema
+#[derive(Clone, Debug)]
+pub struct StartOfFile {
+    element: std::rc::Rc<schema::XsElement>,
+}
+
+impl StartOfFile {
+    pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
+        Self { element }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        if '<' == c {
+            vec![std::rc::Rc::from(Token::ElementStart(ElementOpenStart::new(self.element.clone())))]
+        } else {
+            vec![]
+        }
+    }
+}

lib/xml_schema_validator/src/token/text_content.rs

@@ -0,0 +1,36 @@
+use crate::*;
+
+/// Represents the text content within an XML element
+#[derive(Clone, Debug)]
+pub struct TextContent {
+    element: std::rc::Rc<schema::XsElement>,
+    buffer: String,
+}
+
+impl TextContent {
+    pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
+        Self {
+            element,
+            buffer: String::new(),
+        }
+    }
+
+    /// Create a new TextContent with the given initial buffer
+    pub fn with_buffer(element: std::rc::Rc<schema::XsElement>, buffer: String) -> Self {
+        Self {
+            element,
+            buffer,
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        if c == '<' {
+            vec![std::rc::Rc::new(Token::ElementCloseStart(ElementCloseStart::new(self.element.clone())))]
+        } else {
+            vec![std::rc::Rc::new(Token::TextContent(TextContent::with_buffer(
+                self.element.clone(),
+                self.buffer.clone() + &c.to_string(),
+            )))]
+        }
+    }
+}

lib/xml_schema_validator/src/token/whitespace.rs

@@ -0,0 +1,29 @@
+use crate::*;
+
+/// Represents one or more whitespace characters
+#[derive(Clone, Debug)]
+pub struct Whitespace {
+    next: Vec<std::rc::Rc<Token>>
+}
+
+impl Whitespace {
+    pub fn new(next: Vec<std::rc::Rc<Token>>) -> Self {
+        Self {
+            next
+        }
+    }
+
+    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+        if c.is_whitespace() {
+            vec![std::rc::Rc::new(Token::Whitespace(Self {
+                next: self.next.clone()
+            }))]
+        } else {
+            let mut tokens = vec![];
+            for token in self.next.iter() {
+                tokens.append(&mut token.append(c))
+            }
+            tokens
+        }
+    }
+}

scripts/collect.sh

@@ -4,11 +4,13 @@ set -euo pipefail
 
 declare -A FILTER_SETS=(
   ["py"]="-f .*(\\.py|requirements.txt)$"
+  ["rust"]="-f .*\\.(rs|toml)$"
   ["web"]="-f .*\\.(js|jsx|json|css|html)$"
   ["doc"]="-f .*\\.md$"
-  ["deploy"]="-f .*(Dockerfile|\\.sh|\\.xsd|\\.yaml)$"
+  ["deploy"]="-f .*(Dockerfile|\\.sh|\\.xsd|\\.yaml|\\akefile)$"
   
   ["core"]="-s py ./sia ./tools -s deploy . -f ^(?!procedures/).*\\.md$ ."
+  ["lib"]="-s rust . -s py . -s doc . -s deploy ."
   ["webui"]="-s web ./web"
   ["tests"]="-s py ./test"
   ["procedures"]="-s doc ./procedures"

scripts/deploy.sh

@@ -10,7 +10,7 @@ export MSYS_NO_PATHCONV=1
 # These can be overridden by environment variables or .env file
 
 # Pod configuration
-GPU_TYPE=${GPU_TYPE:-"NVIDIA RTX 6000 Ada Generation"}
+GPU_TYPE=${GPU_TYPE:-"NVIDIA RTX A6000"}
 GPU_COUNT=${GPU_COUNT:-1}
 CONTAINER_DISK_SIZE=${CONTAINER_DISK_SIZE:-200}     # GB
 CPU_COUNT=${CPU_COUNT:-1}                           # vCPUs