From 55a920414d2f3a7a6e10d273eb6ebda7a38794fd Mon Sep 17 00:00:00 2001 From: Geens Date: Sat, 5 Apr 2025 23:15:39 +0200 Subject: [PATCH] Begin XML Schema Validator --- Makefile | 186 ++++++++++++++++++ lib/xml_schema_validator/.gitignore | 2 + lib/xml_schema_validator/build.rs | 15 ++ lib/xml_schema_validator/cargo.toml | 23 +++ lib/xml_schema_validator/pyproject.toml | 25 +++ lib/xml_schema_validator/src/error.rs | 14 ++ lib/xml_schema_validator/src/lib.rs | 104 ++++++++++ lib/xml_schema_validator/src/schema.rs | 64 ++++++ .../src/token/element_close_end.rs | 21 ++ .../src/token/element_close_name.rs | 51 +++++ .../src/token/element_close_start.rs | 36 ++++ .../src/token/element_open_end.rs | 35 ++++ .../src/token/element_open_name.rs | 39 ++++ .../src/token/element_open_start.rs | 24 +++ .../src/token/end_of_file.rs | 16 ++ lib/xml_schema_validator/src/token/mod.rs | 53 +++++ .../src/token/start_of_file.rs | 21 ++ .../src/token/text_content.rs | 36 ++++ .../src/token/whitespace.rs | 29 +++ scripts/collect.sh | 4 +- scripts/deploy.sh | 2 +- 21 files changed, 798 insertions(+), 2 deletions(-) create mode 100644 Makefile create mode 100644 lib/xml_schema_validator/.gitignore create mode 100644 lib/xml_schema_validator/build.rs create mode 100644 lib/xml_schema_validator/cargo.toml create mode 100644 lib/xml_schema_validator/pyproject.toml create mode 100644 lib/xml_schema_validator/src/error.rs create mode 100644 lib/xml_schema_validator/src/lib.rs create mode 100644 lib/xml_schema_validator/src/schema.rs create mode 100644 lib/xml_schema_validator/src/token/element_close_end.rs create mode 100644 lib/xml_schema_validator/src/token/element_close_name.rs create mode 100644 lib/xml_schema_validator/src/token/element_close_start.rs create mode 100644 lib/xml_schema_validator/src/token/element_open_end.rs create mode 100644 lib/xml_schema_validator/src/token/element_open_name.rs create mode 100644 lib/xml_schema_validator/src/token/element_open_start.rs create mode 100644 lib/xml_schema_validator/src/token/end_of_file.rs create mode 100644 lib/xml_schema_validator/src/token/mod.rs create mode 100644 lib/xml_schema_validator/src/token/start_of_file.rs create mode 100644 lib/xml_schema_validator/src/token/text_content.rs create mode 100644 lib/xml_schema_validator/src/token/whitespace.rs diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0faf653 --- /dev/null +++ b/Makefile @@ -0,0 +1,186 @@ +# SIA Project Makefile +# ================== + +# Default variables +PYTHON_DIRS = sia tools test +RUST_DIRS = lib/xml_schema_validator +WEB_DIRS = web + +# Default target +.PHONY: all +all: help + +# ================== +# Clean targets +# ================== + +.PHONY: clean +clean: clean-python clean-rust clean-docker clean-iterations clean-web clean-build clean-temp ## Clean all project artifacts + +.PHONY: clean-python +clean-python: ## Clean Python artifacts + @echo "Cleaning Python artifacts..." + find $(PYTHON_DIRS) -type d -name "__pycache__" -exec rm -rf {} + + find $(PYTHON_DIRS) -type d -name "*.egg-info" -exec rm -rf {} + + find $(PYTHON_DIRS) -type f -name "*.pyc" -delete + find $(PYTHON_DIRS) -type f -name "*.pyo" -delete + find $(PYTHON_DIRS) -type f -name "*.pyd" -delete + find $(PYTHON_DIRS) -type f -name ".coverage" -delete + find $(PYTHON_DIRS) -type d -name "htmlcov" -exec rm -rf {} + + find $(PYTHON_DIRS) -type d -name ".pytest_cache" -exec rm -rf {} + + find $(PYTHON_DIRS) -type d -name ".coverage*" -exec rm -rf {} + + rm -rf .pytest_cache + rm -rf .coverage + rm -rf htmlcov + rm -rf .tox + rm -rf dist + rm -rf build + +.PHONY: clean-rust +clean-rust: ## Clean Rust artifacts + @echo "Cleaning Rust artifacts..." + for dir in $(RUST_DIRS); do \ + if [ -f "$$dir/Cargo.toml" ]; then \ + (cd "$$dir" && cargo clean); \ + fi; \ + done + +.PHONY: clean-docker +clean-docker: ## Clean Docker artifacts + @echo "Cleaning Docker artifacts..." + -docker system prune -f + +.PHONY: clean-iterations +clean-iterations: ## Clean iteration files (WARNING: data loss) + @echo "WARNING: This will delete all iteration files in data/iterations" + @echo "Press Ctrl+C to cancel or Enter to continue" + @read + rm -rf data/iterations/*.xml + +.PHONY: clean-web +clean-web: ## Clean web build artifacts + @echo "Cleaning web build artifacts..." + rm -rf web/dist + rm -rf web/node_modules + rm -rf static + +.PHONY: clean-build +clean-build: ## Clean build artifacts + @echo "Cleaning build artifacts..." + rm -rf build/ + rm -rf dist/ + rm -rf *.egg-info/ + +.PHONY: clean-temp +clean-temp: ## Clean temporary files + @echo "Cleaning temporary files..." + find . -type f -name "*.log" -delete + find . -type f -name "*.tmp" -delete + find . -type f -name ".DS_Store" -delete + find . -type f -name "*.swp" -delete + find . -type f -name "*~" -delete + +# ================== +# Format targets +# ================== + +.PHONY: format +format: format-python format-rust ## Format all code + +.PHONY: format-python +format-python: ## Format Python code + @echo "Formatting Python code..." + black $(PYTHON_DIRS) + +.PHONY: format-rust +format-rust: ## Format Rust code + @echo "Formatting Rust code..." + for dir in $(RUST_DIRS); do \ + if [ -f "$$dir/Cargo.toml" ]; then \ + (cd "$$dir" && cargo fmt); \ + fi; \ + done + +# ================== +# Lint targets +# ================== + +.PHONY: lint +lint: lint-python lint-rust ## Lint all code + +.PHONY: lint-python +lint-python: ## Lint Python code + @echo "Linting Python code..." + flake8 $(PYTHON_DIRS) + +.PHONY: lint-rust +lint-rust: ## Lint Rust code + @echo "Linting Rust code..." + for dir in $(RUST_DIRS); do \ + if [ -f "$$dir/Cargo.toml" ]; then \ + (cd "$$dir" && cargo clippy -- -D warnings); \ + fi; \ + done + +# ================== +# Test targets +# ================== + +.PHONY: test +test: test-python test-rust ## Run all tests + +.PHONY: test-python +test-python: ## Run Python tests + @echo "Running Python tests..." + python -m pytest -xvs + +.PHONY: test-rust +test-rust: ## Run Rust tests + @echo "Running Rust tests..." + for dir in $(RUST_DIRS); do \ + if [ -f "$$dir/Cargo.toml" ]; then \ + (cd "$$dir" && cargo test); \ + fi; \ + done + +# ================== +# Build targets +# ================== + +.PHONY: build +build: build-rust build-web ## Build all components + +.PHONY: build-rust +build-rust: ## Build Rust components + @echo "Building Rust components..." + for dir in $(RUST_DIRS); do \ + if [ -f "$$dir/Cargo.toml" ]; then \ + (cd "$$dir" && cargo build --release); \ + fi; \ + done + +.PHONY: build-web +build-web: ## Build web interface + @echo "Building web interface..." + cd web && npm install && npm run build + +.PHONY: docker-build +docker-build: ## Build Docker image + @echo "Building Docker image..." + docker build -t sia . + +# ================== +# Utility targets +# ================== + +.PHONY: help +help: ## Show this help message + @echo "Usage: make [target]" + @echo "" + @echo "Targets:" + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-15s\033[0m %s\n", $$1, $$2}' + +.PHONY: collect +collect: ## Run collect.sh script + @echo "Running collect.sh script..." + ./scripts/collect.sh -s core -s lib -s webui -o collect.txt \ No newline at end of file diff --git a/lib/xml_schema_validator/.gitignore b/lib/xml_schema_validator/.gitignore new file mode 100644 index 0000000..2630b86 --- /dev/null +++ b/lib/xml_schema_validator/.gitignore @@ -0,0 +1,2 @@ +target/ +Cargo.lock \ No newline at end of file diff --git a/lib/xml_schema_validator/build.rs b/lib/xml_schema_validator/build.rs new file mode 100644 index 0000000..5257dd2 --- /dev/null +++ b/lib/xml_schema_validator/build.rs @@ -0,0 +1,15 @@ +pub fn main() { + println!("cargo:rerun-if-changed=../../action_schema.xsd"); + let output = std::process::Command::new("xml_schema_generator.exe") + .arg("--derive") + .arg("Clone, Debug, Deserialize, Serialize") + .arg("../../action_schema.xsd") + .arg("src/schema.rs") + .output() + .expect("Failed to execute xml_schema_generator"); + + if !output.status.success() { + panic!("xml_schema_generator failed: {}", + String::from_utf8_lossy(&output.stderr)); + } +} \ No newline at end of file diff --git a/lib/xml_schema_validator/cargo.toml b/lib/xml_schema_validator/cargo.toml new file mode 100644 index 0000000..542f77d --- /dev/null +++ b/lib/xml_schema_validator/cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "xml_schema_validator" +version = "0.1.0" +edition = "2021" +authors = ["SIA Team"] +description = "XML Schema validation library optimized for llm token validation" + +[lib] +name = "xml_schema_validator" +# Required for PyO3 +crate-type = ["cdylib", "rlib"] + +[dependencies] +# XML parsing utilities +quick-xml = { version = "0.28.1", features = ["serde", "serialize"] } +serde = { version = "1", features = ["derive"] } +thiserror = "1.0.38" # Better error handling + +# Python bindings +pyo3 = { version = "0.18.0", features = ["extension-module"] } + +[build-dependencies] +xml_schema_generator = "0.6.18" \ No newline at end of file diff --git a/lib/xml_schema_validator/pyproject.toml b/lib/xml_schema_validator/pyproject.toml new file mode 100644 index 0000000..3463f5a --- /dev/null +++ b/lib/xml_schema_validator/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[project] +name = "xml_schema_validator" +version = "0.1.0" +description = "XML Schema validation library for SIA" +readme = "README.md" +requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Text Processing :: Markup :: XML", +] +dependencies = [] + +[project.urls] +repository = "https://github.com/sia/xml_schema_validator" + +[tool.maturin] +python-source = "python" +features = ["pyo3/extension-module"] \ No newline at end of file diff --git a/lib/xml_schema_validator/src/error.rs b/lib/xml_schema_validator/src/error.rs new file mode 100644 index 0000000..4a899a6 --- /dev/null +++ b/lib/xml_schema_validator/src/error.rs @@ -0,0 +1,14 @@ +/// Error types that can occur during XML validation + +#[derive(Clone, Debug, thiserror::Error)] +pub enum Error { + /// Indicates that the provided XSD schema is invalid + #[error("Invalid schema: {0}")] + InvalidSchema(#[from] quick_xml::DeError), + /// Indicates that the provided XML fragment is invalid + #[error("Invalid XML: {0}")] + InvalidXml(String), + /// Not implemented yet + #[error("Not implemented yet")] + NotImplemented, +} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/lib.rs b/lib/xml_schema_validator/src/lib.rs new file mode 100644 index 0000000..a9415e0 --- /dev/null +++ b/lib/xml_schema_validator/src/lib.rs @@ -0,0 +1,104 @@ +#![warn(missing_docs)] +#![warn(rust_2018_idioms)] + +//! A streaming XML validator that checks XML fragments against an XSD schema + +mod error; +mod schema; +mod token; + +use error::Error; +use token::*; + +/// An XML validator that checks XML fragments against an XSD schema +#[derive(Clone, Debug)] +pub struct XmlValidator { + current_tokens: Vec>, +} + +/// The Result type used throughout this crate +/// +/// This is a type alias for `std::result::Result` with our custom `Error` type +pub type Result = std::result::Result; + +impl XmlValidator { + /// Create a new validator with the given schema text + pub fn new(schema_text: &str) -> Result { + let schema: schema::XsSchema = quick_xml::de::from_str(schema_text)?; + let current_tokens = schema.xs_element.iter().map(|element| { + std::rc::Rc::new(Token::StartOfFile(StartOfFile::new(std::rc::Rc::new(element.clone())))) + }).collect(); + + //let current_tokens = vec![std::rc::Rc::new(Token::StartOfFile(StartOfFile))]; + Ok(Self { + current_tokens, + }) + } + + /// Append a fragment of XML to the validator + pub fn append(&mut self, fragment: &str) -> Result<()> { + for c in fragment.chars() { + self.append_char(c)?; + } + Ok(()) + } + + /// Append a single character to the validator + fn append_char(&mut self, c: char) -> Result<()> { + println!("Appending char: {}", c); + let mut new_tokens = vec![]; + for token in self.current_tokens.clone() { + new_tokens.append(&mut token.append(c)); + } + if new_tokens.is_empty() { + return Err(Error::InvalidXml("No continuations".to_string())); + } + self.current_tokens = new_tokens; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_element_open() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let input = ""; + let mut validator = XmlValidator::new(&schema_text).unwrap(); + validator.append(input).unwrap(); + } + + #[test] + fn test_element_open_whitespace() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let input = ""; + let mut validator = XmlValidator::new(&schema_text).unwrap(); + validator.append(input).unwrap(); + } + + #[test] + fn test_element_open_invalid_name() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let input = ", + #[serde(rename = "element")] + pub xs_element: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct XsElement { + #[serde(rename = "@name")] + pub name: String, + #[serde(rename = "$text")] + pub text: Option, + #[serde(rename = "complexType")] + pub xs_complex_type: XsComplexType, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct XsComplexType { + #[serde(rename = "@mixed")] + pub mixed: Option, + #[serde(rename = "$text")] + pub text: Option, + #[serde(rename = "attribute")] + pub xs_attribute: Option>, + #[serde(rename = "sequence")] + pub xs_sequence: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct XsAttribute { + #[serde(rename = "@name")] + pub name: String, + #[serde(rename = "@type")] + pub xs_attribute_type: String, + #[serde(rename = "@use")] + pub xs_attribute_use: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct XsSequence { + #[serde(rename = "$text")] + pub text: Option, + #[serde(rename = "any")] + pub xs_any: XsAny, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct XsAny { + #[serde(rename = "@minOccurs")] + pub min_occurs: String, + #[serde(rename = "@maxOccurs")] + pub max_occurs: String, + #[serde(rename = "@processContents")] + pub process_contents: String, +} + diff --git a/lib/xml_schema_validator/src/token/element_close_end.rs b/lib/xml_schema_validator/src/token/element_close_end.rs new file mode 100644 index 0000000..1ea9bc1 --- /dev/null +++ b/lib/xml_schema_validator/src/token/element_close_end.rs @@ -0,0 +1,21 @@ +use crate::*; + +/// Represents +#[derive(Clone, Debug)] +pub struct ElementCloseEnd { +} + +impl ElementCloseEnd { + pub fn new() -> Self { + Self { + } + } + + pub fn append(&self, c: char) -> Vec> { + if '>' == c { + vec![std::rc::Rc::new(Token::EndOfFile(EndOfFile::new()))] + } else { + vec![] + } + } +} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/element_close_name.rs b/lib/xml_schema_validator/src/token/element_close_name.rs new file mode 100644 index 0000000..5bf8b1f --- /dev/null +++ b/lib/xml_schema_validator/src/token/element_close_name.rs @@ -0,0 +1,51 @@ +use crate::*; + +/// Represents a closing element tag that is in the process of being parsed +/// after the ', + /// Current buffer of characters processed for the element name + buffer: String, +} + +impl ElementCloseName { + /// Create a new instance with the given buffer + pub fn new(element: std::rc::Rc, buffer: String) -> Self { + Self { + element, + buffer, + } + } + + pub fn append(&self, c: char) -> Vec> { + let new_buffer = self.buffer.clone() + &c.to_string(); + + // If still building the element name + if self.element.name.starts_with(&new_buffer) { + // If we've matched the full name + if self.element.name == new_buffer { + // Now we're expecting '>' to close the tag + if c.is_whitespace() { + // Handle whitespace after the element name + vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new( + vec![std::rc::Rc::new(Token::ElementCloseEnd(ElementCloseEnd::new()))] + )))] + } else { + // Directly transition to the closed state + vec![std::rc::Rc::new(Token::ElementCloseEnd(ElementCloseEnd::new()))] + } + } else { + // Still building the element name + vec![std::rc::Rc::new(Token::ElementCloseName(ElementCloseName::new( + self.element.clone(), + new_buffer, + )))] + } + } else { + // The character doesn't match what we expect for this element name + vec![] + } + } +} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/element_close_start.rs b/lib/xml_schema_validator/src/token/element_close_start.rs new file mode 100644 index 0000000..672487c --- /dev/null +++ b/lib/xml_schema_validator/src/token/element_close_start.rs @@ -0,0 +1,36 @@ +use crate::*; + +/// Represents the start of an XML element closing tag (the ', + has_slash: bool, +} + +impl ElementCloseStart { + pub fn new(element: std::rc::Rc) -> Self { + Self { + element, + has_slash: false, + } + } + + pub fn append(&self, c: char) -> Vec> { + if !self.has_slash && '/' == c { + vec![std::rc::Rc::new(Token::ElementCloseStart(Self { + element: self.element.clone(), + has_slash: true, + }))] + } else { + if self.element.name.starts_with(c) { + vec![std::rc::Rc::new(Token::ElementCloseName(ElementCloseName::new( + self.element.clone(), + c.to_string(), + )))] + } else { + vec![] + } + } + } +} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/element_open_end.rs b/lib/xml_schema_validator/src/token/element_open_end.rs new file mode 100644 index 0000000..b7f2bb5 --- /dev/null +++ b/lib/xml_schema_validator/src/token/element_open_end.rs @@ -0,0 +1,35 @@ +use crate::*; + +/// Represents a fully opened XML element, containing the schema element reference +#[derive(Clone, Debug)] +pub struct ElementOpenEnd { + element: std::rc::Rc, +} + +impl ElementOpenEnd { + pub fn new(element: std::rc::Rc) -> Self { + Self { + element, + } + } + + pub fn append(&self, c: char) -> Vec> { + // Check if the element allows text content (mixed="true") + let allows_text = self.element.xs_complex_type.mixed + .as_ref() + .map(|mixed| mixed == "true") + .unwrap_or(false); + + if allows_text { + if c.is_whitespace() { + vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new( + vec![std::rc::Rc::new(Token::TextContent(TextContent::new(self.element.clone())))] + )))] + } else { + vec![std::rc::Rc::new(Token::TextContent(TextContent::new(self.element.clone())))] + } + } else { + vec![] + } + } +} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/element_open_name.rs b/lib/xml_schema_validator/src/token/element_open_name.rs new file mode 100644 index 0000000..65114fa --- /dev/null +++ b/lib/xml_schema_validator/src/token/element_open_name.rs @@ -0,0 +1,39 @@ +use crate::*; + +/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference +#[derive(Clone, Debug)] +pub struct ElementOpenName { + buffer: String, + element: std::rc::Rc, +} + +impl ElementOpenName { + pub fn new(element: std::rc::Rc, buffer: String) -> Result { + if element.name.starts_with(buffer.as_str()) { + Ok(Self { + buffer, + element, + }) + } else { + Err(Error::NotImplemented) + } + } + + pub fn append(&self, c: char) -> Vec> { + let buffer = self.buffer.clone() + &c.to_string(); + if self.element.name.starts_with(buffer.as_str()) { + vec![std::rc::Rc::new(Token::ElementName(ElementOpenName { + buffer, + element: self.element.clone(), + }))] + } else if c.is_whitespace() { + vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new( + vec![std::rc::Rc::new(Token::ElementOpen(ElementOpenEnd::new(self.element.clone())))] + )))] + } else if '>' == c { + vec![std::rc::Rc::new(Token::ElementOpen(ElementOpenEnd::new(self.element.clone())))] + } else { + vec![] + } + } +} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/element_open_start.rs b/lib/xml_schema_validator/src/token/element_open_start.rs new file mode 100644 index 0000000..558821b --- /dev/null +++ b/lib/xml_schema_validator/src/token/element_open_start.rs @@ -0,0 +1,24 @@ +use crate::*; + +/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference +#[derive(Clone, Debug)] +pub struct ElementOpenStart { + element: std::rc::Rc, +} + +impl ElementOpenStart { + pub fn new(element: std::rc::Rc) -> Self { + Self { + element, + } + } + + pub fn append(&self, c: char) -> Vec> { + let element_name = ElementOpenName::new(self.element.clone(), c.to_string()); + if let Ok(element_name) = element_name { + vec![std::rc::Rc::from(Token::ElementName(element_name))] + } else { + vec![] + } + } +} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/end_of_file.rs b/lib/xml_schema_validator/src/token/end_of_file.rs new file mode 100644 index 0000000..526177d --- /dev/null +++ b/lib/xml_schema_validator/src/token/end_of_file.rs @@ -0,0 +1,16 @@ +use crate::*; + +/// Represents the final state at the end of an XML file, containing a reference to the root element schema +#[derive(Clone, Debug)] +pub struct EndOfFile { +} + +impl EndOfFile { + pub fn new() -> Self { + Self {} + } + + pub fn append(&self, _c: char) -> Vec> { + vec![] + } +} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/mod.rs b/lib/xml_schema_validator/src/token/mod.rs new file mode 100644 index 0000000..8ffa5f6 --- /dev/null +++ b/lib/xml_schema_validator/src/token/mod.rs @@ -0,0 +1,53 @@ +mod element_close_end; +mod element_close_name; +mod element_close_start; +mod element_open_name; +mod element_open_end; +mod element_open_start; +mod end_of_file; +mod start_of_file; +mod text_content; +mod whitespace; + +pub use element_close_end::ElementCloseEnd; +pub use element_close_name::ElementCloseName; +pub use element_close_start::ElementCloseStart; +pub use element_open_name::ElementOpenName; +pub use element_open_end::ElementOpenEnd; +pub use element_open_start::ElementOpenStart; +pub use end_of_file::EndOfFile; +pub use start_of_file::StartOfFile; +pub use text_content::TextContent; +pub use whitespace::Whitespace; + +/// Represents the different states of XML token parsing +#[derive(Clone, Debug)] +pub enum Token { + ElementCloseEnd(ElementCloseEnd), + ElementCloseName(ElementCloseName), + ElementCloseStart(ElementCloseStart), + ElementName(ElementOpenName), + ElementOpen(ElementOpenEnd), + ElementStart(ElementOpenStart), + EndOfFile(EndOfFile), + StartOfFile(StartOfFile), + TextContent(TextContent), + Whitespace(Whitespace), +} + +impl Token { + pub fn append(&self, c: char) -> Vec> { + match self { + Token::ElementCloseEnd(element_close_end) => element_close_end.append(c), + Token::ElementCloseName(element_close_name) => element_close_name.append(c), + Token::ElementCloseStart(element_close_start) => element_close_start.append(c), + Token::ElementName(element_name) => element_name.append(c), + Token::ElementOpen(element_open) => element_open.append(c), + Token::ElementStart(element_start) => element_start.append(c), + Token::EndOfFile(end_of_file) => end_of_file.append(c), + Token::StartOfFile(start_of_file) => start_of_file.append(c), + Token::TextContent(text_content) => text_content.append(c), + Token::Whitespace(whitespace) => whitespace.append(c), + } + } +} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/start_of_file.rs b/lib/xml_schema_validator/src/token/start_of_file.rs new file mode 100644 index 0000000..b43b2bb --- /dev/null +++ b/lib/xml_schema_validator/src/token/start_of_file.rs @@ -0,0 +1,21 @@ +use crate::*; + +/// Represents the initial state at the start of an XML file, containing a reference to the root element schema +#[derive(Clone, Debug)] +pub struct StartOfFile { + element: std::rc::Rc, +} + +impl StartOfFile { + pub fn new(element: std::rc::Rc) -> Self { + Self { element } + } + + pub fn append(&self, c: char) -> Vec> { + if '<' == c { + vec![std::rc::Rc::from(Token::ElementStart(ElementOpenStart::new(self.element.clone())))] + } else { + vec![] + } + } +} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/text_content.rs b/lib/xml_schema_validator/src/token/text_content.rs new file mode 100644 index 0000000..646b04c --- /dev/null +++ b/lib/xml_schema_validator/src/token/text_content.rs @@ -0,0 +1,36 @@ +use crate::*; + +/// Represents the text content within an XML element +#[derive(Clone, Debug)] +pub struct TextContent { + element: std::rc::Rc, + buffer: String, +} + +impl TextContent { + pub fn new(element: std::rc::Rc) -> Self { + Self { + element, + buffer: String::new(), + } + } + + /// Create a new TextContent with the given initial buffer + pub fn with_buffer(element: std::rc::Rc, buffer: String) -> Self { + Self { + element, + buffer, + } + } + + pub fn append(&self, c: char) -> Vec> { + if c == '<' { + vec![std::rc::Rc::new(Token::ElementCloseStart(ElementCloseStart::new(self.element.clone())))] + } else { + vec![std::rc::Rc::new(Token::TextContent(TextContent::with_buffer( + self.element.clone(), + self.buffer.clone() + &c.to_string(), + )))] + } + } +} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/whitespace.rs b/lib/xml_schema_validator/src/token/whitespace.rs new file mode 100644 index 0000000..a9dbfc2 --- /dev/null +++ b/lib/xml_schema_validator/src/token/whitespace.rs @@ -0,0 +1,29 @@ +use crate::*; + +/// Represents one or more whitespace characters +#[derive(Clone, Debug)] +pub struct Whitespace { + next: Vec> +} + +impl Whitespace { + pub fn new(next: Vec>) -> Self { + Self { + next + } + } + + pub fn append(&self, c: char) -> Vec> { + if c.is_whitespace() { + vec![std::rc::Rc::new(Token::Whitespace(Self { + next: self.next.clone() + }))] + } else { + let mut tokens = vec![]; + for token in self.next.iter() { + tokens.append(&mut token.append(c)) + } + tokens + } + } +} \ No newline at end of file diff --git a/scripts/collect.sh b/scripts/collect.sh index 71e77ea..af5b90e 100755 --- a/scripts/collect.sh +++ b/scripts/collect.sh @@ -4,11 +4,13 @@ set -euo pipefail declare -A FILTER_SETS=( ["py"]="-f .*(\\.py|requirements.txt)$" + ["rust"]="-f .*\\.(rs|toml)$" ["web"]="-f .*\\.(js|jsx|json|css|html)$" ["doc"]="-f .*\\.md$" - ["deploy"]="-f .*(Dockerfile|\\.sh|\\.xsd|\\.yaml)$" + ["deploy"]="-f .*(Dockerfile|\\.sh|\\.xsd|\\.yaml|\\akefile)$" ["core"]="-s py ./sia ./tools -s deploy . -f ^(?!procedures/).*\\.md$ ." + ["lib"]="-s rust . -s py . -s doc . -s deploy ." ["webui"]="-s web ./web" ["tests"]="-s py ./test" ["procedures"]="-s doc ./procedures" diff --git a/scripts/deploy.sh b/scripts/deploy.sh index ad3a4da..8166460 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -10,7 +10,7 @@ export MSYS_NO_PATHCONV=1 # These can be overridden by environment variables or .env file # Pod configuration -GPU_TYPE=${GPU_TYPE:-"NVIDIA RTX 6000 Ada Generation"} +GPU_TYPE=${GPU_TYPE:-"NVIDIA RTX A6000"} GPU_COUNT=${GPU_COUNT:-1} CONTAINER_DISK_SIZE=${CONTAINER_DISK_SIZE:-200} # GB CPU_COUNT=${CPU_COUNT:-1} # vCPUs