Fix dockerfile for xml schema validator

Dockerfile

@@ -35,7 +35,8 @@ COPY ./scripts/setup_binaries.py /root/sia/scripts/
 COPY ./tools/itb/setup.py /root/sia/tools/itb/setup.py
 RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/tools/itb/setup.py
 RUN python3 -m venv /root/venvs/itb
-RUN /root/venvs/itb/bin/pip install -e /root/sia/tools/itb/
+RUN --mount=type=cache,target=/root/.cache/pip \
+    /root/venvs/itb/bin/pip install -e /root/sia/tools/itb/
 
 # Train tool setup
 FROM base AS train-env
@@ -43,15 +44,23 @@ COPY ./scripts/setup_binaries.py /root/sia/scripts/
 COPY ./tools/train/setup.py /root/sia/tools/train/setup.py
 RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/tools/train/setup.py
 RUN python3 -m venv /root/venvs/train
-RUN /root/venvs/train/bin/pip install -e /root/sia/tools/train/
+RUN --mount=type=cache,target=/root/.cache/pip \
+    /root/venvs/train/bin/pip install -e /root/sia/tools/train/
 
 # SIA core setup
 FROM base AS sia-env
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+COPY action_schema.xsd /root/sia/action_schema.xsd
+
 COPY ./scripts/setup_binaries.py /root/sia/scripts/
 COPY ./setup.py /root/sia/setup.py
+COPY ./lib /root/sia/lib
 RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/setup.py
 RUN python3 -m venv /root/venvs/sia
-RUN /root/venvs/sia/bin/pip install -e /root/sia/
+RUN --mount=type=cache,target=/root/.cache/pip \
+    /root/venvs/sia/bin/pip install -e /root/sia/
 
 # Web frontend build
 FROM node:20-alpine AS web-build

lib/xml_schema_validator/.dockerignore

@@ -0,0 +1,2 @@
+target/
+Cargo.lock

lib/xml_schema_validator/README.md

@@ -0,0 +1,3 @@
+# XML Schema Validator
+
+XML Schema validation library optimized for LLM token validation.

lib/xml_schema_validator/cargo.toml

@@ -20,4 +20,4 @@ thiserror = "1.0.38"    # Better error handling
 pyo3 = { version = "0.18.0", features = ["extension-module"] }
 
 [build-dependencies]
-xml_schema_generator = "0.6.18"
+xml_schema_generator = { version = "0.6.18", features = ["env_logger"] }

lib/xml_schema_validator/pyproject.toml

@@ -1,25 +1,23 @@
-[build-system]
+[build-system]
-requires = ["maturin>=1.0,<2.0"]
+requires = ["maturin>=1.0,<2.0"]
-build-backend = "maturin"
+build-backend = "maturin"
-
+
-[project]
+[project]
-name = "xml_schema_validator"
+name = "xml_schema_validator"
-version = "0.1.0"
+version = "0.1.0"
-description = "XML Schema validation library for SIA"
+description = "XML Schema validation library for SIA"
-readme = "README.md"
+requires-python = ">=3.10"
-requires-python = ">=3.10"
+classifiers = [
-classifiers = [
+    "Programming Language :: Rust",
-    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
-    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.11",
+    "Topic :: Text Processing :: Markup :: XML",
-    "Topic :: Text Processing :: Markup :: XML",
+]
-]
+dependencies = []
-dependencies = []
+
-
+[project.urls]
-[project.urls]
+repository = "https://github.com/sia/xml_schema_validator"
-repository = "https://github.com/sia/xml_schema_validator"
+
-
+[tool.maturin]
-[tool.maturin]
+features = ["pyo3/extension-module"]
-python-source = "python"
-features = ["pyo3/extension-module"]

lib/xml_schema_validator/src/lib.rs

@@ -4,6 +4,7 @@
 //! A streaming XML validator that checks XML fragments against an XSD schema
 
 mod error;
+mod python;
 mod schema;
 mod token;
 
@@ -13,7 +14,7 @@ use token::*;
 /// An XML validator that checks XML fragments against an XSD schema
 #[derive(Clone, Debug)]
 pub struct XmlValidator {
-    current_tokens: Vec<std::rc::Rc<Token>>,
+    current_tokens: Vec<Box<Token>>,
 }
 
 /// The Result type used throughout this crate
@@ -26,10 +27,9 @@ impl XmlValidator {
     pub fn new(schema_text: &str) -> Result<Self> {
         let schema: schema::XsSchema = quick_xml::de::from_str(schema_text)?;
         let current_tokens = schema.xs_element.iter().map(|element| {
-            std::rc::Rc::new(Token::StartOfFile(StartOfFile::new(std::rc::Rc::new(element.clone()))))
+            Box::new(Token::StartOfFile(StartOfFile::new(Box::new(element.clone()))))
         }).collect();
         
-        //let current_tokens = vec![std::rc::Rc::new(Token::StartOfFile(StartOfFile))];
         Ok(Self {
             current_tokens,
         })

lib/xml_schema_validator/src/python.rs

@@ -0,0 +1,32 @@
+use crate::XmlValidator;
+use pyo3::prelude::*;
+use pyo3::exceptions::PyValueError;
+
+/// Python wrapper for XmlValidator
+#[pyclass(unsendable)]
+struct PyXmlValidator {
+    validator: XmlValidator,
+}
+
+#[pymethods]
+impl PyXmlValidator {
+    #[new]
+    fn new(schema_text: &str) -> PyResult<Self> {
+        let validator = XmlValidator::new(schema_text)
+            .map_err(|e| PyValueError::new_err(format!("Failed to create validator: {}", e)))?;
+        
+        Ok(Self { validator })
+    }
+
+    fn append(&mut self, fragment: &str) -> PyResult<()> {
+        self.validator.append(fragment)
+            .map_err(|e| PyValueError::new_err(format!("Validation error: {}", e)))
+    }
+}
+
+/// Python module for XML Schema validation
+#[pymodule]
+fn xml_schema_validator(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
+    m.add_class::<PyXmlValidator>()?;
+    Ok(())
+}

lib/xml_schema_validator/src/token/element_close_end.rs

@@ -11,9 +11,9 @@ impl ElementCloseEnd {
         }
     }
 
-    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+    pub fn append(&self, c: char) -> Vec<Box<Token>> {
         if '>' == c {
-            vec![std::rc::Rc::new(Token::EndOfFile(EndOfFile::new()))]
+            vec![Box::new(Token::EndOfFile(EndOfFile::new()))]
         } else {
             vec![]
         }

lib/xml_schema_validator/src/token/element_close_name.rs

@@ -5,21 +5,21 @@ use crate::*;
 #[derive(Clone, Debug)]
 pub struct ElementCloseName {
     /// The element that was previously opened and is now being closed
-    element: std::rc::Rc<schema::XsElement>,
+    element: Box<schema::XsElement>,
     /// Current buffer of characters processed for the element name
     buffer: String,
 }
 
 impl ElementCloseName {
     /// Create a new instance with the given buffer
-    pub fn new(element: std::rc::Rc<schema::XsElement>, buffer: String) -> Self {
+    pub fn new(element: Box<schema::XsElement>, buffer: String) -> Self {
         Self {
             element,
             buffer,
         }
     }
 
-    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+    pub fn append(&self, c: char) -> Vec<Box<Token>> {
         let new_buffer = self.buffer.clone() + &c.to_string();
         
         // If still building the element name
@@ -29,17 +29,17 @@ impl ElementCloseName {
                 // Now we're expecting '>' to close the tag
                 if c.is_whitespace() {
                     // Handle whitespace after the element name
-                    vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new(
+                    vec![Box::new(Token::Whitespace(Whitespace::new(
-                        vec![std::rc::Rc::new(Token::ElementCloseEnd(ElementCloseEnd::new()))]
+                        vec![Box::new(Token::ElementCloseEnd(ElementCloseEnd::new()))]
                     )))]
                 } else {
                     // Directly transition to the closed state
-                    vec![std::rc::Rc::new(Token::ElementCloseEnd(ElementCloseEnd::new()))]
+                    vec![Box::new(Token::ElementCloseEnd(ElementCloseEnd::new()))]
                 }
             } else {
                 // Still building the element name
-                vec![std::rc::Rc::new(Token::ElementCloseName(ElementCloseName::new(
+                vec![Box::new(Token::ElementCloseName(ElementCloseName::new(
-                    self.element.clone(),
+                    Box::new((*self.element).clone()),
                     new_buffer,
                 )))]
             }

lib/xml_schema_validator/src/token/element_close_start.rs

@@ -4,28 +4,28 @@ use crate::*;
 #[derive(Clone, Debug)]
 pub struct ElementCloseStart {
     /// The element that was previously opened and is now being closed
-    element: std::rc::Rc<schema::XsElement>,
+    element: Box<schema::XsElement>,
     has_slash: bool,
 }
 
 impl ElementCloseStart {
-    pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
+    pub fn new(element: Box<schema::XsElement>) -> Self {
         Self {
             element,
             has_slash: false,
         }
     }
 
-    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+    pub fn append(&self, c: char) -> Vec<Box<Token>> {
         if !self.has_slash && '/' == c {
-            vec![std::rc::Rc::new(Token::ElementCloseStart(Self {
+            vec![Box::new(Token::ElementCloseStart(Self {
-                element: self.element.clone(),
+                element: Box::new((*self.element).clone()),
                 has_slash: true,
             }))]
         } else {
             if self.element.name.starts_with(c) {
-                vec![std::rc::Rc::new(Token::ElementCloseName(ElementCloseName::new(
+                vec![Box::new(Token::ElementCloseName(ElementCloseName::new(
-                    self.element.clone(),
+                    Box::new((*self.element).clone()),
                     c.to_string(),
                 )))]
             } else {

lib/xml_schema_validator/src/token/element_open_end.rs

@@ -3,17 +3,17 @@ use crate::*;
 /// Represents a fully opened XML element, containing the schema element reference
 #[derive(Clone, Debug)]
 pub struct ElementOpenEnd {
-    element: std::rc::Rc<schema::XsElement>,
+    element: Box<schema::XsElement>,
 }
 
 impl ElementOpenEnd {
-    pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
+    pub fn new(element: Box<schema::XsElement>) -> Self {
         Self {
             element,
         }
     }
 
-    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+    pub fn append(&self, c: char) -> Vec<Box<Token>> {
         // Check if the element allows text content (mixed="true")
         let allows_text = self.element.xs_complex_type.mixed
             .as_ref()
@@ -22,11 +22,11 @@ impl ElementOpenEnd {
 
         if allows_text {
             if c.is_whitespace() {
-                vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new(
+                vec![Box::new(Token::Whitespace(Whitespace::new(
-                    vec![std::rc::Rc::new(Token::TextContent(TextContent::new(self.element.clone())))]
+                    vec![Box::new(Token::TextContent(TextContent::new(Box::new((*self.element).clone()))))]
                 )))]
             } else {
-                vec![std::rc::Rc::new(Token::TextContent(TextContent::new(self.element.clone())))]
+                vec![Box::new(Token::TextContent(TextContent::new(Box::new((*self.element).clone()))))]
             }
         } else {
             vec![]

lib/xml_schema_validator/src/token/element_open_name.rs

@@ -4,11 +4,11 @@ use crate::*;
 #[derive(Clone, Debug)]
 pub struct ElementOpenName {
     buffer: String,
-    element: std::rc::Rc<schema::XsElement>,
+    element: Box<schema::XsElement>,
 }
 
 impl ElementOpenName {
-    pub fn new(element: std::rc::Rc<schema::XsElement>, buffer: String) -> Result<Self> {
+    pub fn new(element: Box<schema::XsElement>, buffer: String) -> Result<Self> {
         if element.name.starts_with(buffer.as_str()) {
             Ok(Self {
                 buffer,
@@ -19,19 +19,19 @@ impl ElementOpenName {
         }
     }
 
-    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+    pub fn append(&self, c: char) -> Vec<Box<Token>> {
         let buffer = self.buffer.clone() + &c.to_string();
         if self.element.name.starts_with(buffer.as_str()) {
-            vec![std::rc::Rc::new(Token::ElementName(ElementOpenName {
+            vec![Box::new(Token::ElementName(ElementOpenName {
                 buffer,
-                element: self.element.clone(),
+                element: Box::new((*self.element).clone()),
             }))]
         } else if c.is_whitespace() {
-            vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new(
+            vec![Box::new(Token::Whitespace(Whitespace::new(
-                vec![std::rc::Rc::new(Token::ElementOpen(ElementOpenEnd::new(self.element.clone())))]
+                vec![Box::new(Token::ElementOpen(ElementOpenEnd::new(Box::new((*self.element).clone()))))]
             )))]
         } else if '>' == c {
-            vec![std::rc::Rc::new(Token::ElementOpen(ElementOpenEnd::new(self.element.clone())))]
+            vec![Box::new(Token::ElementOpen(ElementOpenEnd::new(Box::new((*self.element).clone()))))]
         } else {
             vec![]
         }

lib/xml_schema_validator/src/token/element_open_start.rs

@@ -3,20 +3,20 @@ use crate::*;
 /// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
 #[derive(Clone, Debug)]
 pub struct ElementOpenStart {
-    element: std::rc::Rc<schema::XsElement>,
+    element: Box<schema::XsElement>,
 }
 
 impl ElementOpenStart {
-    pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
+    pub fn new(element: Box<schema::XsElement>) -> Self {
         Self {
             element,
         }
     }
 
-    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+    pub fn append(&self, c: char) -> Vec<Box<Token>> {
-        let element_name = ElementOpenName::new(self.element.clone(), c.to_string());
+        let element_name = ElementOpenName::new(Box::new((*self.element).clone()), c.to_string());
         if let Ok(element_name) = element_name {
-            vec![std::rc::Rc::from(Token::ElementName(element_name))]
+            vec![Box::new(Token::ElementName(element_name))]
         } else {
             vec![]
         }

lib/xml_schema_validator/src/token/end_of_file.rs

@@ -10,7 +10,7 @@ impl EndOfFile {
         Self {}
     }
 
-    pub fn append(&self, _c: char) -> Vec<std::rc::Rc<Token>> {
+    pub fn append(&self, _c: char) -> Vec<Box<Token>> {
         vec![]
     }
 }

lib/xml_schema_validator/src/token/mod.rs

@@ -36,7 +36,7 @@ pub enum Token {
 }
 
 impl Token {
-    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+    pub fn append(&self, c: char) -> Vec<Box<Token>> {
         match self {
             Token::ElementCloseEnd(element_close_end) => element_close_end.append(c),
             Token::ElementCloseName(element_close_name) => element_close_name.append(c),

lib/xml_schema_validator/src/token/start_of_file.rs

@@ -3,17 +3,17 @@ use crate::*;
 /// Represents the initial state at the start of an XML file, containing a reference to the root element schema
 #[derive(Clone, Debug)]
 pub struct StartOfFile {
-    element: std::rc::Rc<schema::XsElement>,
+    element: Box<schema::XsElement>,
 }
 
 impl StartOfFile {
-    pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
+    pub fn new(element: Box<schema::XsElement>) -> Self {
         Self { element }
     }
 
-    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+    pub fn append(&self, c: char) -> Vec<Box<Token>> {
         if '<' == c {
-            vec![std::rc::Rc::from(Token::ElementStart(ElementOpenStart::new(self.element.clone())))]
+            vec![Box::new(Token::ElementStart(ElementOpenStart::new(Box::new((*self.element).clone()))))]
         } else {
             vec![]
         }

lib/xml_schema_validator/src/token/text_content.rs

@@ -3,12 +3,12 @@ use crate::*;
 /// Represents the text content within an XML element
 #[derive(Clone, Debug)]
 pub struct TextContent {
-    element: std::rc::Rc<schema::XsElement>,
+    element: Box<schema::XsElement>,
     buffer: String,
 }
 
 impl TextContent {
-    pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
+    pub fn new(element: Box<schema::XsElement>) -> Self {
         Self {
             element,
             buffer: String::new(),
@@ -16,19 +16,19 @@ impl TextContent {
     }
 
     /// Create a new TextContent with the given initial buffer
-    pub fn with_buffer(element: std::rc::Rc<schema::XsElement>, buffer: String) -> Self {
+    pub fn with_buffer(element: Box<schema::XsElement>, buffer: String) -> Self {
         Self {
             element,
             buffer,
         }
     }
 
-    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+    pub fn append(&self, c: char) -> Vec<Box<Token>> {
         if c == '<' {
-            vec![std::rc::Rc::new(Token::ElementCloseStart(ElementCloseStart::new(self.element.clone())))]
+            vec![Box::new(Token::ElementCloseStart(ElementCloseStart::new(Box::new((*self.element).clone()))))]
         } else {
-            vec![std::rc::Rc::new(Token::TextContent(TextContent::with_buffer(
+            vec![Box::new(Token::TextContent(TextContent::with_buffer(
-                self.element.clone(),
+                Box::new((*self.element).clone()),
                 self.buffer.clone() + &c.to_string(),
             )))]
         }

lib/xml_schema_validator/src/token/whitespace.rs

@@ -3,19 +3,19 @@ use crate::*;
 /// Represents one or more whitespace characters
 #[derive(Clone, Debug)]
 pub struct Whitespace {
-    next: Vec<std::rc::Rc<Token>>
+    next: Vec<Box<Token>>
 }
 
 impl Whitespace {
-    pub fn new(next: Vec<std::rc::Rc<Token>>) -> Self {
+    pub fn new(next: Vec<Box<Token>>) -> Self {
         Self {
             next
         }
     }
 
-    pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
+    pub fn append(&self, c: char) -> Vec<Box<Token>> {
         if c.is_whitespace() {
-            vec![std::rc::Rc::new(Token::Whitespace(Self {
+            vec![Box::new(Token::Whitespace(Self {
                 next: self.next.clone()
             }))]
         } else {

scripts/bootstrap.sh

@@ -19,7 +19,18 @@ fi
 # Install required packages
 apt-get update
 apt-get install -y \
-    vim
+    gnupg \
+    vim \
+    wget \
+    ;
+wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
+echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list
+apt-get update && \
+    apt-get install -y \
+    google-chrome-stable
+
+curl https://sh.rustup.rs -sSf | bash -s -- -y
+export PATH="/root/.cargo/bin:${PATH}"
 
 # Create directory structure
 echo "Creating directory structure..."

scripts/collect.sh

@@ -7,7 +7,7 @@ declare -A FILTER_SETS=(
   ["rust"]="-f .*\\.(rs|toml)$"
   ["web"]="-f .*\\.(js|jsx|json|css|html)$"
   ["doc"]="-f .*\\.md$"
-  ["deploy"]="-f .*(Dockerfile|\\.sh|\\.xsd|\\.yaml|\\akefile)$"
+  ["deploy"]="-f .*(Dockerfile|akefile|\\.toml|\\.sh|\\.xsd|\\.yaml)$"
   
   ["core"]="-s py ./sia ./tools -s deploy . -f ^(?!procedures/).*\\.md$ ."
   ["lib"]="-s rust . -s py . -s doc . -s deploy ."

setup.py

@@ -24,6 +24,7 @@ setup(
         'python-dotenv>=1.0.0',
         'tiktoken>=0.4.0',
         'transformers>=4.30.0',
+        'xml_schema_validator @ file:///root/sia/lib/xml_schema_validator',
     ],
     python_requires='>=3.10',
 )