diff --git a/Dockerfile b/Dockerfile index 4d3b648..7bf0554 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,8 @@ COPY ./scripts/setup_binaries.py /root/sia/scripts/ COPY ./tools/itb/setup.py /root/sia/tools/itb/setup.py RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/tools/itb/setup.py RUN python3 -m venv /root/venvs/itb -RUN /root/venvs/itb/bin/pip install -e /root/sia/tools/itb/ +RUN --mount=type=cache,target=/root/.cache/pip \ + /root/venvs/itb/bin/pip install -e /root/sia/tools/itb/ # Train tool setup FROM base AS train-env @@ -43,15 +44,23 @@ COPY ./scripts/setup_binaries.py /root/sia/scripts/ COPY ./tools/train/setup.py /root/sia/tools/train/setup.py RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/tools/train/setup.py RUN python3 -m venv /root/venvs/train -RUN /root/venvs/train/bin/pip install -e /root/sia/tools/train/ +RUN --mount=type=cache,target=/root/.cache/pip \ + /root/venvs/train/bin/pip install -e /root/sia/tools/train/ # SIA core setup FROM base AS sia-env +RUN curl https://sh.rustup.rs -sSf | bash -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +COPY action_schema.xsd /root/sia/action_schema.xsd + COPY ./scripts/setup_binaries.py /root/sia/scripts/ COPY ./setup.py /root/sia/setup.py +COPY ./lib /root/sia/lib RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/setup.py RUN python3 -m venv /root/venvs/sia -RUN /root/venvs/sia/bin/pip install -e /root/sia/ +RUN --mount=type=cache,target=/root/.cache/pip \ + /root/venvs/sia/bin/pip install -e /root/sia/ # Web frontend build FROM node:20-alpine AS web-build diff --git a/lib/xml_schema_validator/.dockerignore b/lib/xml_schema_validator/.dockerignore new file mode 100644 index 0000000..2630b86 --- /dev/null +++ b/lib/xml_schema_validator/.dockerignore @@ -0,0 +1,2 @@ +target/ +Cargo.lock \ No newline at end of file diff --git a/lib/xml_schema_validator/README.md b/lib/xml_schema_validator/README.md new file mode 100644 index 0000000..7698587 --- /dev/null +++ b/lib/xml_schema_validator/README.md @@ -0,0 +1,3 @@ +# XML Schema Validator + +XML Schema validation library optimized for LLM token validation. diff --git a/lib/xml_schema_validator/cargo.toml b/lib/xml_schema_validator/cargo.toml index 542f77d..3013479 100644 --- a/lib/xml_schema_validator/cargo.toml +++ b/lib/xml_schema_validator/cargo.toml @@ -20,4 +20,4 @@ thiserror = "1.0.38" # Better error handling pyo3 = { version = "0.18.0", features = ["extension-module"] } [build-dependencies] -xml_schema_generator = "0.6.18" \ No newline at end of file +xml_schema_generator = { version = "0.6.18", features = ["env_logger"] } \ No newline at end of file diff --git a/lib/xml_schema_validator/pyproject.toml b/lib/xml_schema_validator/pyproject.toml index 3463f5a..06b71dc 100644 --- a/lib/xml_schema_validator/pyproject.toml +++ b/lib/xml_schema_validator/pyproject.toml @@ -1,25 +1,23 @@ -[build-system] -requires = ["maturin>=1.0,<2.0"] -build-backend = "maturin" - -[project] -name = "xml_schema_validator" -version = "0.1.0" -description = "XML Schema validation library for SIA" -readme = "README.md" -requires-python = ">=3.10" -classifiers = [ - "Programming Language :: Rust", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Topic :: Text Processing :: Markup :: XML", -] -dependencies = [] - -[project.urls] -repository = "https://github.com/sia/xml_schema_validator" - -[tool.maturin] -python-source = "python" -features = ["pyo3/extension-module"] \ No newline at end of file +[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[project] +name = "xml_schema_validator" +version = "0.1.0" +description = "XML Schema validation library for SIA" +requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Text Processing :: Markup :: XML", +] +dependencies = [] + +[project.urls] +repository = "https://github.com/sia/xml_schema_validator" + +[tool.maturin] +features = ["pyo3/extension-module"] diff --git a/lib/xml_schema_validator/src/lib.rs b/lib/xml_schema_validator/src/lib.rs index a9415e0..bac12aa 100644 --- a/lib/xml_schema_validator/src/lib.rs +++ b/lib/xml_schema_validator/src/lib.rs @@ -4,6 +4,7 @@ //! A streaming XML validator that checks XML fragments against an XSD schema mod error; +mod python; mod schema; mod token; @@ -13,7 +14,7 @@ use token::*; /// An XML validator that checks XML fragments against an XSD schema #[derive(Clone, Debug)] pub struct XmlValidator { - current_tokens: Vec>, + current_tokens: Vec>, } /// The Result type used throughout this crate @@ -26,10 +27,9 @@ impl XmlValidator { pub fn new(schema_text: &str) -> Result { let schema: schema::XsSchema = quick_xml::de::from_str(schema_text)?; let current_tokens = schema.xs_element.iter().map(|element| { - std::rc::Rc::new(Token::StartOfFile(StartOfFile::new(std::rc::Rc::new(element.clone())))) + Box::new(Token::StartOfFile(StartOfFile::new(Box::new(element.clone())))) }).collect(); - //let current_tokens = vec![std::rc::Rc::new(Token::StartOfFile(StartOfFile))]; Ok(Self { current_tokens, }) diff --git a/lib/xml_schema_validator/src/python.rs b/lib/xml_schema_validator/src/python.rs new file mode 100644 index 0000000..e948475 --- /dev/null +++ b/lib/xml_schema_validator/src/python.rs @@ -0,0 +1,32 @@ +use crate::XmlValidator; +use pyo3::prelude::*; +use pyo3::exceptions::PyValueError; + +/// Python wrapper for XmlValidator +#[pyclass(unsendable)] +struct PyXmlValidator { + validator: XmlValidator, +} + +#[pymethods] +impl PyXmlValidator { + #[new] + fn new(schema_text: &str) -> PyResult { + let validator = XmlValidator::new(schema_text) + .map_err(|e| PyValueError::new_err(format!("Failed to create validator: {}", e)))?; + + Ok(Self { validator }) + } + + fn append(&mut self, fragment: &str) -> PyResult<()> { + self.validator.append(fragment) + .map_err(|e| PyValueError::new_err(format!("Validation error: {}", e))) + } +} + +/// Python module for XML Schema validation +#[pymodule] +fn xml_schema_validator(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + m.add_class::()?; + Ok(()) +} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/element_close_end.rs b/lib/xml_schema_validator/src/token/element_close_end.rs index 1ea9bc1..ababfe1 100644 --- a/lib/xml_schema_validator/src/token/element_close_end.rs +++ b/lib/xml_schema_validator/src/token/element_close_end.rs @@ -11,9 +11,9 @@ impl ElementCloseEnd { } } - pub fn append(&self, c: char) -> Vec> { + pub fn append(&self, c: char) -> Vec> { if '>' == c { - vec![std::rc::Rc::new(Token::EndOfFile(EndOfFile::new()))] + vec![Box::new(Token::EndOfFile(EndOfFile::new()))] } else { vec![] } diff --git a/lib/xml_schema_validator/src/token/element_close_name.rs b/lib/xml_schema_validator/src/token/element_close_name.rs index 5bf8b1f..57802ba 100644 --- a/lib/xml_schema_validator/src/token/element_close_name.rs +++ b/lib/xml_schema_validator/src/token/element_close_name.rs @@ -5,21 +5,21 @@ use crate::*; #[derive(Clone, Debug)] pub struct ElementCloseName { /// The element that was previously opened and is now being closed - element: std::rc::Rc, + element: Box, /// Current buffer of characters processed for the element name buffer: String, } impl ElementCloseName { /// Create a new instance with the given buffer - pub fn new(element: std::rc::Rc, buffer: String) -> Self { + pub fn new(element: Box, buffer: String) -> Self { Self { element, buffer, } } - pub fn append(&self, c: char) -> Vec> { + pub fn append(&self, c: char) -> Vec> { let new_buffer = self.buffer.clone() + &c.to_string(); // If still building the element name @@ -29,17 +29,17 @@ impl ElementCloseName { // Now we're expecting '>' to close the tag if c.is_whitespace() { // Handle whitespace after the element name - vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new( - vec![std::rc::Rc::new(Token::ElementCloseEnd(ElementCloseEnd::new()))] + vec![Box::new(Token::Whitespace(Whitespace::new( + vec![Box::new(Token::ElementCloseEnd(ElementCloseEnd::new()))] )))] } else { // Directly transition to the closed state - vec![std::rc::Rc::new(Token::ElementCloseEnd(ElementCloseEnd::new()))] + vec![Box::new(Token::ElementCloseEnd(ElementCloseEnd::new()))] } } else { // Still building the element name - vec![std::rc::Rc::new(Token::ElementCloseName(ElementCloseName::new( - self.element.clone(), + vec![Box::new(Token::ElementCloseName(ElementCloseName::new( + Box::new((*self.element).clone()), new_buffer, )))] } diff --git a/lib/xml_schema_validator/src/token/element_close_start.rs b/lib/xml_schema_validator/src/token/element_close_start.rs index 672487c..bb13a5a 100644 --- a/lib/xml_schema_validator/src/token/element_close_start.rs +++ b/lib/xml_schema_validator/src/token/element_close_start.rs @@ -4,28 +4,28 @@ use crate::*; #[derive(Clone, Debug)] pub struct ElementCloseStart { /// The element that was previously opened and is now being closed - element: std::rc::Rc, + element: Box, has_slash: bool, } impl ElementCloseStart { - pub fn new(element: std::rc::Rc) -> Self { + pub fn new(element: Box) -> Self { Self { element, has_slash: false, } } - pub fn append(&self, c: char) -> Vec> { + pub fn append(&self, c: char) -> Vec> { if !self.has_slash && '/' == c { - vec![std::rc::Rc::new(Token::ElementCloseStart(Self { - element: self.element.clone(), + vec![Box::new(Token::ElementCloseStart(Self { + element: Box::new((*self.element).clone()), has_slash: true, }))] } else { if self.element.name.starts_with(c) { - vec![std::rc::Rc::new(Token::ElementCloseName(ElementCloseName::new( - self.element.clone(), + vec![Box::new(Token::ElementCloseName(ElementCloseName::new( + Box::new((*self.element).clone()), c.to_string(), )))] } else { diff --git a/lib/xml_schema_validator/src/token/element_open_end.rs b/lib/xml_schema_validator/src/token/element_open_end.rs index b7f2bb5..680101e 100644 --- a/lib/xml_schema_validator/src/token/element_open_end.rs +++ b/lib/xml_schema_validator/src/token/element_open_end.rs @@ -3,17 +3,17 @@ use crate::*; /// Represents a fully opened XML element, containing the schema element reference #[derive(Clone, Debug)] pub struct ElementOpenEnd { - element: std::rc::Rc, + element: Box, } impl ElementOpenEnd { - pub fn new(element: std::rc::Rc) -> Self { + pub fn new(element: Box) -> Self { Self { element, } } - pub fn append(&self, c: char) -> Vec> { + pub fn append(&self, c: char) -> Vec> { // Check if the element allows text content (mixed="true") let allows_text = self.element.xs_complex_type.mixed .as_ref() @@ -22,11 +22,11 @@ impl ElementOpenEnd { if allows_text { if c.is_whitespace() { - vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new( - vec![std::rc::Rc::new(Token::TextContent(TextContent::new(self.element.clone())))] + vec![Box::new(Token::Whitespace(Whitespace::new( + vec![Box::new(Token::TextContent(TextContent::new(Box::new((*self.element).clone()))))] )))] } else { - vec![std::rc::Rc::new(Token::TextContent(TextContent::new(self.element.clone())))] + vec![Box::new(Token::TextContent(TextContent::new(Box::new((*self.element).clone()))))] } } else { vec![] diff --git a/lib/xml_schema_validator/src/token/element_open_name.rs b/lib/xml_schema_validator/src/token/element_open_name.rs index 65114fa..d2194c5 100644 --- a/lib/xml_schema_validator/src/token/element_open_name.rs +++ b/lib/xml_schema_validator/src/token/element_open_name.rs @@ -4,11 +4,11 @@ use crate::*; #[derive(Clone, Debug)] pub struct ElementOpenName { buffer: String, - element: std::rc::Rc, + element: Box, } impl ElementOpenName { - pub fn new(element: std::rc::Rc, buffer: String) -> Result { + pub fn new(element: Box, buffer: String) -> Result { if element.name.starts_with(buffer.as_str()) { Ok(Self { buffer, @@ -19,19 +19,19 @@ impl ElementOpenName { } } - pub fn append(&self, c: char) -> Vec> { + pub fn append(&self, c: char) -> Vec> { let buffer = self.buffer.clone() + &c.to_string(); if self.element.name.starts_with(buffer.as_str()) { - vec![std::rc::Rc::new(Token::ElementName(ElementOpenName { + vec![Box::new(Token::ElementName(ElementOpenName { buffer, - element: self.element.clone(), + element: Box::new((*self.element).clone()), }))] } else if c.is_whitespace() { - vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new( - vec![std::rc::Rc::new(Token::ElementOpen(ElementOpenEnd::new(self.element.clone())))] + vec![Box::new(Token::Whitespace(Whitespace::new( + vec![Box::new(Token::ElementOpen(ElementOpenEnd::new(Box::new((*self.element).clone()))))] )))] } else if '>' == c { - vec![std::rc::Rc::new(Token::ElementOpen(ElementOpenEnd::new(self.element.clone())))] + vec![Box::new(Token::ElementOpen(ElementOpenEnd::new(Box::new((*self.element).clone()))))] } else { vec![] } diff --git a/lib/xml_schema_validator/src/token/element_open_start.rs b/lib/xml_schema_validator/src/token/element_open_start.rs index 558821b..612bf0d 100644 --- a/lib/xml_schema_validator/src/token/element_open_start.rs +++ b/lib/xml_schema_validator/src/token/element_open_start.rs @@ -3,20 +3,20 @@ use crate::*; /// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference #[derive(Clone, Debug)] pub struct ElementOpenStart { - element: std::rc::Rc, + element: Box, } impl ElementOpenStart { - pub fn new(element: std::rc::Rc) -> Self { + pub fn new(element: Box) -> Self { Self { element, } } - pub fn append(&self, c: char) -> Vec> { - let element_name = ElementOpenName::new(self.element.clone(), c.to_string()); + pub fn append(&self, c: char) -> Vec> { + let element_name = ElementOpenName::new(Box::new((*self.element).clone()), c.to_string()); if let Ok(element_name) = element_name { - vec![std::rc::Rc::from(Token::ElementName(element_name))] + vec![Box::new(Token::ElementName(element_name))] } else { vec![] } diff --git a/lib/xml_schema_validator/src/token/end_of_file.rs b/lib/xml_schema_validator/src/token/end_of_file.rs index 526177d..3f50925 100644 --- a/lib/xml_schema_validator/src/token/end_of_file.rs +++ b/lib/xml_schema_validator/src/token/end_of_file.rs @@ -10,7 +10,7 @@ impl EndOfFile { Self {} } - pub fn append(&self, _c: char) -> Vec> { + pub fn append(&self, _c: char) -> Vec> { vec![] } } \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/mod.rs b/lib/xml_schema_validator/src/token/mod.rs index 8ffa5f6..95be565 100644 --- a/lib/xml_schema_validator/src/token/mod.rs +++ b/lib/xml_schema_validator/src/token/mod.rs @@ -36,7 +36,7 @@ pub enum Token { } impl Token { - pub fn append(&self, c: char) -> Vec> { + pub fn append(&self, c: char) -> Vec> { match self { Token::ElementCloseEnd(element_close_end) => element_close_end.append(c), Token::ElementCloseName(element_close_name) => element_close_name.append(c), diff --git a/lib/xml_schema_validator/src/token/start_of_file.rs b/lib/xml_schema_validator/src/token/start_of_file.rs index b43b2bb..ae7176f 100644 --- a/lib/xml_schema_validator/src/token/start_of_file.rs +++ b/lib/xml_schema_validator/src/token/start_of_file.rs @@ -3,17 +3,17 @@ use crate::*; /// Represents the initial state at the start of an XML file, containing a reference to the root element schema #[derive(Clone, Debug)] pub struct StartOfFile { - element: std::rc::Rc, + element: Box, } impl StartOfFile { - pub fn new(element: std::rc::Rc) -> Self { + pub fn new(element: Box) -> Self { Self { element } } - pub fn append(&self, c: char) -> Vec> { + pub fn append(&self, c: char) -> Vec> { if '<' == c { - vec![std::rc::Rc::from(Token::ElementStart(ElementOpenStart::new(self.element.clone())))] + vec![Box::new(Token::ElementStart(ElementOpenStart::new(Box::new((*self.element).clone()))))] } else { vec![] } diff --git a/lib/xml_schema_validator/src/token/text_content.rs b/lib/xml_schema_validator/src/token/text_content.rs index 646b04c..1ab25dc 100644 --- a/lib/xml_schema_validator/src/token/text_content.rs +++ b/lib/xml_schema_validator/src/token/text_content.rs @@ -3,12 +3,12 @@ use crate::*; /// Represents the text content within an XML element #[derive(Clone, Debug)] pub struct TextContent { - element: std::rc::Rc, + element: Box, buffer: String, } impl TextContent { - pub fn new(element: std::rc::Rc) -> Self { + pub fn new(element: Box) -> Self { Self { element, buffer: String::new(), @@ -16,19 +16,19 @@ impl TextContent { } /// Create a new TextContent with the given initial buffer - pub fn with_buffer(element: std::rc::Rc, buffer: String) -> Self { + pub fn with_buffer(element: Box, buffer: String) -> Self { Self { element, buffer, } } - pub fn append(&self, c: char) -> Vec> { + pub fn append(&self, c: char) -> Vec> { if c == '<' { - vec![std::rc::Rc::new(Token::ElementCloseStart(ElementCloseStart::new(self.element.clone())))] + vec![Box::new(Token::ElementCloseStart(ElementCloseStart::new(Box::new((*self.element).clone()))))] } else { - vec![std::rc::Rc::new(Token::TextContent(TextContent::with_buffer( - self.element.clone(), + vec![Box::new(Token::TextContent(TextContent::with_buffer( + Box::new((*self.element).clone()), self.buffer.clone() + &c.to_string(), )))] } diff --git a/lib/xml_schema_validator/src/token/whitespace.rs b/lib/xml_schema_validator/src/token/whitespace.rs index a9dbfc2..b4964fb 100644 --- a/lib/xml_schema_validator/src/token/whitespace.rs +++ b/lib/xml_schema_validator/src/token/whitespace.rs @@ -3,19 +3,19 @@ use crate::*; /// Represents one or more whitespace characters #[derive(Clone, Debug)] pub struct Whitespace { - next: Vec> + next: Vec> } impl Whitespace { - pub fn new(next: Vec>) -> Self { + pub fn new(next: Vec>) -> Self { Self { next } } - pub fn append(&self, c: char) -> Vec> { + pub fn append(&self, c: char) -> Vec> { if c.is_whitespace() { - vec![std::rc::Rc::new(Token::Whitespace(Self { + vec![Box::new(Token::Whitespace(Self { next: self.next.clone() }))] } else { diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 2568b9f..0957e8d 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -19,7 +19,18 @@ fi # Install required packages apt-get update apt-get install -y \ - vim + gnupg \ + vim \ + wget \ + ; +wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - +echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list +apt-get update && \ + apt-get install -y \ + google-chrome-stable + +curl https://sh.rustup.rs -sSf | bash -s -- -y +export PATH="/root/.cargo/bin:${PATH}" # Create directory structure echo "Creating directory structure..." diff --git a/scripts/collect.sh b/scripts/collect.sh index af5b90e..ce30041 100755 --- a/scripts/collect.sh +++ b/scripts/collect.sh @@ -7,7 +7,7 @@ declare -A FILTER_SETS=( ["rust"]="-f .*\\.(rs|toml)$" ["web"]="-f .*\\.(js|jsx|json|css|html)$" ["doc"]="-f .*\\.md$" - ["deploy"]="-f .*(Dockerfile|\\.sh|\\.xsd|\\.yaml|\\akefile)$" + ["deploy"]="-f .*(Dockerfile|akefile|\\.toml|\\.sh|\\.xsd|\\.yaml)$" ["core"]="-s py ./sia ./tools -s deploy . -f ^(?!procedures/).*\\.md$ ." ["lib"]="-s rust . -s py . -s doc . -s deploy ." diff --git a/setup.py b/setup.py index ddbe62d..a35bf98 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ setup( 'python-dotenv>=1.0.0', 'tiktoken>=0.4.0', 'transformers>=4.30.0', + 'xml_schema_validator @ file:///root/sia/lib/xml_schema_validator', ], python_requires='>=3.10', ) \ No newline at end of file