llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix dockerfile for xml schema validator

Browse Source
This commit is contained in:
Geens
2025-04-06 16:11:36 +02:00
parent 55a920414d
commit ea93adc4c8
21 changed files with 143 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
Dockerfile
View File

@@ -35,7 +35,8 @@ COPY ./scripts/setup_binaries.py /root/sia/scripts/
COPY ./tools/itb/setup.py /root/sia/tools/itb/setup.py
RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/tools/itb/setup.py
RUN python3 -m venv /root/venvs/itb
RUN /root/venvs/itb/bin/pip install -e /root/sia/tools/itb/
RUN --mount=type=cache,target=/root/.cache/pip \
/root/venvs/itb/bin/pip install -e /root/sia/tools/itb/
# Train tool setup
FROM base AS train-env
@@ -43,15 +44,23 @@ COPY ./scripts/setup_binaries.py /root/sia/scripts/
COPY ./tools/train/setup.py /root/sia/tools/train/setup.py
RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/tools/train/setup.py
RUN python3 -m venv /root/venvs/train
RUN /root/venvs/train/bin/pip install -e /root/sia/tools/train/
RUN --mount=type=cache,target=/root/.cache/pip \
/root/venvs/train/bin/pip install -e /root/sia/tools/train/
# SIA core setup
FROM base AS sia-env
RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
COPY action_schema.xsd /root/sia/action_schema.xsd
COPY ./scripts/setup_binaries.py /root/sia/scripts/
COPY ./setup.py /root/sia/setup.py
COPY ./lib /root/sia/lib
RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/setup.py
RUN python3 -m venv /root/venvs/sia
RUN /root/venvs/sia/bin/pip install -e /root/sia/
RUN --mount=type=cache,target=/root/.cache/pip \
/root/venvs/sia/bin/pip install -e /root/sia/
# Web frontend build
FROM node:20-alpine AS web-build

2
lib/xml_schema_validator/.dockerignore Normal file
View File

@@ -0,0 +1,2 @@
target/
Cargo.lock

3
lib/xml_schema_validator/README.md Normal file
View File

@@ -0,0 +1,3 @@
# XML Schema Validator
XML Schema validation library optimized for LLM token validation.

2
lib/xml_schema_validator/cargo.toml
View File

@@ -20,4 +20,4 @@ thiserror = "1.0.38" # Better error handling
pyo3 = { version = "0.18.0", features = ["extension-module"] }
[build-dependencies]
xml_schema_generator = "0.6.18"
xml_schema_generator = { version = "0.6.18", features = ["env_logger"] }

2
lib/xml_schema_validator/pyproject.toml
View File

@@ -6,7 +6,6 @@ build-backend = "maturin"
name = "xml_schema_validator"
version = "0.1.0"
description = "XML Schema validation library for SIA"
readme = "README.md"
requires-python = ">=3.10"
classifiers = [
"Programming Language :: Rust",
@@ -21,5 +20,4 @@ dependencies = []
repository = "https://github.com/sia/xml_schema_validator"
[tool.maturin]
python-source = "python"
features = ["pyo3/extension-module"]

6
lib/xml_schema_validator/src/lib.rs
View File

@@ -4,6 +4,7 @@
//! A streaming XML validator that checks XML fragments against an XSD schema
mod error;
mod python;
mod schema;
mod token;
@@ -13,7 +14,7 @@ use token::*;
/// An XML validator that checks XML fragments against an XSD schema
#[derive(Clone, Debug)]
pub struct XmlValidator {
current_tokens: Vec<std::rc::Rc<Token>>,
current_tokens: Vec<Box<Token>>,
}
/// The Result type used throughout this crate
@@ -26,10 +27,9 @@ impl XmlValidator {
pub fn new(schema_text: &str) -> Result<Self> {
let schema: schema::XsSchema = quick_xml::de::from_str(schema_text)?;
let current_tokens = schema.xs_element.iter().map(|element| {
std::rc::Rc::new(Token::StartOfFile(StartOfFile::new(std::rc::Rc::new(element.clone()))))
Box::new(Token::StartOfFile(StartOfFile::new(Box::new(element.clone()))))
}).collect();
//let current_tokens = vec![std::rc::Rc::new(Token::StartOfFile(StartOfFile))];
Ok(Self {
current_tokens,
})

32
lib/xml_schema_validator/src/python.rs Normal file
View File

@@ -0,0 +1,32 @@
use crate::XmlValidator;
use pyo3::prelude::*;
use pyo3::exceptions::PyValueError;
/// Python wrapper for XmlValidator
#[pyclass(unsendable)]
struct PyXmlValidator {
validator: XmlValidator,
}
#[pymethods]
impl PyXmlValidator {
#[new]
fn new(schema_text: &str) -> PyResult<Self> {
let validator = XmlValidator::new(schema_text)
.map_err(|e| PyValueError::new_err(format!("Failed to create validator: {}", e)))?;
Ok(Self { validator })
}
fn append(&mut self, fragment: &str) -> PyResult<()> {
self.validator.append(fragment)
.map_err(|e| PyValueError::new_err(format!("Validation error: {}", e)))
}
}
/// Python module for XML Schema validation
#[pymodule]
fn xml_schema_validator(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
m.add_class::<PyXmlValidator>()?;
Ok(())
}

4
lib/xml_schema_validator/src/token/element_close_end.rs
View File

@@ -11,9 +11,9 @@ impl ElementCloseEnd {
}
}
pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
pub fn append(&self, c: char) -> Vec<Box<Token>> {
if '>' == c {
vec![std::rc::Rc::new(Token::EndOfFile(EndOfFile::new()))]
vec![Box::new(Token::EndOfFile(EndOfFile::new()))]
} else {
vec![]
}

16
lib/xml_schema_validator/src/token/element_close_name.rs
View File

@@ -5,21 +5,21 @@ use crate::*;
#[derive(Clone, Debug)]
pub struct ElementCloseName {
/// The element that was previously opened and is now being closed
element: std::rc::Rc<schema::XsElement>,
element: Box<schema::XsElement>,
/// Current buffer of characters processed for the element name
buffer: String,
}
impl ElementCloseName {
/// Create a new instance with the given buffer
pub fn new(element: std::rc::Rc<schema::XsElement>, buffer: String) -> Self {
pub fn new(element: Box<schema::XsElement>, buffer: String) -> Self {
Self {
element,
buffer,
}
}
pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
pub fn append(&self, c: char) -> Vec<Box<Token>> {
let new_buffer = self.buffer.clone() + &c.to_string();
// If still building the element name
@@ -29,17 +29,17 @@ impl ElementCloseName {
// Now we're expecting '>' to close the tag
if c.is_whitespace() {
// Handle whitespace after the element name
vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new(
vec![std::rc::Rc::new(Token::ElementCloseEnd(ElementCloseEnd::new()))]
vec![Box::new(Token::Whitespace(Whitespace::new(
vec![Box::new(Token::ElementCloseEnd(ElementCloseEnd::new()))]
)))]
} else {
// Directly transition to the closed state
vec![std::rc::Rc::new(Token::ElementCloseEnd(ElementCloseEnd::new()))]
vec![Box::new(Token::ElementCloseEnd(ElementCloseEnd::new()))]
}
} else {
// Still building the element name
vec![std::rc::Rc::new(Token::ElementCloseName(ElementCloseName::new(
self.element.clone(),
vec![Box::new(Token::ElementCloseName(ElementCloseName::new(
Box::new((*self.element).clone()),
new_buffer,
)))]
}

14
lib/xml_schema_validator/src/token/element_close_start.rs
View File

@@ -4,28 +4,28 @@ use crate::*;
#[derive(Clone, Debug)]
pub struct ElementCloseStart {
/// The element that was previously opened and is now being closed
element: std::rc::Rc<schema::XsElement>,
element: Box<schema::XsElement>,
has_slash: bool,
}
impl ElementCloseStart {
pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
pub fn new(element: Box<schema::XsElement>) -> Self {
Self {
element,
has_slash: false,
}
}
pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
pub fn append(&self, c: char) -> Vec<Box<Token>> {
if !self.has_slash && '/' == c {
vec![std::rc::Rc::new(Token::ElementCloseStart(Self {
element: self.element.clone(),
vec![Box::new(Token::ElementCloseStart(Self {
element: Box::new((*self.element).clone()),
has_slash: true,
}))]
} else {
if self.element.name.starts_with(c) {
vec![std::rc::Rc::new(Token::ElementCloseName(ElementCloseName::new(
self.element.clone(),
vec![Box::new(Token::ElementCloseName(ElementCloseName::new(
Box::new((*self.element).clone()),
c.to_string(),
)))]
} else {

12
lib/xml_schema_validator/src/token/element_open_end.rs
View File

@@ -3,17 +3,17 @@ use crate::*;
/// Represents a fully opened XML element, containing the schema element reference
#[derive(Clone, Debug)]
pub struct ElementOpenEnd {
element: std::rc::Rc<schema::XsElement>,
element: Box<schema::XsElement>,
}
impl ElementOpenEnd {
pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
pub fn new(element: Box<schema::XsElement>) -> Self {
Self {
element,
}
}
pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
pub fn append(&self, c: char) -> Vec<Box<Token>> {
// Check if the element allows text content (mixed="true")
let allows_text = self.element.xs_complex_type.mixed
.as_ref()
@@ -22,11 +22,11 @@ impl ElementOpenEnd {
if allows_text {
if c.is_whitespace() {
vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new(
vec![std::rc::Rc::new(Token::TextContent(TextContent::new(self.element.clone())))]
vec![Box::new(Token::Whitespace(Whitespace::new(
vec![Box::new(Token::TextContent(TextContent::new(Box::new((*self.element).clone()))))]
)))]
} else {
vec![std::rc::Rc::new(Token::TextContent(TextContent::new(self.element.clone())))]
vec![Box::new(Token::TextContent(TextContent::new(Box::new((*self.element).clone()))))]
}
} else {
vec![]

16
lib/xml_schema_validator/src/token/element_open_name.rs
View File

@@ -4,11 +4,11 @@ use crate::*;
#[derive(Clone, Debug)]
pub struct ElementOpenName {
buffer: String,
element: std::rc::Rc<schema::XsElement>,
element: Box<schema::XsElement>,
}
impl ElementOpenName {
pub fn new(element: std::rc::Rc<schema::XsElement>, buffer: String) -> Result<Self> {
pub fn new(element: Box<schema::XsElement>, buffer: String) -> Result<Self> {
if element.name.starts_with(buffer.as_str()) {
Ok(Self {
buffer,
@@ -19,19 +19,19 @@ impl ElementOpenName {
}
}
pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
pub fn append(&self, c: char) -> Vec<Box<Token>> {
let buffer = self.buffer.clone() + &c.to_string();
if self.element.name.starts_with(buffer.as_str()) {
vec![std::rc::Rc::new(Token::ElementName(ElementOpenName {
vec![Box::new(Token::ElementName(ElementOpenName {
buffer,
element: self.element.clone(),
element: Box::new((*self.element).clone()),
}))]
} else if c.is_whitespace() {
vec![std::rc::Rc::new(Token::Whitespace(Whitespace::new(
vec![std::rc::Rc::new(Token::ElementOpen(ElementOpenEnd::new(self.element.clone())))]
vec![Box::new(Token::Whitespace(Whitespace::new(
vec![Box::new(Token::ElementOpen(ElementOpenEnd::new(Box::new((*self.element).clone()))))]
)))]
} else if '>' == c {
vec![std::rc::Rc::new(Token::ElementOpen(ElementOpenEnd::new(self.element.clone())))]
vec![Box::new(Token::ElementOpen(ElementOpenEnd::new(Box::new((*self.element).clone()))))]
} else {
vec![]
}

10
lib/xml_schema_validator/src/token/element_open_start.rs
View File

@@ -3,20 +3,20 @@ use crate::*;
/// Represents an XML element that is in the process of being opened, containing the text buffer and schema element reference
#[derive(Clone, Debug)]
pub struct ElementOpenStart {
element: std::rc::Rc<schema::XsElement>,
element: Box<schema::XsElement>,
}
impl ElementOpenStart {
pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
pub fn new(element: Box<schema::XsElement>) -> Self {
Self {
element,
}
}
pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
let element_name = ElementOpenName::new(self.element.clone(), c.to_string());
pub fn append(&self, c: char) -> Vec<Box<Token>> {
let element_name = ElementOpenName::new(Box::new((*self.element).clone()), c.to_string());
if let Ok(element_name) = element_name {
vec![std::rc::Rc::from(Token::ElementName(element_name))]
vec![Box::new(Token::ElementName(element_name))]
} else {
vec![]
}

2
lib/xml_schema_validator/src/token/end_of_file.rs
View File

@@ -10,7 +10,7 @@ impl EndOfFile {
Self {}
}
pub fn append(&self, _c: char) -> Vec<std::rc::Rc<Token>> {
pub fn append(&self, _c: char) -> Vec<Box<Token>> {
vec![]
}
}

2
lib/xml_schema_validator/src/token/mod.rs
View File

@@ -36,7 +36,7 @@ pub enum Token {
}
impl Token {
pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
pub fn append(&self, c: char) -> Vec<Box<Token>> {
match self {
Token::ElementCloseEnd(element_close_end) => element_close_end.append(c),
Token::ElementCloseName(element_close_name) => element_close_name.append(c),

8
lib/xml_schema_validator/src/token/start_of_file.rs
View File

@@ -3,17 +3,17 @@ use crate::*;
/// Represents the initial state at the start of an XML file, containing a reference to the root element schema
#[derive(Clone, Debug)]
pub struct StartOfFile {
element: std::rc::Rc<schema::XsElement>,
element: Box<schema::XsElement>,
}
impl StartOfFile {
pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
pub fn new(element: Box<schema::XsElement>) -> Self {
Self { element }
}
pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
pub fn append(&self, c: char) -> Vec<Box<Token>> {
if '<' == c {
vec![std::rc::Rc::from(Token::ElementStart(ElementOpenStart::new(self.element.clone())))]
vec![Box::new(Token::ElementStart(ElementOpenStart::new(Box::new((*self.element).clone()))))]
} else {
vec![]
}

14
lib/xml_schema_validator/src/token/text_content.rs
View File

@@ -3,12 +3,12 @@ use crate::*;
/// Represents the text content within an XML element
#[derive(Clone, Debug)]
pub struct TextContent {
element: std::rc::Rc<schema::XsElement>,
element: Box<schema::XsElement>,
buffer: String,
}
impl TextContent {
pub fn new(element: std::rc::Rc<schema::XsElement>) -> Self {
pub fn new(element: Box<schema::XsElement>) -> Self {
Self {
element,
buffer: String::new(),
@@ -16,19 +16,19 @@ impl TextContent {
}
/// Create a new TextContent with the given initial buffer
pub fn with_buffer(element: std::rc::Rc<schema::XsElement>, buffer: String) -> Self {
pub fn with_buffer(element: Box<schema::XsElement>, buffer: String) -> Self {
Self {
element,
buffer,
}
}
pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
pub fn append(&self, c: char) -> Vec<Box<Token>> {
if c == '<' {
vec![std::rc::Rc::new(Token::ElementCloseStart(ElementCloseStart::new(self.element.clone())))]
vec![Box::new(Token::ElementCloseStart(ElementCloseStart::new(Box::new((*self.element).clone()))))]
} else {
vec![std::rc::Rc::new(Token::TextContent(TextContent::with_buffer(
self.element.clone(),
vec![Box::new(Token::TextContent(TextContent::with_buffer(
Box::new((*self.element).clone()),
self.buffer.clone() + &c.to_string(),
)))]
}

8
lib/xml_schema_validator/src/token/whitespace.rs
View File

@@ -3,19 +3,19 @@ use crate::*;
/// Represents one or more whitespace characters
#[derive(Clone, Debug)]
pub struct Whitespace {
next: Vec<std::rc::Rc<Token>>
next: Vec<Box<Token>>
}
impl Whitespace {
pub fn new(next: Vec<std::rc::Rc<Token>>) -> Self {
pub fn new(next: Vec<Box<Token>>) -> Self {
Self {
next
}
}
pub fn append(&self, c: char) -> Vec<std::rc::Rc<Token>> {
pub fn append(&self, c: char) -> Vec<Box<Token>> {
if c.is_whitespace() {
vec![std::rc::Rc::new(Token::Whitespace(Self {
vec![Box::new(Token::Whitespace(Self {
next: self.next.clone()
}))]
} else {

13
scripts/bootstrap.sh
View File

@@ -19,7 +19,18 @@ fi
# Install required packages
apt-get update
apt-get install -y \
vim
gnupg \
vim \
wget \
;
wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list
apt-get update && \
apt-get install -y \
google-chrome-stable
curl https://sh.rustup.rs -sSf | bash -s -- -y
export PATH="/root/.cargo/bin:${PATH}"
# Create directory structure
echo "Creating directory structure..."

2
scripts/collect.sh
View File

@@ -7,7 +7,7 @@ declare -A FILTER_SETS=(
["rust"]="-f .*\\.(rs|toml)$"
["web"]="-f .*\\.(js|jsx|json|css|html)$"
["doc"]="-f .*\\.md$"
["deploy"]="-f .*(Dockerfile|\\.sh|\\.xsd|\\.yaml|\\akefile)$"
["deploy"]="-f .*(Dockerfile|akefile|\\.toml|\\.sh|\\.xsd|\\.yaml)$"
["core"]="-s py ./sia ./tools -s deploy . -f ^(?!procedures/).*\\.md$ ."
["lib"]="-s rust . -s py . -s doc . -s deploy ."

1
setup.py
View File

@@ -24,6 +24,7 @@ setup(
'python-dotenv>=1.0.0',
'tiktoken>=0.4.0',
'transformers>=4.30.0',
'xml_schema_validator @ file:///root/sia/lib/xml_schema_validator',
],
python_requires='>=3.10',
)