From 243e4177cb17146827e0935ef71f0a6de4fc43dc Mon Sep 17 00:00:00 2001 From: Geens Date: Sun, 20 Apr 2025 12:08:43 +0200 Subject: [PATCH] Reject whitespace in iteration parser --- setup.py | 44 +++--- sia/iteration_parser.py | 296 ++++++++++++++++++++-------------------- 2 files changed, 168 insertions(+), 172 deletions(-) diff --git a/setup.py b/setup.py index 4f1c9de..a35bf98 100644 --- a/setup.py +++ b/setup.py @@ -1,34 +1,30 @@ from setuptools import setup, find_packages setup( - name="train", + name="sia", version="0.1.0", packages=find_packages(), - scripts=[ - 'bin/train' - ], - + entry_points={ + 'console_scripts': [ + 'sia=sia.__main__:main', + ], + }, install_requires=[ - 'accelerate>=0.25.0', - 'bitsandbytes>=0.45.0', - 'black>=22.0.0', - 'datasets>=2.14.6', - 'einops>=0.7.0', - 'flake8>=4.0.0', - 'ipykernel>=6.0.0', - 'ipywidgets>=8.0.0', - 'peft>=0.8.0', - 'peft>=0.8.0', - 'pytest-cov>=4.0.0', - 'pytest>=7.0.0', - 'pyyaml>=6.0', - 'requests>=2.28.0', - 'sentencepiece>=0.1.99', 'torch>=2.0.0', + 'accelerate>=0.26.0', + 'aiohttp>=3.8.0', + 'bitsandbytes>=0.41.0', + 'dotenv-python>=0.0.1', + 'huggingface_hub>=0.16.0', + 'lxml>=4.9.0', + 'mistral-common>=1.0.0', + 'mistralai>=0.0.7', + 'openai>=1.0.0', + 'psutil>=5.9.0', + 'python-dotenv>=1.0.0', + 'tiktoken>=0.4.0', 'transformers>=4.30.0', - 'trl>=0.7.8', - 'unsloth>=2025.3', - 'vllm==0.8.2', + 'xml_schema_validator @ file:///root/sia/lib/xml_schema_validator', ], python_requires='>=3.10', -) +) \ No newline at end of file diff --git a/sia/iteration_parser.py b/sia/iteration_parser.py index 1aeff23..f6c11a7 100644 --- a/sia/iteration_parser.py +++ b/sia/iteration_parser.py @@ -1,149 +1,149 @@ -from pathlib import Path -from typing import List -import xml.etree.ElementTree as ET - -from .entry import Entry -from .entry.background_entry import BackgroundEntry -from .entry.parse_error_entry import ParseErrorEntry -from .entry.read_entry import ReadEntry -from .entry.reasoning_entry import ReasoningEntry -from .entry.repeat_entry import RepeatEntry -from .entry.single_entry import SingleEntry -from .entry.write_entry import WriteEntry - -class IterationParser: - """Parses iteration XML files into entries""" - - @staticmethod - def parse_iteration(content: str, work_dir: Path, io_buffer) -> tuple[str, str, List[Entry]]: - """ - Parse iteration XML content into context, response and entries - - Args: - content: Raw XML content as string - io_buffer: Buffer to use for IO operations - - Returns: - Tuple of (context_str, response_str, entry_list) - """ - root = ET.fromstring(content) - context_elem = root.find("context") - - if context_elem is None: - raise ValueError("Invalid iteration file - missing context") - - context = ET.fromstring(context_elem.text) - - entries = [] - for elem in context: - if elem.tag == "background": - entries.append(IterationParser._parse_background(elem, work_dir)) - elif elem.tag == "parse_error": - entries.append(IterationParser._parse_parse_error(elem)) - elif elem.tag == "read_stdin": - entries.append(IterationParser._parse_read(elem, io_buffer)) - elif elem.tag == "reasoning": - entries.append(IterationParser._parse_reasoning(elem)) - elif elem.tag == "repeat": - entries.append(IterationParser._parse_repeat(elem, work_dir)) - elif elem.tag == "single": - entries.append(IterationParser._parse_single(elem, work_dir)) - elif elem.tag == "write_stdout": - entries.append(IterationParser._parse_write(elem, io_buffer)) - - return entries - - @staticmethod - def _parse_background(elem: ET.Element, work_dir: Path) -> BackgroundEntry: - entry = BackgroundEntry( - id=elem.get("id"), - script=elem.text, - work_dir=work_dir - ) - if elem.get("exit_code"): - entry.exit_code = int(elem.get("exit_code")) - stdout = elem.find("stdout") - if stdout is not None: - entry.stdout = stdout.text or "" - stderr = elem.find("stderr") - if stderr is not None: - entry.stderr = stderr.text or "" - return entry - - @staticmethod - def _parse_parse_error(elem: ET.Element) -> ParseErrorEntry: - error = elem.find("error") - content = elem.find("content") - return ParseErrorEntry( - id=elem.get("id"), - content=content.text if content is not None else "", - error=error.text if error is not None else "" - ) - - @staticmethod - def _parse_read(elem: ET.Element, io_buffer) -> ReadEntry: - entry = ReadEntry( - id=elem.get("id"), - io_buffer=io_buffer - ) - entry.content = elem.text or "" - entry.read = True - return entry - - @staticmethod - def _parse_reasoning(elem: ET.Element) -> ReasoningEntry: - return ReasoningEntry( - id=elem.get("id"), - content=elem.text or "" - ) - - @staticmethod - def _parse_repeat(elem: ET.Element, work_dir: Path) -> RepeatEntry: - entry = RepeatEntry( - id=elem.get("id"), - work_dir=work_dir, - script=elem.text, - timeout=float(elem.get("timeout")) if elem.get("timeout") else None, - limit=int(elem.get("limit")) if elem.get("limit") else None - ) - if elem.get("exit_code"): - entry.exit_code = int(elem.get("exit_code")) - entry.timed_out = elem.get("timed_out") == "true" - stdout = elem.find("stdout") - if stdout is not None: - entry.stdout = stdout.text or "" - stderr = elem.find("stderr") - if stderr is not None: - entry.stderr = stderr.text or "" - return entry - - @staticmethod - def _parse_single(elem: ET.Element, work_dir: Path) -> SingleEntry: - entry = SingleEntry( - id=elem.get("id"), - work_dir=work_dir, - script=elem.text, - timeout=float(elem.get("timeout")) if elem.get("timeout") else None, - limit=int(elem.get("limit")) if elem.get("limit") else None - ) - if elem.get("exit_code"): - entry.exit_code = int(elem.get("exit_code")) - entry.executed = True - entry.timed_out = elem.get("timed_out") == "true" - stdout = elem.find("stdout") - if stdout is not None: - entry.stdout = stdout.text or "" - stderr = elem.find("stderr") - if stderr is not None: - entry.stderr = stderr.text or "" - return entry - - @staticmethod - def _parse_write(elem: ET.Element, io_buffer) -> WriteEntry: - entry = WriteEntry( - id=elem.get("id"), - content=elem.text or "", - io_buffer=io_buffer - ) - entry.written = True +from pathlib import Path +from typing import List +import xml.etree.ElementTree as ET + +from .entry import Entry +from .entry.background_entry import BackgroundEntry +from .entry.parse_error_entry import ParseErrorEntry +from .entry.read_entry import ReadEntry +from .entry.reasoning_entry import ReasoningEntry +from .entry.repeat_entry import RepeatEntry +from .entry.single_entry import SingleEntry +from .entry.write_entry import WriteEntry + +class IterationParser: + """Parses iteration XML files into entries""" + + @staticmethod + def parse_iteration(content: str, work_dir: Path, io_buffer) -> tuple[str, str, List[Entry]]: + """ + Parse iteration XML content into context, response and entries + + Args: + content: Raw XML content as string + io_buffer: Buffer to use for IO operations + + Returns: + Tuple of (context_str, response_str, entry_list) + """ + root = ET.fromstring(content) + context_elem = root.find("context") + + if context_elem is None: + raise ValueError("Invalid iteration file - missing context") + + context = ET.fromstring(context_elem.text) + + entries = [] + for elem in context: + if elem.tag == "background": + entries.append(IterationParser._parse_background(elem, work_dir)) + elif elem.tag == "parse_error": + entries.append(IterationParser._parse_parse_error(elem)) + elif elem.tag == "read_stdin": + entries.append(IterationParser._parse_read(elem, io_buffer)) + elif elem.tag == "reasoning": + entries.append(IterationParser._parse_reasoning(elem)) + elif elem.tag == "repeat": + entries.append(IterationParser._parse_repeat(elem, work_dir)) + elif elem.tag == "single": + entries.append(IterationParser._parse_single(elem, work_dir)) + elif elem.tag == "write_stdout": + entries.append(IterationParser._parse_write(elem, io_buffer)) + + return entries + + @staticmethod + def _parse_background(elem: ET.Element, work_dir: Path) -> BackgroundEntry: + entry = BackgroundEntry( + id=elem.get("id"), + script=(elem.text or "").strip(), + work_dir=work_dir + ) + if elem.get("exit_code"): + entry.exit_code = int(elem.get("exit_code")) + stdout = elem.find("stdout") + if stdout is not None: + entry.stdout = (stdout.text or "").strip() + stderr = elem.find("stderr") + if stderr is not None: + entry.stderr = (stderr.text or "").strip() + return entry + + @staticmethod + def _parse_parse_error(elem: ET.Element) -> ParseErrorEntry: + error = elem.find("error") + content = elem.find("content") + return ParseErrorEntry( + id=elem.get("id"), + content=(content.text or "").strip(), + error=(error.text or "").strip() + ) + + @staticmethod + def _parse_read(elem: ET.Element, io_buffer) -> ReadEntry: + entry = ReadEntry( + id=elem.get("id"), + io_buffer=io_buffer + ) + entry.content = (elem.text or "").strip() + entry.read = True + return entry + + @staticmethod + def _parse_reasoning(elem: ET.Element) -> ReasoningEntry: + return ReasoningEntry( + id=elem.get("id"), + content=(elem.text or "").strip() + ) + + @staticmethod + def _parse_repeat(elem: ET.Element, work_dir: Path) -> RepeatEntry: + entry = RepeatEntry( + id=elem.get("id"), + work_dir=work_dir, + script=(elem.text or "").strip(), + timeout=float(elem.get("timeout")) if elem.get("timeout") else None, + limit=int(elem.get("limit")) if elem.get("limit") else None + ) + if elem.get("exit_code"): + entry.exit_code = int(elem.get("exit_code")) + entry.timed_out = elem.get("timed_out") == "true" + stdout = elem.find("stdout") + if stdout is not None: + entry.stdout = (stdout.text or "").strip() + stderr = elem.find("stderr") + if stderr is not None: + entry.stderr = (stderr.text or "").strip() + return entry + + @staticmethod + def _parse_single(elem: ET.Element, work_dir: Path) -> SingleEntry: + entry = SingleEntry( + id=elem.get("id"), + work_dir=work_dir, + script=(elem.text or "").strip(), + timeout=float(elem.get("timeout")) if elem.get("timeout") else None, + limit=int(elem.get("limit")) if elem.get("limit") else None + ) + if elem.get("exit_code"): + entry.exit_code = int(elem.get("exit_code")) + entry.executed = True + entry.timed_out = elem.get("timed_out") == "true" + stdout = elem.find("stdout") + if stdout is not None: + entry.stdout = (stdout.text or "").strip() + stderr = elem.find("stderr") + if stderr is not None: + entry.stderr = (stderr.text or "").strip() + return entry + + @staticmethod + def _parse_write(elem: ET.Element, io_buffer) -> WriteEntry: + entry = WriteEntry( + id=elem.get("id"), + content=(elem.text or "").strip(), + io_buffer=io_buffer + ) + entry.written = True return entry \ No newline at end of file