Reject whitespace in iteration parser
This commit is contained in:
44
setup.py
44
setup.py
@@ -1,34 +1,30 @@
|
|||||||
from setuptools import setup, find_packages
|
from setuptools import setup, find_packages
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
name="train",
|
name="sia",
|
||||||
version="0.1.0",
|
version="0.1.0",
|
||||||
packages=find_packages(),
|
packages=find_packages(),
|
||||||
scripts=[
|
entry_points={
|
||||||
'bin/train'
|
'console_scripts': [
|
||||||
],
|
'sia=sia.__main__:main',
|
||||||
|
],
|
||||||
|
},
|
||||||
install_requires=[
|
install_requires=[
|
||||||
'accelerate>=0.25.0',
|
|
||||||
'bitsandbytes>=0.45.0',
|
|
||||||
'black>=22.0.0',
|
|
||||||
'datasets>=2.14.6',
|
|
||||||
'einops>=0.7.0',
|
|
||||||
'flake8>=4.0.0',
|
|
||||||
'ipykernel>=6.0.0',
|
|
||||||
'ipywidgets>=8.0.0',
|
|
||||||
'peft>=0.8.0',
|
|
||||||
'peft>=0.8.0',
|
|
||||||
'pytest-cov>=4.0.0',
|
|
||||||
'pytest>=7.0.0',
|
|
||||||
'pyyaml>=6.0',
|
|
||||||
'requests>=2.28.0',
|
|
||||||
'sentencepiece>=0.1.99',
|
|
||||||
'torch>=2.0.0',
|
'torch>=2.0.0',
|
||||||
|
'accelerate>=0.26.0',
|
||||||
|
'aiohttp>=3.8.0',
|
||||||
|
'bitsandbytes>=0.41.0',
|
||||||
|
'dotenv-python>=0.0.1',
|
||||||
|
'huggingface_hub>=0.16.0',
|
||||||
|
'lxml>=4.9.0',
|
||||||
|
'mistral-common>=1.0.0',
|
||||||
|
'mistralai>=0.0.7',
|
||||||
|
'openai>=1.0.0',
|
||||||
|
'psutil>=5.9.0',
|
||||||
|
'python-dotenv>=1.0.0',
|
||||||
|
'tiktoken>=0.4.0',
|
||||||
'transformers>=4.30.0',
|
'transformers>=4.30.0',
|
||||||
'trl>=0.7.8',
|
'xml_schema_validator @ file:///root/sia/lib/xml_schema_validator',
|
||||||
'unsloth>=2025.3',
|
|
||||||
'vllm==0.8.2',
|
|
||||||
],
|
],
|
||||||
python_requires='>=3.10',
|
python_requires='>=3.10',
|
||||||
)
|
)
|
||||||
@@ -1,149 +1,149 @@
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import List
|
from typing import List
|
||||||
import xml.etree.ElementTree as ET
|
import xml.etree.ElementTree as ET
|
||||||
|
|
||||||
from .entry import Entry
|
from .entry import Entry
|
||||||
from .entry.background_entry import BackgroundEntry
|
from .entry.background_entry import BackgroundEntry
|
||||||
from .entry.parse_error_entry import ParseErrorEntry
|
from .entry.parse_error_entry import ParseErrorEntry
|
||||||
from .entry.read_entry import ReadEntry
|
from .entry.read_entry import ReadEntry
|
||||||
from .entry.reasoning_entry import ReasoningEntry
|
from .entry.reasoning_entry import ReasoningEntry
|
||||||
from .entry.repeat_entry import RepeatEntry
|
from .entry.repeat_entry import RepeatEntry
|
||||||
from .entry.single_entry import SingleEntry
|
from .entry.single_entry import SingleEntry
|
||||||
from .entry.write_entry import WriteEntry
|
from .entry.write_entry import WriteEntry
|
||||||
|
|
||||||
class IterationParser:
|
class IterationParser:
|
||||||
"""Parses iteration XML files into entries"""
|
"""Parses iteration XML files into entries"""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def parse_iteration(content: str, work_dir: Path, io_buffer) -> tuple[str, str, List[Entry]]:
|
def parse_iteration(content: str, work_dir: Path, io_buffer) -> tuple[str, str, List[Entry]]:
|
||||||
"""
|
"""
|
||||||
Parse iteration XML content into context, response and entries
|
Parse iteration XML content into context, response and entries
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
content: Raw XML content as string
|
content: Raw XML content as string
|
||||||
io_buffer: Buffer to use for IO operations
|
io_buffer: Buffer to use for IO operations
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (context_str, response_str, entry_list)
|
Tuple of (context_str, response_str, entry_list)
|
||||||
"""
|
"""
|
||||||
root = ET.fromstring(content)
|
root = ET.fromstring(content)
|
||||||
context_elem = root.find("context")
|
context_elem = root.find("context")
|
||||||
|
|
||||||
if context_elem is None:
|
if context_elem is None:
|
||||||
raise ValueError("Invalid iteration file - missing context")
|
raise ValueError("Invalid iteration file - missing context")
|
||||||
|
|
||||||
context = ET.fromstring(context_elem.text)
|
context = ET.fromstring(context_elem.text)
|
||||||
|
|
||||||
entries = []
|
entries = []
|
||||||
for elem in context:
|
for elem in context:
|
||||||
if elem.tag == "background":
|
if elem.tag == "background":
|
||||||
entries.append(IterationParser._parse_background(elem, work_dir))
|
entries.append(IterationParser._parse_background(elem, work_dir))
|
||||||
elif elem.tag == "parse_error":
|
elif elem.tag == "parse_error":
|
||||||
entries.append(IterationParser._parse_parse_error(elem))
|
entries.append(IterationParser._parse_parse_error(elem))
|
||||||
elif elem.tag == "read_stdin":
|
elif elem.tag == "read_stdin":
|
||||||
entries.append(IterationParser._parse_read(elem, io_buffer))
|
entries.append(IterationParser._parse_read(elem, io_buffer))
|
||||||
elif elem.tag == "reasoning":
|
elif elem.tag == "reasoning":
|
||||||
entries.append(IterationParser._parse_reasoning(elem))
|
entries.append(IterationParser._parse_reasoning(elem))
|
||||||
elif elem.tag == "repeat":
|
elif elem.tag == "repeat":
|
||||||
entries.append(IterationParser._parse_repeat(elem, work_dir))
|
entries.append(IterationParser._parse_repeat(elem, work_dir))
|
||||||
elif elem.tag == "single":
|
elif elem.tag == "single":
|
||||||
entries.append(IterationParser._parse_single(elem, work_dir))
|
entries.append(IterationParser._parse_single(elem, work_dir))
|
||||||
elif elem.tag == "write_stdout":
|
elif elem.tag == "write_stdout":
|
||||||
entries.append(IterationParser._parse_write(elem, io_buffer))
|
entries.append(IterationParser._parse_write(elem, io_buffer))
|
||||||
|
|
||||||
return entries
|
return entries
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _parse_background(elem: ET.Element, work_dir: Path) -> BackgroundEntry:
|
def _parse_background(elem: ET.Element, work_dir: Path) -> BackgroundEntry:
|
||||||
entry = BackgroundEntry(
|
entry = BackgroundEntry(
|
||||||
id=elem.get("id"),
|
id=elem.get("id"),
|
||||||
script=elem.text,
|
script=(elem.text or "").strip(),
|
||||||
work_dir=work_dir
|
work_dir=work_dir
|
||||||
)
|
)
|
||||||
if elem.get("exit_code"):
|
if elem.get("exit_code"):
|
||||||
entry.exit_code = int(elem.get("exit_code"))
|
entry.exit_code = int(elem.get("exit_code"))
|
||||||
stdout = elem.find("stdout")
|
stdout = elem.find("stdout")
|
||||||
if stdout is not None:
|
if stdout is not None:
|
||||||
entry.stdout = stdout.text or ""
|
entry.stdout = (stdout.text or "").strip()
|
||||||
stderr = elem.find("stderr")
|
stderr = elem.find("stderr")
|
||||||
if stderr is not None:
|
if stderr is not None:
|
||||||
entry.stderr = stderr.text or ""
|
entry.stderr = (stderr.text or "").strip()
|
||||||
return entry
|
return entry
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _parse_parse_error(elem: ET.Element) -> ParseErrorEntry:
|
def _parse_parse_error(elem: ET.Element) -> ParseErrorEntry:
|
||||||
error = elem.find("error")
|
error = elem.find("error")
|
||||||
content = elem.find("content")
|
content = elem.find("content")
|
||||||
return ParseErrorEntry(
|
return ParseErrorEntry(
|
||||||
id=elem.get("id"),
|
id=elem.get("id"),
|
||||||
content=content.text if content is not None else "",
|
content=(content.text or "").strip(),
|
||||||
error=error.text if error is not None else ""
|
error=(error.text or "").strip()
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _parse_read(elem: ET.Element, io_buffer) -> ReadEntry:
|
def _parse_read(elem: ET.Element, io_buffer) -> ReadEntry:
|
||||||
entry = ReadEntry(
|
entry = ReadEntry(
|
||||||
id=elem.get("id"),
|
id=elem.get("id"),
|
||||||
io_buffer=io_buffer
|
io_buffer=io_buffer
|
||||||
)
|
)
|
||||||
entry.content = elem.text or ""
|
entry.content = (elem.text or "").strip()
|
||||||
entry.read = True
|
entry.read = True
|
||||||
return entry
|
return entry
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _parse_reasoning(elem: ET.Element) -> ReasoningEntry:
|
def _parse_reasoning(elem: ET.Element) -> ReasoningEntry:
|
||||||
return ReasoningEntry(
|
return ReasoningEntry(
|
||||||
id=elem.get("id"),
|
id=elem.get("id"),
|
||||||
content=elem.text or ""
|
content=(elem.text or "").strip()
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _parse_repeat(elem: ET.Element, work_dir: Path) -> RepeatEntry:
|
def _parse_repeat(elem: ET.Element, work_dir: Path) -> RepeatEntry:
|
||||||
entry = RepeatEntry(
|
entry = RepeatEntry(
|
||||||
id=elem.get("id"),
|
id=elem.get("id"),
|
||||||
work_dir=work_dir,
|
work_dir=work_dir,
|
||||||
script=elem.text,
|
script=(elem.text or "").strip(),
|
||||||
timeout=float(elem.get("timeout")) if elem.get("timeout") else None,
|
timeout=float(elem.get("timeout")) if elem.get("timeout") else None,
|
||||||
limit=int(elem.get("limit")) if elem.get("limit") else None
|
limit=int(elem.get("limit")) if elem.get("limit") else None
|
||||||
)
|
)
|
||||||
if elem.get("exit_code"):
|
if elem.get("exit_code"):
|
||||||
entry.exit_code = int(elem.get("exit_code"))
|
entry.exit_code = int(elem.get("exit_code"))
|
||||||
entry.timed_out = elem.get("timed_out") == "true"
|
entry.timed_out = elem.get("timed_out") == "true"
|
||||||
stdout = elem.find("stdout")
|
stdout = elem.find("stdout")
|
||||||
if stdout is not None:
|
if stdout is not None:
|
||||||
entry.stdout = stdout.text or ""
|
entry.stdout = (stdout.text or "").strip()
|
||||||
stderr = elem.find("stderr")
|
stderr = elem.find("stderr")
|
||||||
if stderr is not None:
|
if stderr is not None:
|
||||||
entry.stderr = stderr.text or ""
|
entry.stderr = (stderr.text or "").strip()
|
||||||
return entry
|
return entry
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _parse_single(elem: ET.Element, work_dir: Path) -> SingleEntry:
|
def _parse_single(elem: ET.Element, work_dir: Path) -> SingleEntry:
|
||||||
entry = SingleEntry(
|
entry = SingleEntry(
|
||||||
id=elem.get("id"),
|
id=elem.get("id"),
|
||||||
work_dir=work_dir,
|
work_dir=work_dir,
|
||||||
script=elem.text,
|
script=(elem.text or "").strip(),
|
||||||
timeout=float(elem.get("timeout")) if elem.get("timeout") else None,
|
timeout=float(elem.get("timeout")) if elem.get("timeout") else None,
|
||||||
limit=int(elem.get("limit")) if elem.get("limit") else None
|
limit=int(elem.get("limit")) if elem.get("limit") else None
|
||||||
)
|
)
|
||||||
if elem.get("exit_code"):
|
if elem.get("exit_code"):
|
||||||
entry.exit_code = int(elem.get("exit_code"))
|
entry.exit_code = int(elem.get("exit_code"))
|
||||||
entry.executed = True
|
entry.executed = True
|
||||||
entry.timed_out = elem.get("timed_out") == "true"
|
entry.timed_out = elem.get("timed_out") == "true"
|
||||||
stdout = elem.find("stdout")
|
stdout = elem.find("stdout")
|
||||||
if stdout is not None:
|
if stdout is not None:
|
||||||
entry.stdout = stdout.text or ""
|
entry.stdout = (stdout.text or "").strip()
|
||||||
stderr = elem.find("stderr")
|
stderr = elem.find("stderr")
|
||||||
if stderr is not None:
|
if stderr is not None:
|
||||||
entry.stderr = stderr.text or ""
|
entry.stderr = (stderr.text or "").strip()
|
||||||
return entry
|
return entry
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _parse_write(elem: ET.Element, io_buffer) -> WriteEntry:
|
def _parse_write(elem: ET.Element, io_buffer) -> WriteEntry:
|
||||||
entry = WriteEntry(
|
entry = WriteEntry(
|
||||||
id=elem.get("id"),
|
id=elem.get("id"),
|
||||||
content=elem.text or "",
|
content=(elem.text or "").strip(),
|
||||||
io_buffer=io_buffer
|
io_buffer=io_buffer
|
||||||
)
|
)
|
||||||
entry.written = True
|
entry.written = True
|
||||||
return entry
|
return entry
|
||||||
Reference in New Issue
Block a user