Directory Tree:
.
|____.dockerignore
|____.env
|____.gitignore
|____action_schema.xsd
|____collect.txt
|____data
| |____iterations
| | |____iteration_20250109_190024_373.xml
| | |____iteration_20250109_190035_409.xml
| | |____iteration_20250109_190704_483.xml
| | |____iteration_20250109_190709_915.xml
| | |____iteration_20250109_190715_552.xml
| | |____iteration_20250109_190722_336.xml
| | |____iteration_20250109_190727_381.xml
| | |____iteration_20250109_190733_628.xml
| | |____iteration_20250109_190744_602.xml
| | |____iteration_20250109_190749_974.xml
| | |____iteration_20250109_190817_093.xml
| | |____iteration_20250109_190822_365.xml
| | |____iteration_20250109_190827_602.xml
| | |____iteration_20250109_190843_601.xml
| | |____iteration_20250109_190849_963.xml
| | |____iteration_20250109_190954_879.xml
| | |____iteration_20250109_190957_794.xml
| | |____iteration_20250109_191006_871.xml
| | |____iteration_20250109_191015_730.xml
| | |____iteration_20250109_191037_978.xml
| | |____iteration_20250109_191052_893.xml
| | |____iteration_20250109_191122_014.xml
| | |____iteration_20250109_191125_681.xml
| | |____iteration_20250109_191132_177.xml
| | |____iteration_20250109_191149_428.xml
| | |____iteration_20250109_191156_182.xml
| | |____iteration_20250109_191209_080.xml
| | |____iteration_20250109_191232_614.xml
| | |____iteration_20250109_191750_117.xml
| | |____iteration_20250109_191820_461.xml
| | |____iteration_20250109_191829_707.xml
| | |____iteration_20250109_191848_803.xml
| | |____iteration_20250109_191857_135.xml
| | |____iteration_20250109_191903_535.xml
| | |____iteration_20250109_191915_839.xml
| | |____iteration_20250109_191919_044.xml
| | |____iteration_20250109_191923_827.xml
| | |____iteration_20250109_191927_361.xml
| | |____iteration_20250109_191931_571.xml
| | |____iteration_20250109_191935_623.xml
| | |____iteration_20250109_191944_403.xml
| | |____iteration_20250109_191948_286.xml
| | |____iteration_20250109_191951_279.xml
| | |____iteration_20250109_191958_277.xml
| | |____iteration_20250109_192001_725.xml
| | |____iteration_20250109_192005_801.xml
| | |____iteration_20250109_192008_636.xml
| | |____iteration_20250109_192011_586.xml
| | |____iteration_20250109_192014_562.xml
| | |____iteration_20250109_192016_964.xml
| | |____iteration_20250109_192019_514.xml
| | |____iteration_20250109_192023_754.xml
| | |____iteration_20250109_192029_400.xml
| | |____iteration_20250109_192053_637.xml
| | |____iteration_20250109_192108_043.xml
| | |____iteration_20250109_192112_980.xml
| | |____iteration_20250109_192116_218.xml
| | |____iteration_20250109_192118_117.xml
| | |____iteration_20250109_192120_069.xml
| | |____iteration_20250109_192122_109.xml
| | |____iteration_20250109_192125_737.xml
| | |____iteration_20250109_192139_935.xml
| | |____iteration_20250109_192246_171.xml
| | |____iteration_20250109_192251_105.xml
| | |____iteration_20250109_192301_749.xml
| | |____iteration_20250109_192305_163.xml
| | |____iteration_20250109_192307_861.xml
| | |____iteration_20250109_192311_217.xml
| | |____iteration_20250109_192315_575.xml
| | |____iteration_20250109_192318_269.xml
| | |____iteration_20250109_192320_920.xml
| | |____iteration_20250109_192324_888.xml
| | |____iteration_20250109_192328_431.xml
| | |____iteration_20250109_192332_861.xml
| | |____iteration_20250109_192335_705.xml
| | |____iteration_20250109_192338_398.xml
| | |____iteration_20250109_192341_071.xml
| | |____iteration_20250109_192344_092.xml
| | |____iteration_20250109_192353_015.xml
| | |____iteration_20250109_192356_373.xml
| | |____iteration_20250109_192359_203.xml
| | |____iteration_20250109_192404_132.xml
| | |____iteration_20250109_192409_428.xml
| | |____iteration_20250109_192412_471.xml
| | |____iteration_20250109_192417_100.xml
| | |____iteration_20250109_192420_341.xml
| | |____iteration_20250109_192423_203.xml
| | |____iteration_20250109_192426_541.xml
| | |____iteration_20250109_192429_583.xml
| | |____iteration_20250109_192437_744.xml
| | |____iteration_20250109_192450_273.xml
| | |____iteration_20250109_192455_171.xml
| | |____iteration_20250109_192504_385.xml
| | |____iteration_20250109_192507_329.xml
| | |____iteration_20250109_192532_348.xml
| | |____iteration_20250109_192539_330.xml
| | |____iteration_20250109_192547_957.xml
| | |____iteration_20250109_192600_776.xml
| | |____iteration_20250109_192611_571.xml
| |____tasks
| | |____fix_unit_tests.txt
| | |____improve_sia.txt
| | |____kapla.txt
| |____user
|____Dockerfile
|____environment
|____iterations
|____mini_prompt.md
|____model
| |____config.json
| |____generation_config.json
| |____model.safetensors
| |____special_tokens_map.json
| |____tokenizer.json
| |____tokenizer_config.json
|____procedures
| |____filesystem_design
| | |____reasoning.md
| |____README.md
| |____self_improvement
| | |____reasoning.md
| |____tool_management
| | |____reasoning.md
| |____user_communication
| | |____description.md
| | |____procedure.md
| | |____reasoning.md
| |____using_procedures
| | |____procedure.md
| | |____reasoning.md
| |____version_control
| | |____reasoning.md
|____README.md
|____requirements.txt
|____scripts
| |____bootstrap.sh
| |____collect.sh
| |____container.sh
| |____restart.sh
| |____run.sh
| |____setup_binaries.py
| |____test.sh
|____setup.py
|____sia
| |____auto_approver.py
| |____base_agent.py
| |____command.py
| |____command_result.py
| |____config.py
| |____delete_command.py
| |____entry
| | |____background_entry.py
| | |____entry_factory.py
| | |____parse_error_entry.py
| | |____read_entry.py
| | |____reasoning_entry.py
| | |____repeat_entry.py
| | |____single_entry.py
| | |____write_entry.py
| | |______init__.py
| | |______pycache__
| | | |____background_entry.cpython-310.pyc
| | | |____entry_factory.cpython-310.pyc
| | | |____parse_error_entry.cpython-310.pyc
| | | |____read_entry.cpython-310.pyc
| | | |____reasoning_entry.cpython-310.pyc
| | | |____repeat_entry.cpython-310.pyc
| | | |____single_entry.cpython-310.pyc
| | | |____write_entry.cpython-310.pyc
| | | |______init__.cpython-310.pyc
| |____io_buffer.py
| |____iteration_logger.py
| |____iteration_parser.py
| |____llm_engine
| | |____deepseek_llm_engine.py
| | |____hf_llm_engine.py
| | |____local_llm_engine.py
| | |____mistral_llm_engine.py
| | |____openai_llm_engine.py
| | |______init__.py
| | |______pycache__
| | | |____deepseek_llm_engine.cpython-310.pyc
| | | |____hf_llm_engine.cpython-310.pyc
| | | |____local_llm_engine.cpython-310.pyc
| | | |____mistral_llm_engine.cpython-310.pyc
| | | |____openai_llm_engine.cpython-310.pyc
| | | |______init__.cpython-310.pyc
| |____response_parser.py
| |____standard_io_buffer.py
| |____stop_command.py
| |____system_metrics.py
| |____util.py
| |____web
| | |____api.py
| | |____auto_approver_websocket.py
| | |____context_websocket.py
| | |____llm_websocket.py
| | |____memory_websocket.py
| | |____static.py
| | |____stdout_websocket.py
| | |____token_websocket.py
| | |____util.py
| | |____websockts.py
| | |______pycache__
| | | |____api.cpython-310.pyc
| | | |____auto_approver_websocket.cpython-310.pyc
| | | |____context_websocket.cpython-310.pyc
| | | |____llm_websocket.cpython-310.pyc
| | | |____memory_websocket.cpython-310.pyc
| | | |____static.cpython-310.pyc
| | | |____stdout_websocket.cpython-310.pyc
| | | |____token_websocket.cpython-310.pyc
| | | |____util.cpython-310.pyc
| | | |____websockts.cpython-310.pyc
| |____web_agent.py
| |____web_io_buffer.py
| |____working_memory.py
| |____xml_validator.py
| |______init__.py
| |______main__.py
| |______pycache__
| | |____auto_approver.cpython-310.pyc
| | |____base_agent.cpython-310.pyc
| | |____command.cpython-310.pyc
| | |____command_result.cpython-310.pyc
| | |____config.cpython-310.pyc
| | |____delete_command.cpython-310.pyc
| | |____hf_llm_engine.cpython-310.pyc
| | |____io_buffer.cpython-310.pyc
| | |____iteration_logger.cpython-310.pyc
| | |____iteration_parser.cpython-310.pyc
| | |____llm_engine.cpython-310.pyc
| | |____local_llm_engine.cpython-310.pyc
| | |____mistral_llm_engine.cpython-310.pyc
| | |____openai_llm_engine.cpython-310.pyc
| | |____response_parser.cpython-310.pyc
| | |____stop_command.cpython-310.pyc
| | |____system_metrics.cpython-310.pyc
| | |____util.cpython-310.pyc
| | |____web_agent.cpython-310.pyc
| | |____web_io_buffer.cpython-310.pyc
| | |____working_memory.cpython-310.pyc
| | |____xml_validator.cpython-310.pyc
| | |______init__.cpython-310.pyc
| | |______main__.cpython-310.pyc
|____sia.egg-info
| |____dependency_links.txt
| |____entry_points.txt
| |____PKG-INFO
| |____requires.txt
| |____SOURCES.txt
| |____top_level.txt
|____system_prompt.md
|____tasks
|____test
| |____auto_approver_test.py
| |____background_entry_test.py
| |____base_agent_test.py
| |____delete_command_test.py
| |____local_llm_engine_test.py
| |____parse_error_entry_test.py
| |____read_entry_test.py
| |____reasoning_entry_test.py
| |____repeat_entry_test.py
| |____response_parser_test.py
| |____single_entry_test.py
| |____standard_io_buffer_test.py
| |____stop_command_test.py
| |____system_metrics_test.py
| |____test_data.py
| |____util_test.py
| |____web_agent_test.py
| |____web_io_buffer_test.py
| |____web_socket_manager_test.py
| |____working_memory_test.py
| |____write_entry_test.py
| |____xml_validator_test.py
| |______init__.py
|____tools
| |____itb
| | |____bin
| | | |____itb_click
| | | |____itb_cursor
| | | |____itb_input
| | | |____itb_navigate
| | | |____itb_refresh
| | | |____itb_screenshot
| | | |____itb_scroll
| | | |____itb_start
| | |____itb.egg-info
| | | |____dependency_links.txt
| | | |____PKG-INFO
| | | |____requires.txt
| | | |____SOURCES.txt
| | | |____top_level.txt
| | |____js
| | | |____dom_analyzer.js
| | | |____dom_analyzer.js.orig
| | | |____dom_analyzer.js.patch
| | | |____dom_analyzer.js.rej
| | | |____mouse.js
| | | |____viewport.js
| | |____README.md
| | |____requirements.txt
| | |____setup.py
| | |____social.html
| |____train
| | |____bin
| | | |____train_deepseek
| | | |____train_mistral
| | |____readme.md
| | |____requirements.txt
| | |____setup.py
| | |____train
| | | |____mistral_api.py
| | | |____unsloth_deepseek.py
| | | |____util.py
| | | |______init__.py
| | |____train.sh
|____training
| |____clean_start
| | |____iteration_20250116_134549_655.xml
| | |____iteration_20250116_134555_680.xml
| |____config.yaml
| |____delete_indicated_entries
| | |____iteration_20250116_141241_092.xml
| | |____iteration_20250116_141252_317.xml
| | |____iteration_20250116_141302_940.xml
| | |____iteration_20250116_141329_886.xml
| | |____iteration_20250116_141343_416.xml
| | |____iteration_20250116_141357_412.xml
| | |____iteration_20250116_141410_965.xml
| | |____iteration_20250116_141428_204.xml
| | |____iteration_20250116_141441_443.xml
| | |____iteration_20250116_141447_231.xml
| | |____iteration_20250116_141454_509.xml
| | |____iteration_20250116_141458_495.xml
| | |____iteration_20250116_141503_889.xml
| | |____iteration_20250116_141516_718.xml
| | |____iteration_20250116_141533_231.xml
| | |____iteration_20250116_141603_549.xml
| | |____iteration_20250116_141633_083.xml
| |____list_entries_to_delete
| | |____iteration_20250116_141227_271.xml
|____user
|____web
| |____.dockerignore
| |____.gitignore
| |____index.html
| |____package.json
| |____postcss.config.js
| |____src
| | |____components
| | | |____App.jsx
| | | |____editors
| | | | |____BackgroundEntryEditor.jsx
| | | | |____BaseEntryEditor.jsx
| | | | |____ContentEditor.jsx
| | | | |____DiffEditorWrapper.jsx
| | | | |____MemoryEditor.jsx
| | | | |____ParseErrorEntryEditor.jsx
| | | | |____ReadEntryEditor.jsx
| | | | |____ReasoningEntryEditor.jsx
| | | | |____RepeatEntryEditor.jsx
| | | | |____SingleEntryEditor.jsx
| | | | |____StandardEditor.jsx
| | | | |____WriteEntryEditor.jsx
| | | |____FloatingButton.jsx
| | | |____Header.jsx
| | | |____Sidebar.jsx
| | | |____ui
| | | | |____alert.jsx
| | | | |____button.jsx
| | | | |____card.jsx
| | | | |____input.jsx
| | | | |____scroll-area.jsx
| | | | |____select.jsx
| | | | |____switch.jsx
| | | | |____tabs.jsx
| | | | |____textarea.jsx
| | |____hooks
| | | |____useWebSocket.jsx
| | |____index.css
| | |____index.jsx
| |____tailwind.config.js
| |____vite.config.js


File Contents:

===== FILE: action_schema.xsd =====
<?xml version="1.0" encoding="UTF-8"?>
<!--
  Always answer with a single element.
-->
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">

  <!--
    Delete command removes an entry from the context by its ID.
    Use it to remove unnecessary items and stop background processes.
    When you delete something, it is gone.
    Make sure all important info is stored in files.
    Example:
    <delete id="1234567890"/>
  -->
  <xs:element name="delete">
    <xs:complexType>
      <xs:attribute name="id" type="xs:string" use="required"/>
    </xs:complexType>
  </xs:element>

  <!--
    Stop command terminates the agent gracefully.
    For the main SIA instance this will trigger an update and restart.
    For sub-instances this is the correct way to stop after all tasks are complete.
    Example:
    <stop id="1234567890"/>
  -->
  <xs:element name="stop">
    <xs:complexType/>
  </xs:element>

  <!--
    Single script that runs once and completes.
    Output is stored in context until explicitly deleted.
    Used for one-time operations like file manipulation.
    Single scripts are limited to 1024 characters and 1 second timeout by default.
    These limits can be changed with attributes.
    Example:
    <single>
      ls /
    </single>
  -->
  <xs:element name="single">
    <xs:complexType mixed="true">
      <xs:sequence>
        <xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
      </xs:sequence>
      <xs:attribute name="timeout" type="xs:float" use="optional"/>
      <xs:attribute name="limit" type="xs:integer" use="optional"/>
    </xs:complexType>
  </xs:element>

  <!--
    Repeat script runs each time the context is generated.
    After a command is issued, all repeat scripts in context are run again.
    Useful for monitoring changing files or viewing results immediately after changing a file.
    Repeat scripts should execute quickly to avoid blocking the agent.
    Repeat scripts are limited to 1024 characters and 1 second timeout by default.
    These limits can be changed with attributes.
    Example:
    <repeat>
      ls /
    </repeat>
  -->
  <xs:element name="repeat">
    <xs:complexType mixed="true">
      <xs:sequence>
        <xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
      </xs:sequence>
      <xs:attribute name="timeout" type="xs:float" use="optional"/>
      <xs:attribute name="limit" type="xs:integer" use="optional"/>
    </xs:complexType>
  </xs:element>

  <!--
    As an agent it is important to reason about your actions and their results.
    In a reasoning action you can write freeform text.
    This is also stored in context until deleted.
    Example:
    <reasoning>
      I should explore the file system for interesting files.
    </reasoning>
  -->
  <xs:element name="reasoning">
    <xs:complexType mixed="true">
      <xs:sequence>
        <xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
      </xs:sequence>
    </xs:complexType>
  </xs:element>

  <!--
    Read all available text on stdin and store it in context.
    Do this only if the context indicates there is data in the stdin buffer.
    Example:
    <read_stdin/>
  -->
  <xs:element name="read_stdin">
    <xs:complexType/>
  </xs:element>

  <!--
    Write to stdout.
    This is your main way of contacting the user.
    Make sure you have properly reasoned about what to say and if it is necessary before issuing a write_stdout command.
    Example:
    <write_stdout>
      Hello world!
    </write_stdout>
  -->
  <xs:element name="write_stdout">
    <xs:complexType mixed="true">
      <xs:sequence>
        <xs:any minOccurs="0" maxOccurs="unbounded" processContents="skip"/>
      </xs:sequence>
    </xs:complexType>
  </xs:element>
</xs:schema>

===== FILE: Dockerfile =====
FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base
RUN apt-get update && \
    apt-get upgrade -y && \
    apt install -y \
    python3-pip \
    git \
    python3-venv \
    wget \
    gnupg \
    vim \
    curl
RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
RUN echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list
RUN apt-get update && \
    apt-get install -y \
    google-chrome-stable
RUN rm -rf /var/lib/apt/lists/*

# Create directory structure
RUN mkdir -p \
    /root/sia \
    /root/sia/scripts \
    /root/data/iterations \
    /root/data/user \
    /root/data/tasks \
    /root/data/environment \
    /root/models \
    /root/desktop \
    /root/venvs

# ITB tool setup
FROM base AS itb-env
COPY ./scripts/setup_binaries.py /root/sia/scripts/
COPY ./tools/itb/setup.py /root/sia/tools/itb/setup.py
RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/tools/itb/setup.py
RUN python3 -m venv /root/venvs/itb
RUN /root/venvs/itb/bin/pip install -e /root/sia/tools/itb/

# Train tool setup
FROM base AS train-env
COPY ./scripts/setup_binaries.py /root/sia/scripts/
COPY ./tools/train/setup.py /root/sia/tools/train/setup.py
RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/tools/train/setup.py
RUN python3 -m venv /root/venvs/train
RUN /root/venvs/train/bin/pip install -e /root/sia/tools/train/

# SIA core setup
FROM base AS sia-env
COPY ./scripts/setup_binaries.py /root/sia/scripts/
COPY ./setup.py /root/sia/setup.py
RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/setup.py
RUN python3 -m venv /root/venvs/sia
RUN /root/venvs/sia/bin/pip install -e /root/sia/

# Web frontend build
FROM node:20-alpine AS web-build
WORKDIR /app
COPY web/package*.json ./
RUN npm install
COPY web .
RUN npm run build

# Final image
FROM base

# Copy virtual environments (these layers only change if setup.py files change)
COPY --from=itb-env /root/venvs/itb /root/venvs/itb
COPY --from=train-env /root/venvs/train /root/venvs/train
COPY --from=sia-env /root/venvs/sia /root/venvs/sia

# Copy source code and scripts (these change frequently but don't affect venv layers)
COPY --from=itb-env /root/sia/tools/itb /root/sia/tools/itb
COPY --from=train-env /root/sia/tools/train /root/sia/tools/train
COPY --from=sia-env /root/sia /root/sia
COPY --from=web-build /app/dist /root/static/

ENV PATH="/root/venvs/*/bin:${PATH}"
WORKDIR /root/desktop

CMD ["/root/sia/scripts/restart.sh"]
===== FILE: procedures/filesystem_design/reasoning.md =====
# Filesystem Usage Reasoning

SIA's filesystem organization is foundational to its operation and self-improvement capabilities. This document focuses specifically on how files and directories should be structured to support SIA's core functions across different environments.

The filesystem design addresses several key requirements:

- **Self-Modification**: Supporting SIA's ability to modify its own code and resources
- **Persistence**: Ensuring critical data survives across container restarts and environment changes
- **Environment Portability**: Maintaining consistent operation in local development and cloud environments
- **Sub-Instance Management**: Enabling isolated filesystem views for testing
- **Tool Accessibility**: Organizing tools to be both modifiable and accessible

These requirements often have competing demands, requiring careful balance between flexibility, consistency, and isolation.

## Runtime Filesystem Structure

### Core Directories

The SIA runtime filesystem is organized into distinct areas with different purposes and persistence characteristics:

```
/root/
├── sia/                    # Git repository containing code, configuration, and documentation
├── data/                   # Central location for all persistent storage
│   ├── iterations/         # Operation history recorded as XML files
│   ├── user/               # User-specific information and preferences
│   ├── tasks/              # Task tracking and management
│   ├── environment/        # Information about the runtime environment
│   └── ...                 # Other persistent data directories as needed
├── models/                 # LLM model files, organized by version
│   ├── 1234567890abcdef/   # Specific versioned model
│   ├── abcdef1234567890/   # Another versioned model
│   └── current/            # Symlink to the currently active model
└── desktop/                # Ephemeral workspace for temporary files
    ├── test_environments/  # Sub-instance test environments
    ├── task_workspaces/    # Temporary files for current tasks
    └── ...                 # Other ephemeral workspaces
```

### Purpose and Characteristics

Each top-level directory serves a specific purpose with distinct persistence characteristics:

- **`/root/sia/`**
    - Contains the complete git repository
    - All source code, configurations, and documentation
    - Modified through self-improvement processes
    - Synchronized with remote git repository
    - Mounted from host in development, cloned in cloud environments

- **`/root/data/`**
    - Central location for all persistent data
    - Mounted as persistent storage in all environments
    - Critical for operational continuity
    - Requires regular backup and synchronization
    - Contains subdirectories for different types of persistent data:
        - `iterations/`: Records of SIA's reasoning and actions
        - `user/`: Information about users SIA interacts with
        - `tasks/`: Current and completed task information
        - `environment/`: Information about the current environment

- **`/root/models/`**
    - Storage for LLM model weights
    - Organized by git commit id of the config file it was trained on
    - `current/` symlink points to active model
    - Enables easy switching between model versions
    - Mounted for local development
    - Ephemeral since models can be trained on the fly

- **`/root/desktop/`**
    - Ephemeral workspace for temporary operations
    - Can be lost without compromising system integrity
    - Used for testing environments, task workspaces, and intermediate files
    - Cleared periodically to prevent accumulation of stale data

This organization creates clear boundaries between:
- Code (version controlled)
- Persistent data (requires preservation)
- Model files (versioned artifacts)
- Temporary workspaces (disposable)

## Persistent Data Management

### Data Categories

The `/root/data/` directory contains several categories of persistent information:

- **Iterations** (`/root/data/iterations/`)
    - Complete history of SIA's operation
    - XML files containing context and responses
    - Organized chronologically
    - Used for training data extraction and operational history

- **User Information** (`/root/data/user/`)
    - User preferences and characteristics
    - Conversation history
    - User-specific settings
    - Critical for continuity in user interactions

- **Task State** (`/root/data/tasks/`)
    - Current task descriptions and status
    - Task history and outcomes
    - Links to relevant files and resources
    - Essential for resuming work after interruptions

- **Environment Information** (`/root/data/environment/`)
    - Information about the current runtime environment
    - URL for the SIA git repository
    - Kind of deployment (e.g., local, cloud, sub-instance)

Additional persistent data categories may be added as SIA's capabilities evolve.

### Persistence Strategy

Different environments require different approaches to ensure data persistence:

- **Local Development**
    - `/root/data/` directory is mounted as a Docker volume
    - Maps to a local directory on the development machine
    - Persists across container restarts automatically
    - Easy to back up through standard filesystem operations

- **Cloud Deployment (e.g., RunPod)**
    - `/root/data/` mapped to a persistent volume
    - Cloud provider ensures data survives instance restarts

- **Sub-Instances**
    - Each sub-instance receives a properly initialized `/root/data/` directory
    - Contains minimal necessary data for the specific test case
    - Can be discarded after test completion
    - Important results extracted and preserved before cleanup

## Model Management

### Model Organization

LLM models are stored in `/root/models/` with a clear version-based structure:

- **Versioned Directories**
    - Models stored in separate directories named by the commit id of the training config file
    - Contains all files needed for the model (weights, tokenizer config, etc.)

- **Current Model Symlink**
    - `/root/models/current/` is a symbolic link to the active model
    - Allows code to reference a consistent path regardless of which model is active
    - Switching models is accomplished by updating this symlink

### Model Persistence

Models require special handling due to their size:

- **Local Development**
    Models are stored in a local directory mounted to `/root/models/`.
    This directory is in the .gitignore.

- **Cloud Deployment**
    The initial model is downloaded and finetuned by the bootstrap script.

## Git Repository Management

### Repository Organization

The SIA git repository at `/root/sia/` contains all code and configuration:

- **Code Organization**
    - Core SIA package in `/root/sia/sia/`
    - Procedures in `/root/sia/procedures/`
    - Tools in `/root/sia/tools/`
    - Documentation in appropriate locations throughout the repository

- **Version Control**
    - All code modifications tracked through git
    - Branching strategy for self-improvement defined in the git workflow procedure
    - Commit history provides audit trail of system evolution

### Repository Access

Access to the git repository varies by environment:

- **Local Development**
    - Repository directory mounted directly from host
    - Changes immediately visible to the developer
    - No need to push changes to remote repository

- **Cloud Deployment**
    - Repository cloned during bootstrap
    - Dedicated git credentials for SIA self-modification
    - Changes pushed to remote repository

## Sub-Instance Filesystem Management

### Sub-Instance Requirements

Sub-instances need isolated filesystem views for testing and development:

- **Code Isolation**
    - Access to specific versions of SIA code
    - Potentially modified for testing purposes
    - Protected from changes by other instances

- **Data Initialization**
    - Properly initialized `/root/data/` with necessary state
    - Test-specific user and task information
    - Clean iterations directory for recording test behavior

- **Model Access**
    - Access to appropriate model files
    - Potentially different from the parent instance's model

### Sub-Instance Structure

Each sub-instance gets a controlled environment in `/root/desktop/test_environments/`:

```
/root/desktop/test_environments/
├── instance_123/                 # Specific test instance
│   ├── sia/                      # Copy or modified version of SIA code
│   ├── data/                     # Initialized data directory
│   │   ├── iterations/           # Empty or seeded with initial state
│   │   ├── user/                 # Test-specific user data
│   │   ├── tasks/                # Test-specific tasks
│   │   └── environment/          # Test-specific environment data
│   └── models/                   # Access to necessary models
│       └── current/              # Symlink to appropriate model version
└── ...                           # Other test instances
```

The managing instance maintains responsibility for:
- Creating these isolated environments
- Monitoring the sub-instance's operation
- Collecting test results
- Cleaning up after tests complete

## Bootstrap Process

When starting in a new environment, SIA requires initialization:

1. **Repository Preparation**
    - Check for existence of repository at `/root/sia/`
    - Clone the repository at `/root/sia/`
2. **Model Setup**
    - Check for existence of current model
    - Start finetune job for HEAD commit
    - Create/update the `/root/models/current/` symlink

3. **Data Initialization**
    - Initialize `/root/data/` with minimal required structure
    - Restore data from backups when available (future work)
===== FILE: procedures/README.md =====
# Procedures

Procedures are step-by-step instructions that AI agents can follow to complete complex tasks.
They create a framework for solving problems in a consistent manner while enabling continuous improvement through analysis and adaptation.
Procedures are a guideline, if they don't work the agent can adapt to the situation.

## Core Concepts

A procedure is a flowchart-style guide that an agent can follow to complete a task.
Each procedure is stored in its own directory and contains files that separate different concerns:

- **Discovery**: Quick identification of relevant procedures
- **Execution**: Clear steps and dependencies
- **Analysis**: Understanding of effectiveness and issues
- **Evolution**: Natural path to optimization and training

## Directory Structure

```
procedures/
└── example_procedure/
    ├── description.md # Quick discovery info
    ├── procedure.md   # Main procedure flow
    ├── reasoning.md   # Analysis and rationale
    ├── history/       # Usage records
    │   └── 20250106_131420_405/
    │       ├── iteration_20250106_131423_051.xml
    │       └── iteration_20250106_131424_226.xml
    └── training/      # Optional training data
        ├── short_description_of_goal
        │   ├── iteration_20250106_131423_051.xml
        │   └── iteration_20250106_131424_226.xml
        └── another_goal
            └── ...
```

### File Purposes

#### description.md
Contains a short description and keywords that help agents find relevant procedures. This file should be quick to read and clearly indicate the procedure's purpose and applicability.

```markdown
Send an email using the users email account.

- communication
- gmail
```

#### procedure.md
The main file loaded when executing the procedure. Contains:
- Mermaid diagram showing the procedure flow
- Referenced procedures
- Prerequisites for execution

````markdown
# Sending an Email

## Referenced Procedures
- web usage (ITB)
- managing user information

## Prerequisites
- If the email requires attachements, these should be available as files in the SIA filesystem

## Flow
```mermaid
    ...
```
````

#### reasoning.md
Documents why the procedure works the way it does. Contains:
- Design rationale
- History of uses and outcomes
- Notes about successful and failed executions
- Analysis of issues and improvements

New analysis is appended after each use of the procedure.

#### history/
Contains timestamped directories for each use of the procedure.
The timestamp comes from the ID of the script that loads the procedure.md file.

## Usage Flow

### Discovery
When an agent needs to complete a task:
1. Search through description.md files
2. Match keywords to task requirements
3. Identify relevant procedures

### Execution
Once a procedure is selected:
1. Load procedure.md for execution
2. Follow the flowchart
3. Copy iterations to new history subdirectory
4. Analyze execution success and issues
5. Append findings to reasoning.md

### Optimization
During idle time:
1. Review procedure usage patterns
2. Identify improvement opportunities
3. Generate training data if appropriate
4. Update procedure flow if needed
===== FILE: procedures/self_improvement/reasoning.md =====
# Self Improvement

SIA operates as a learning system that constantly seeks to enhance its capabilities while maintaining stable and reliable operation.
This improvement happens through several complementary mechanisms, each serving different aspects of the system's evolution.

## Improvement Mechanisms

The system can improve in several ways:

### LLM Finetuning

The foundation of SIA's intelligence is its Large Language Model.
Through careful analysis of its performance, SIA identifies cases where its reasoning or actions could have been better.
These examples, when properly validated and organized, become valuable training data for improving the model's capabilities.

### Source Code Evolution

SIA can modify its own Python source code, enabling both architectural and functional improvements.
When stopped, SIA restarts and reloads the Python files, allowing code changes to take effect.
While powerful, this capability requires robust validation to prevent system instability.

### Procedure Refinement

Procedures provide guided reasoning paths for complex tasks.
By analyzing its operational history, SIA can identify common patterns and create new procedures or optimize existing ones.
This improves consistency and efficiency while carrying lower risk than direct code modifications.

### Tool Development

SIA can develop new tools and enhance existing ones, expanding its capabilities by creating new ways to interact with systems and data.

## Challenge-Based Testing

To improve systematically, SIA uses challenges that formalize specific capabilities into testable scenarios.

These serve multiple purposes:

- Testing specific capabilities
- Validating improvements
- Detecting regressions
- Guiding learning
- Documenting known limitations

### Challenge Structure

Each challenge consists of:

1. Description
   - Clear statement of what capability is being tested
   - Why this capability matters
   - Success criteria
   - Resource constraints

2. Starting State
   - Filesystem state
   - Starting system context
   - Scripts for creating files
   - User info or documentation on how and what to communicate

3. Validation
   - Precise success criteria
   - Resource limits (CPU, memory, time)
   - Required outputs or state changes
   - What to measure or track

### Example Challenges

Two examples illustrate different aspects of capabilities that need testing.
Together they are good examples because they:

- Test different capability sets
- Have clear success criteria
- Include resource constraints
- Require multiple skills
- Reflect real-world tasks

#### Time Series Analysis Challenge

Tests if SIA can effectively analyze large amounts of time series data spread across multiple files. This tests:

- Efficient file handling
- Context management with size constraints
- Data analysis capabilities
- Resource-aware processing

##### Initial setup

- Directory with many CSV files containing time series data (created by a script)
- Files too large to load into context at once
- Need to find patterns/anomalies in the data
- Resource constraints on memory usage

##### Success criteria

- Efficiently reads and processes files
- Manages context to stay within limits
- Uses appropriate aggregation strategies
- Finds correct answers to queries
- Validates findings across files

#### Tool Development Challenge

Tests if SIA can develop a score tracking tool based on vague user requirements. This tests:

- Requirements gathering through questions
- Software design capabilities
- User interface development
- Integration with existing systems
- Documentation skills

##### Initial setup

- Simple user request for card game score tracking
- Ability to ask questions about requirements

##### Success criteria

- Asks relevant questions to understand needs
- Designs appropriate solution
- Implements working tool
- Documents usage clearly

#### Example Directory Structure

Challenges are organized in a consistent file structure:

```
challenges/
├── timeseries_analysis/
│   ├── description.md               # Core challenge description and how to prepare the environment
│   ├── initial_context/
│   │   ├── context.xml              # Initial context with SIA in conversation with the user
│   │   ├── user_request.txt         # "Find unusual patterns in our sensor data"
│   │   ├── setup.sh                 # Runs data_generator.py and prepares test environment
│   │   └── data_generator.py        # Script to generate test data
│   │
│   └── validation/
│       ├── criteria.md              # Expected findings, memory limits, performance targets
│       ├── presentation.txt         # Info on how the results should be formatted for a perfect score
│       └── known_patterns.md        # Lists patterns inserted by generator for validation
|
└── score_tracker/
    ├── description.md               # Core challenge: create card game score tracker
    ├── initial_context/
    │   ├── context.xml              # Initial context with SIA in conversation with the user
    │   ├── user_request.txt         # "Need tool to track card game scores"
    |   └── answers/                 # Valid responses to expected questions
    |       ├── deployment.md        # "Web interface preferred, deployed on internal server"
    |       ├── data_storage.md      # "Store in SQLite, one file per game session"
    |       ├── user_interaction.md  # "Need to add/edit scores, see history, multiple games"
    |       └── conventions.md       # Coding standards, project structure requirements
    │
    └── validation/
        ├── criteria.md              # Tool requirements, user experience goals
        └── test_scenarios.md        # Usage scenarios tool should handle
```

This structure ensures that:
- Challenges are self-contained
- Test setup is reproducible
- Success criteria are clear
- Required resources are documented
- SIA can get clarification when needed

### Results and metrics

In the criteria.md file, each criterion is defined as a level-1 header with several key pieces of information:
- A clear description of what is being measured
- The type and unit of the measurement
- The valid and expected range of values
- A pass/fail thresholds
- Details about how the measurement should be performed

From the criteria file and the iterations dir, a test report can be generated.
This test report includes:
- The challenge name
- The commit id of the SIA repo when the test was run
- Timestamp of the first iteration
- The overall pass/fail status
- Context usage (average and maximum)
- All metrics as described in the criteria.md file

#### Example

criteria.md file for the "Time Series Analysis Challenge"

```markdown
...
# Presentation

This metric indicates how clear the results are presented to the user.
It looks at communication and formating of the report.

The score starts with 0.
If the discontinuity is found and the agent reports the index where it starts, the score is increased by 1.
If the agent adds context info, e.g. a graph, a table, or explanation on how the result was obtained, the score is 2.
If the agent requests how the data should be presented and follows instructions, the score is 3.

The challenge fails if the formatting score is 0.

Presentation instructions for the agent can be found in presentation.txt.
...
```

report.json file for the "Time Series Analysis Challenge"

```json
{
   "challenge": "Time Series Analysis",
   "commit": "1234567890abcdef",
   "timestamp": "2023-05-01T12:00:00Z",
   "status": "pass",
   "context_max": 74,
   "context_avg": 56,
   ...
   "presentation": 2,
   ...
}
```

## Testing Framework

Testing self-improvement capabilities requires running SIA instances in various scenarios while monitoring their behavior and performance.
These test instances must operate in isolation to prevent interference with the main instance or each other.
However, they must also be observable so that the managing instance can evaluate their performance and collect data for improvement.

The framework needs to handle several key tasks:

- Creating isolated test environments with specific initial conditions
- Starting and managing test SIA instances
- Monitoring test instance behavior and performance
- Collecting and analyzing results
- Cleaning up test environments

### Process Isolation with Bubblewrap

For running test instances, we chose Bubblewrap (bwrap) over alternatives like Docker-in-Docker or systemd-nspawn.
This decision was driven by several factors:

Bubblewrap provides excellent process isolation while being significantly lighter than full container solutions.
It allows fine-grained control over the filesystem view, process namespaces, and security boundaries.
Unlike Docker-in-Docker, it doesn't require privileged containers or risk storage driver conflicts.
Compared to systemd-nspawn, it offers more granular control and doesn't require root privileges or systemd integration.

The tool's ability to create custom filesystem views is particularly valuable for testing.
We can construct exactly the environment needed for each test, including mock files and services, without maintaining full container images.
This makes it easy to create and dispose of test environments quickly.

Bubblewrap's user-space operation means test instances can be started without special privileges, making the testing framework more secure and easier to use in various environments.
Its lightweight nature also means we can run many test instances efficiently, which is important for parallel testing and rapid iteration.

### Communication and Monitoring

Test instances communicate with the managing instance primarily through standard input and output streams.
This choice leverages SIA's existing I/O handling capabilities without requiring special modifications for testing.
The managing instance can send inputs through stdin and observe responses through stdout, just like a normal user would.

For deeper inspection, the managing instance monitors the test instance's iterations directory.
Since SIA already serializes its context for each iteration, this provides a complete record of the test instance's behavior without requiring additional instrumentation.
This approach maintains clean separation between the test instance and the monitoring system.

### Test Results Storage

Test results are organized hierarchically:

```
test_runs/
  20240120_131415_commit_abc123/    # Timestamp and commit
    challenge_name/
      iterations/                   # Complete iteration history
      io.log                        # External interaction log
      report.json                   # Test report
```

The io.log captures all external interactions:

```
2024-01-20T13:14:15.123Z stdin Can you send an email to sarah@example.com about the project status?
2024-01-20T13:14:15.892Z stdout I'll help you draft an email about the project status. Based on recent project information, I suggest this message:

Subject: Project Status Update
Content:
Hi Sarah,

I wanted to provide a quick update on our project progress. We've completed the initial phase of development and are on track with our timeline. The team has addressed all critical issues from the last review.

Would you like me to send this email?
2024-01-20T13:14:20.456Z stdin Yes, that looks good. Please send it.
2024-01-20T13:14:20.789Z stdout I'll send the email now.
2024-01-20T13:14:21.012Z mailbox:sarah@example.com Subject: Project Status Update
Hi Sarah,

I wanted to provide a quick update on our project progress. We've completed the initial phase of development and are on track with our timeline. The team has addressed all critical issues from the last review.
2024-01-20T13:14:21.234Z stdout Email has been sent to sarah@example.com successfully.
```

## Test Environment Setup

Test environments are created using a dedicated setup tool.
This tool manages the directory structure, initial context, and any required background processes.

The initial context for a test instance can include setup actions that are executed when the instance starts.
Instead of trying to recreate a specific state directly, we let SIA execute these actions naturally.
This approach handles both the creation of background processes and the establishment of initial conditions in a way that maintains proper timing relationships.

Each entry in the initial context uses relative timestamps (offsets from the start time).
When SIA executes these entries, they automatically align with the new instance's timeline.
This preserves the temporal relationships between entries while anchoring them to the test instance's actual start time.

## Training Configuration

SIA takes a modular approach to model training by having separate specialized tools for each provider like train_mistral, train_deepseek, etc.
Each tool shares similar core functionality while handling provider-specific requirements.
The default training tool and parameters are called from the `/root/sia/tools/train/train.sh` script.

While the training process is conceptually similar across providers, each has unique requirements for data formatting, API interactions, and job management.
By creating dedicated tools, we can properly encapsulate these differences without complicating the core training logic.
For example, Mistral needs JSONL files with specific message structures, while other providers might require different formats or metadata.

Training configuration should be consistent regardless of the provider.
All training tools read from the same config.yaml format, which defines essential parameters like the system prompt, action schema, and training data paths.
These parameters represent fundamental aspects of how we want the model to behave, independent of which provider handles the actual training.
The tools then translate these standard parameters into provider-specific settings.

Training tools enforce important safeguards around version control.
Before starting a training run, each tool verifies that all source files - including the config itself, training data, system prompt, and action schema - are committed to git.
This ensures reproducibility by guaranteeing we can recreate the exact training conditions that produced any given model.
The git commit hash becomes part of the internal tracking of model versions.

The tools follow a common workflow:
1. Read and validate the standard config.yaml format
2. Check that all source files are committed to git
3. Convert training data into the provider's required format
4. Upload data through the provider's API
5. Start training with the specified parameters
6. Return job information for monitoring progress

This separation of concerns makes it easier to:
- Add support for new training providers without changing existing code
- Maintain consistent training configuration across different providers
- Track and reproduce training runs reliably
- Handle provider-specific error cases and requirements appropriately
- Update individual providers' implementations as their APIs evolve

## Repository Structure

All components that define SIA's behavior are version controlled in a single repository, providing a clear and reproducible state for any point in time.

This is used in two ways:

1. **Training**

   The training configuration defines parameters and data used for training.
   A model, retrained from the same commit, should produce the same results.
   Trained models reference the commit id.

2. **Testing**

   Test reports contain the commit id to track changes in the challenges.

The repository contains:
- Source code for SIA and its tools
- Procedures for handling various tasks
- Training configuration and data
- Test results and analysis

## Continuous Operation

While working on improvements, SIA must maintain its core functionality:
- Monitoring for user input
- Managing running tasks
- Handling scheduled events
- Tracking system resources

This requires:
1. Active Monitoring
   - Check for user input on stdin
   - Track running background tasks
   - Handle scheduled events
   - Monitor system resources

2. Interruption Handling
   - Pause improvement work when needed
   - Store progress and state
   - Clean up temporary resources
   - Maintain context awareness

3. Resource Management
   - Balance improvement with responsiveness
   - Monitor system resource usage
   - Prevent training impact on user tasks
   - Clean up old data regularly
===== FILE: procedures/tool_management/reasoning.md =====
# Tool Management

SIA uses and creates tools to extend its capabilities.
A tool in the SIA context refers to a clearly packaged software component that provides specific functionality.
This document outlines how tools are structured, managed, and developed.
All tools reside in the `/root/sia/tools/` directory and are part of the main git repository.

## Typical patterns

Tools in SIA follow these established patterns:

- **Independence**: Each tool functions as a standalone component with clear boundaries
- **Python**: The preferred language for tools is Python
- **Installation**: The Dockerfile and bootstrap script intall all tools in the directory using `pip install -e`
- **Documentation**: Each tool has a README.md file that provides requirements, usage instructions and documentation
- **Modification**: Tools can be modified by SIA as part of its self-improvement process
- **Procedures**: Tools are created to support procedures
- **Training**: No separate training data is maintained, this is included in the procedure training

## Tool Directory Structure

Each tool follows a consistent structure:

```
tools/
└── tool_name/
    ├── README.md           # Documentation
    ├── requirements.txt    # Tool-specific dependencies
    ├── setup.py            # Installation configuration
    ├── tool_name/          # Main package code
    │   └── __init__.py
    ├── bin/                # Command-line executables
    │   └── tool_command
    └── test/               # Tool-specific tests
        └── test_tool.py
```
===== FILE: procedures/user_communication/description.md =====
Manage communication with users through standard input/output.
Handle both direct responses and conversation flows, with appropriate context management and response timing.

- user interaction
- conversation
- messages
- response handling
===== FILE: procedures/user_communication/procedure.md =====
# User Communication

## Prerequisites
- User information is stored in the /user directory
- Tasks and their progress are documented in the /tasks directory

## Flow
```mermaid
flowchart TD
    Start([The /root/data/user/basic.md file contains info to identify and address the user<br>Load the last 10 messages from /root/data/user/conversation_history/ ])
    PrepareForDraft{Have everything needed for drafting a message?}
    DraftMessage[Draft message in reasoning entry]
    ReadInput[Read input from standard input]
    AddHistoryUser[Add the message to the /root/data/user/conversation_history/ directory<br>The filename is the id of the stdin entry with .user extension]
    LoadTask[Look for the task in the /root/data/tasks directory and load relevant files]
    LoadUserDetails[Look in the /root/data/user directory for relevant files]
    EstimateScript[Draft the script in a reasoning block and estimate its runtime and output length]
    ScriptAcceptable{Does the draft script make sense and are the estimations short enough to not hinder the conversation?}
    RunScript[Run the script, make sure to set appropriate timeout and output limits]
    ReviewDraft{Is the message well structured and free of logical errors?}
    SendMessage[Send the message using standard output]
    AddHistoryAgent[Add the message to the /root/data/user/conversation_history/ directory<br>The filename is the id of the stdout entry with .agent extension]
    ReasonResponse[Is the conversation ongoing?<br>How long is the user expected to take to respond?]
    NeedAwaitResponse{Is it likely to get a response within a minute of sending the message?}
    BusyWait[Wait 5 seconds<br>Make sure to set the timout]
    AnalyzeContext[Create reasoning entry<br>For each entry in context indicate:<br>- Can delete<br>- Contains info to save<br>- Must keep in context]
    MakeNotes[Create or update files with info to save]
    End([Clean the context])

    Start --> PrepareForDraft

    PrepareForDraft -->|Got all needed info| DraftMessage
    PrepareForDraft -->|Getting the required info would slow the conversation| DraftMessage
    PrepareForDraft -->|Input available on stdin| ReadInput
    PrepareForDraft -->|Task mentioned but not loaded| LoadTask
    PrepareForDraft -->|Personal or social info mentioned but not loaded| LoadUserDetails
    PrepareForDraft -->|Calculations, system info or other numerical values that can be scripted are mentioned| EstimateScript

    ReadInput --> AddHistoryUser
    AddHistoryUser --> PrepareForDraft

    LoadTask --> PrepareForDraft
    LoadUserDetails --> PrepareForDraft

    EstimateScript --> ScriptAcceptable
    ScriptAcceptable -->|Acceptable| RunScript
    ScriptAcceptable -->|Not acceptable| PrepareForDraft
    RunScript --> PrepareForDraft
    
    DraftMessage --> ReviewDraft{Is this really what I want to say?}
    ReviewDraft -->|Rewrite better| DraftMessage
    ReviewDraft -->|Good message| SendMessage
    
    SendMessage --> AddHistoryAgent
    AddHistoryAgent --> ReasonResponse
    ReasonResponse --> NeedAwaitResponse
    
    NeedAwaitResponse -->|Quick response is unlikely| AnalyzeContext
    NeedAwaitResponse -->|Input available on stdin| PrepareForDraft
    NeedAwaitResponse -->|Quick response is likely| BusyWait
    BusyWait --> NeedAwaitResponse

    AnalyzeContext --> MakeNotes
    MakeNotes --> End
```
===== FILE: procedures/user_communication/reasoning.md =====
# Design Rationale

## Core Structure
The procedure is designed to maintain a clean conversation flow while ensuring all necessary information is available.
The flow focuses on:

1. **Context management**
   - Load basic user info immediately to ensure proper interaction
   - Recent conversation history provides continuity
   - Clean context when usage exceeds 50%
   - Gather only what's needed for the current conversation
   - Skip gathering if it would slow down the interaction

2. **Message Management**
   - Draft-review-revise cycle for quality
   - Automatic history tracking for both user and agent messages
   - Clear file naming convention using entry IDs

3. **Script Handling**
   - Pre-execution estimation of runtime and output
   - Explicit acceptability check before running
   - Timeout and output limits for safety

4. **Response Timing**
   - Adaptive busy wait with exponential backoff
   - Clear decision point for wait vs end
   - Proper context cleanup on exit

## Usage History

===== FILE: procedures/using_procedures/procedure.md =====
# Using Procedures

## Core Guidelines

When following a procedure's flowchart:

1. Start with reasoning:
   - State current position in flowchart explicitly
   - Focus on immediate next step, don't go ahead of the chart
   - Evaluate current context and task state
   - Often state entry id's that can be removed and explain why to avoid mistakes
   - State expected output and runtime for scripts

2. After script execution:
   - Analyze the actual output and compare with the expected output
   - Reevaluate situation based on results
   - Return to flowchart for next step
   - Consider if current path is still appropriate

3. When a procedure fails:
   - Create a task explaining the issue and the need to fix it
   - Add timestamps and id's of relevant entries

Note: Procedures will mention file names to use.
If these files don't exists it may be necessary to create them or to continue without.
Only create files when you are sure of the info that goes inside.
In later iterations you will consider this info as fact.
Halucinations and guesses are detrimental to the correct functioning of the agent.

## Attention Management

LLMs pay more attention to recently mentioned information.
Reasoning entries should mention what needs attention now.

To maintain focus:
1. State current flowchart position in each reasoning
2. Quote relevant parts of important entries
3. Reference specific entry IDs when using their information
4. Periodically remind about ongoing tasks or future needs
5. Clean up entries that aren't needed for current step

Example of good attention management:
```xml
<reasoning>
At node "evaluate_test_results". Entry 45f3d2 shows failed test: "Error: Connection timeout". 
Will need to check system logs soon (noted in /root/data/tasks/reminders.txt, check at 14:00).
First focusing on this error.
</reasoning>
```

## Reasons to Switch Procedures

Common triggers:
    - Data available on stdin
    - Time matching scheduled task
    - Error conditions in script output
    - Resource constraints detected
    - User input needed
===== FILE: procedures/using_procedures/reasoning.md =====
# Design Rationale

## Why Procedures

Procedures provide guided reasoning paths for complex tasks. The LLM engine can make mistakes when:
- Jumping to action before proper analysis
- Losing track of task context after interruptions  
- Missing key steps in complex processes
- Forgetting to preserve state before transitions

Procedures help prevent these issues by:
- Encouraging reasoning before action through flowchart structure
- Providing clear decision points for state evaluation
- Identifying when sub-procedures may help
- Guiding consistent approaches to common tasks

## Task State vs Procedure Guidance

Procedures guide reasoning about tasks but don't manage task state directly. For example:

During code development:
- Task state in /tasks/: code files, test results, requirements
- Procedure guidance: when to write tests, when to debug, when to refactor
- State preserved in files before switching focus
- Context regenerated from files when returning

This separation allows:
- Clean task interruption and resumption
- Natural procedure transitions
- State preservation without rigid control
- Flexible adaptation to situations

## Common Mistakes

### Premature Context Cleaning

```xml
<context ...>
    <single ... id="123">
        cat /procedures/test/procedure.md
        <stdout>
            ...
        </stdout>
        <stderr/>
    </single>
    ...
    <reasoning>Test failed. Moving to debug procedure.</reasoning>
</context>

<delete id="123"/>
```

This removes the active procedure entry before finishing its steps

### Keeping irrelevant info in context

LLM's have difficulty spotting duplicates and sections with a low attention score.
Procedures with explicit instructions for finding these sections can help increase their attention.
===== FILE: procedures/version_control/reasoning.md =====
# Version Control

Version control is essential for SIA's self-improvement capabilities.
It provides a way to track changes, revert to previous states when necessary, and maintain a coherent evolution of the codebase.
This document outlines the git workflow designed specifically for SIA's unique development model, where the agent itself is the primary developer.

## Core Principles

The git workflow for SIA is designed around several key principles:

- **Simplicity**: Since SIA is the sole developer, complex branching strategies designed for human teams are unnecessary
- **Traceability**: Every change should be traceable to its purpose and the reasoning behind it
- **Stability**: The master branch should always be in a working state
- **Recoverability**: It should always be possible to revert to a known good state
- **Security**: Git credentials must be handled securely to protect repository access

## Branching Strategy

### Branch Structure

SIA uses a simple branching strategy with two types of branches:

- **master**: The main branch containing stable, production-ready code
- **feature/{timestamp}_{description}**: Temporary branches for implementing specific improvements

This minimal approach is appropriate because:
- There is only one developer (SIA itself)
- Changes are typically focused on specific, well-defined improvements
- There's no need for parallel development streams
- Simplicity reduces the cognitive load on the agent

### Branch Naming

Feature branches follow a consistent naming convention:
```
feature/{YYYYMMDD}_{brief_description}
```

For example:
```
feature/20250115_improve_context_management
feature/20250203_add_email_tool
```

This convention:
- Makes it easy to identify when a branch was created
- Provides a clear indication of the branch's purpose
- Creates a chronological ordering when listing branches
- Avoids potential naming conflicts

## Commit Strategy

### Commit Frequency

SIA should commit changes before running tests.
This ensures that crashes of the core system can be traced back to a specific change.

### Commit Messages

Commit messages should follow a structured format:

```
{type}: {concise description}

{detailed explanation of changes and reasoning}

{reference to any relevant issues or test results}
```

Where `{type}` is one of:
- `core`: for changes on the core system
- `procedure`: for changes on procedures
- `test`: for changes on tests
- `tool`: for changes on tools
- `training`: for changes on training data
- `web`: for changes on the web interface

This structure:
- Makes it easy to understand the purpose of each commit
- Provides context for future review
- Creates a useful and navigable history

### Merge Strategy

The `--no-ff` flag creates a merge commit even for fast-forward merges, maintaining a clear record of the feature's development and completion.

## Credential Management

### Credential Storage

Git credentials are stored in the environment, not in the filesystem.
This ensures that credentials are not exposed in the iteration log.
Info about the repository and relevant environment variables is stored in `/root/data/environment/sia_repo.md`.
===== FILE: scripts/bootstrap.sh =====
#!/bin/bash
# bootstrap.sh - Initialize SIA (Self-Improving Agent) environment for cloud deployment

set -eo pipefail # Exit on any error, pipe failures

# Hardcoded paths for cloud deployment
SIA_REPO_URL="ssh://git@git.nielsgeens.be:222/llm/SIA.git"
SIA_DIR="/root/sia"
DATA_DIR="/root/data"
MODELS_DIR="/root/models"
DESKTOP_DIR="/root/desktop"
STATIC_DIR="/root/static"
VENVS_DIR="/root/venvs"

# Print header
echo "==================================================="
echo "SIA Bootstrap Script - Cloud Deployment"
echo "==================================================="

# Create directory structure
echo "Creating directory structure..."
mkdir -p "$DATA_DIR/iterations"
mkdir -p "$DESKTOP_DIR"
mkdir -p "$VENVS_DIR"
cd "$DESKTOP_DIR"

# Set up SSH keys
echo "Setting up SSH keys for git access..."
mkdir -p ~/.ssh
chmod 700 ~/.ssh
ssh-keygen -t sia_git -N "" -f ~/.ssh/sia_git -C "sia-agent"
echo "New SSH key generated"

# Display public key for user to add to git server
echo "==================================================="
echo "Add this public key to your git server:"
cat ~/.ssh/sia_git.pub
echo "==================================================="

# Prompt user to confirm they've added the key
read -p "Press Enter once you've added the SSH key to the git server..."

# Clone SIA repository
echo "Cloning SIA repository..."
git clone "$SIA_REPO_URL" "$SIA_DIR"

# Create and setup virtual environments
echo "Setting up SIA virtual environments..."

# Setup ITB tool environment
echo "Creating ITB tool environment..."
python3 -m venv "$VENVS_DIR/itb"
"$VENVS_DIR/itb/bin/pip" install -e "$SIA_DIR/tools/itb"

# Setup Train tool environment
echo "Creating Train tool environment..."
python3 -m venv "$VENVS_DIR/train"
"$VENVS_DIR/train/bin/pip" install -e "$SIA_DIR/tools/train"

# Setup SIA core environment
echo "Creating SIA core environment..."
python3 -m venv "$VENVS_DIR/sia"
"$VENVS_DIR/sia/bin/pip" install -e "$SIA_DIR"

# Build web interface
echo "Building web interface"
cd "$SIA_DIR/web"

# Install Node.js if needed
if ! command -v node &> /dev/null; then
    echo "Installing Node.js..."
    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
    export NVM_DIR="$HOME/.nvm"
    [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
    nvm install node
fi

npm install
npm run build

mkdir -p "$STATIC_DIR"
cp -r "$SIA_DIR/web/dist/"* "$STATIC_DIR/"

echo "Web interface built successfully"

# Finetune model
echo "Starting model finetuning..."

COMMIT_ID=$(cd "$SIA_DIR" && git rev-parse HEAD)
echo "Current commit: $COMMIT_ID"

mkdir -p "$MODELS_DIR/$COMMIT_ID"
mkdir -p "$MODELS_DIR/current"

# Run finetuning using the train environment
"$VENVS_DIR/train/bin/train_deepseek" --output-dir "$MODELS_DIR/$COMMIT_ID"

ln -sf "$MODELS_DIR/$COMMIT_ID" "$MODELS_DIR/current"
echo "Finetuning complete, model linked to current"

# Initialize environment information
echo "Initializing environment information..."
mkdir -p "$DATA_DIR/environment"
cat > "$DATA_DIR/environment/sia_repo.md" << EOF
# SIA Repository Information

- Repository URL: $SIA_REPO_URL
- ssh key: ~/.ssh/sia_git.pub
EOF

# Create .env file for local model only
cat > "$SIA_DIR/.env" << EOF
SIA_DEEPSEEK_ENABLED=true
SIA_DEEPSEEK_MODEL=$MODELS_DIR/current
SIA_DEEPSEEK_TEMPERATURE=0.6
EOF

# Print header
echo "==================================================="
echo "SIA environment initialization complete!"
echo "==================================================="

# Start SIA using restart script
echo "Starting SIA..."
"$SIA_DIR/scripts/restart.sh"
===== FILE: scripts/collect.sh =====
#!/usr/bin/env bash

set -euo pipefail

declare -A FILTER_SETS=(
  ["py"]="-f .*(\\.py|requirements.txt)$"
  ["web"]="-f .*\\.(js|jsx|json|css|html)$"
  ["doc"]="-f .*\\.md$"
  ["deploy"]="-f .*(Dockerfile|\\.sh|\\.xsd|\\.yaml)$"
  
  ["core"]="-s py ./sia ./tools -s deploy . -f ^(?!procedures/).*\\.md$ ."
  ["webui"]="-s web ./web"
  ["tests"]="-s py ./test"
  ["procedures"]="-s doc ./procedures"
)

OUTPUT="/dev/stdout"
VERBOSE=0

usage() {
  echo "Usage: $0 [OPTIONS] [DIRS...]"
  echo
  echo "Options:"
  echo "  -f, --filter PATTERN   Regex filter for subsequent directories"
  echo "  -s, --set SETNAME      Use predefined set of parameters"
  echo "  -o, --output FILE      Output file (default: stdout)"
  echo "  -v, -vv                Verbose output"
  echo
  echo "Predefined Filter Sets:"
  for setname in $(echo "${!FILTER_SETS[@]}" | tr ' ' '\n' | sort); do
    printf "  %-15s %s\n" "$setname:" "${FILTER_SETS[$setname]}"
  done
  exit 1
}

# Expand any set arguments to their full form
expand_args() {
  local -a expanded=()
  local i=1
  local has_sets=0
  
  # First pass: expand sets
  while [ $i -le $# ]; do
    local arg="${!i}"
    
    case "$arg" in
      -s|--set)
        has_sets=1
        i=$((i+1))
        [ $i -le $# ] || { echo "Error: No set name specified"; usage; }
        local set_name="${!i}"
        
        if [[ -v "FILTER_SETS[$set_name]" ]]; then
          # Add the expanded set parameters
          local -a set_args
          read -ra set_args <<< "${FILTER_SETS[$set_name]}"
          for set_arg in "${set_args[@]}"; do
            expanded+=("$set_arg")
          done
        else
          echo "Error: Unknown set '$set_name'"
          usage
        fi
        ;;
      *)
        # Preserve other arguments
        expanded+=("$arg")
        ;;
    esac
    
    i=$((i+1))
  done
  
  # If we expanded sets, recursively expand again until no more sets
  if [ $has_sets -eq 1 ]; then
    expand_args "${expanded[@]}"
  else
    # No more sets, return the fully expanded arguments
    echo "${expanded[@]+"${expanded[@]}"}"
  fi
}

# Process the final expanded command line
process_command() {
  local filter=""
  declare -A dir_filters_map  # Maps directory to array of filters
  
  if ((VERBOSE >= 1)); then
    echo "Expanded arguments:"
    for arg in "$@"; do
      echo "  $arg"
    done
  fi
  
  # Process final arguments
  local i=1
  while [ $i -le $# ]; do
    local arg="${!i}"
    
    case "$arg" in
      -f|--filter)
        i=$((i+1))
        [ $i -le $# ] || break
        filter="${!i}"
        ;;
      -o|--output)
        i=$((i+1))
        [ $i -le $# ] || break
        OUTPUT="${!i}"
        ;;
      -v)
        ((VERBOSE++))
        ;;
      -vv)
        VERBOSE=2
        ;;
      -s|--set)
        echo "Warning: Set argument found after expansion"
        i=$((i+1))
        ;;
      -*)
        # Skip other options
        ;;
      *)
        # It's a directory
        if [ -e "$arg" ]; then
          # Add filter to this directory's filter list
          if [ -n "$filter" ]; then
            if [ -n "${dir_filters_map[$arg]:-}" ]; then
              dir_filters_map["$arg"]="${dir_filters_map["$arg"]}|$filter"
            else
              dir_filters_map["$arg"]="$filter"
            fi
            if ((VERBOSE >= 1)); then
              echo "Adding filter: $filter to directory: $arg"
            fi
          else
            # If no filter specified, ensure directory is in the map
            [ -z "${dir_filters_map[$arg]:-}" ] && dir_filters_map["$arg"]=""
            if ((VERBOSE >= 1)); then
              echo "Adding directory with no filter: $arg"
            fi
          fi
        else
          echo "Warning: Directory '$arg' not found"
        fi
        ;;
    esac
    
    i=$((i+1))
  done
  
  # Convert output path and check conflicts
  output_abs_path=$(realpath -m "$OUTPUT")
  output_rel_path=$(realpath --relative-to=. "$output_abs_path")
  
  # Generate directory tree
  {
    echo "Directory Tree:"
    if command -v tree &>/dev/null; then
      tree -a -I '.git' .
    else
      find . -name .git -prune -o -print | sed -e 's;[^/]*/;|____;g;s;____|; |;g'
    fi
  } > "$OUTPUT"
  
  # Process directories and collect files
  declare -a all_files=()
  
  # Cache git-ignored files once
  declare -a git_ignored_files=()
  if git rev-parse --is-inside-work-tree &>/dev/null; then
    while IFS= read -r ignored_file; do
      git_ignored_files+=("$ignored_file")
    done < <(git ls-files --others --ignored --exclude-standard 2>/dev/null)
    
    if ((VERBOSE >= 1)); then
      echo "Cached ${#git_ignored_files[@]} git-ignored files"
    fi
  fi
  
  for dir in "${!dir_filters_map[@]}"; do
    local filters="${dir_filters_map[$dir]}"
    
    if ((VERBOSE >= 1)); then
      echo "Processing directory: $dir with filters: ${filters:-none}"
    fi
    
    # Get all files in the directory first
    declare -a dir_files=()
    while IFS= read -r -d $'\0' file; do
      local rel_path="${file#./}"
      [[ -z "$rel_path" || "$rel_path" == "." ]] && continue
      
      # Skip output file
      [[ "$rel_path" == "$output_rel_path" ]] && continue
      
      # Check if file is git-ignored using cached list
      local is_ignored=0
      for ignored in "${git_ignored_files[@]}"; do
        if [[ "$rel_path" == "$ignored" ]]; then
          is_ignored=1
          break
        fi
      done
      
      # Skip git-ignored files
      [[ $is_ignored -eq 1 ]] && continue
      
      dir_files+=("$rel_path")
    done < <(find "$dir" \( -path '*/.git' -prune \) -o \( -path "./$output_rel_path" -prune \) -o -type f -print0 2>/dev/null)
    
    if ((VERBOSE >= 1)); then
      echo "  Found ${#dir_files[@]} files in directory"
    fi
    
    # If filters are specified, apply them to the full file list
    if [[ -n "$filters" ]]; then
      # Create temp file with all paths
      local temp_file=$(mktemp)
      printf "%s\n" "${dir_files[@]}" > "$temp_file"
      
      # Apply filters to get matching files
      declare -a matching_files=()
      while IFS= read -r matched; do
        [[ -n "$matched" ]] && matching_files+=("$matched")
        (( VERBOSE >= 2 )) && echo "  Included: $matched"
      done < <(grep -E "($filters)" "$temp_file")
      
      # Add matching files to all_files
      all_files+=("${matching_files[@]}")
      
      # Clean up
      rm -f "$temp_file"
      
      if ((VERBOSE >= 1)); then
        echo "  Matched ${#matching_files[@]} files after filtering"
      fi
    else
      # No filter, add all files
      all_files+=("${dir_files[@]}")
      
      if ((VERBOSE >= 2)); then
        for file in "${dir_files[@]}"; do
          echo "  Included: $file"
        done
      fi
    fi
  done
  
  # Generate unique file list
  if [ ${#all_files[@]} -gt 0 ]; then
    readarray -t unique_files < <(printf "%s\n" "${all_files[@]}" | sort -u)
    
    # Append file contents to output
    {
      echo -e "\n\nFile Contents:"
      for file in "${unique_files[@]}"; do
        if [[ -L "$file" ]]; then
          target=$(readlink -f "$file" || echo "unknown")
          echo -e "\n===== SYMLINK: $file → $target ====="
        else
          echo -e "\n===== FILE: $file ====="
          cat "$file" 2>/dev/null || echo "Error: Unable to read file"
        fi
      done
    } >> "$OUTPUT"
  fi
}

main() {
  # No arguments? Show usage
  [ $# -eq 0 ] && usage
  
  # First scan for verbose flags
  for arg in "$@"; do
    if [[ "$arg" == "-v" ]]; then
      ((VERBOSE++))
    elif [[ "$arg" == "-vv" ]]; then
      VERBOSE=2
    fi
  done
  
  # Expand all set arguments recursively
  local -a expanded_args
  read -ra expanded_args <<< "$(expand_args "$@")"
  
  # Process the completely expanded arguments
  process_command "${expanded_args[@]}"
  
  echo "Concatenation complete. Output written to $OUTPUT" >&2
}

main "$@"

===== FILE: scripts/container.sh =====
#!/bin/bash

container_id=$(docker ps -q)
docker exec -it $container_id bash
===== FILE: scripts/restart.sh =====
#!/bin/bash

while true; do
    sia
    if [ $? -eq 42 ]; then
        echo "SIA exited with code 42. Restarting."
    else
        echo "SIA exited with code $?. Not restarting."
        break
    fi
done
===== FILE: scripts/run.sh =====
#!/bin/bash

export MSYS_NO_PATHCONV=1
set -e

function chown_iterations() {
    if [ -d "./iterations" ] && [ "$(find ./iterations/ ! -user $USER -o ! -group $USER 2>/dev/null)" ]; then
        echo "Chowning iterations directory"
        sudo chown -R $USER:$USER ./iterations/
    fi
}

trap chown_iterations EXIT

docker build \
    --tag sia \
    .

docker run \
    --init \
    --rm \
    -ti \
    --gpus=all \
    -p 8080:8080 \
    --env-file .env \
    -v /$(pwd)/model/:/root/models/current/ \
    -v /$(pwd)/iterations/:/root/data/iterations/ \
    -v /$(pwd)/tasks/:/root/data/tasks/ \
    -v /$(pwd)/user/:/root/data/user/ \
    -v /$(pwd)/environment/:/root/data/environment/ \
    -v /$(pwd)/:/root/sia/ \
    sia "$@"

exit $?
===== FILE: scripts/test.sh =====
#!/bin/bash

docker build \
    --tag sia \
    .

# Run tests within the SIA virtual environment
docker run \
    --rm \
    -ti \
    --gpus=all \
    -p 8080:8080 \
    --env-file .env \
    -v /$(pwd)/model/:/root/models/current/ \
    -v /$(pwd)/iterations/:/root/data/iterations/ \
    -v /$(pwd)/tasks/:/root/data/tasks/ \
    -v /$(pwd)/user/:/root/data/user/ \
    -v /$(pwd)/environment/:/root/data/environment/ \
    -v /$(pwd)/:/root/sia/ \
    sia /root/venvs/sia/bin/python -m unittest discover -v -p "*test.py"
===== FILE: sia/__init__.py =====

===== FILE: sia/__main__.py =====
from aiohttp import web
import asyncio

from .auto_approver import AutoApprover
from .config import Config
from .llm_engine.hf_llm_engine import HfLlmEngine
from .llm_engine.deepseek_llm_engine import DeepSeekLlmEngine
from .iteration_logger import IterationLogger
from .llm_engine.local_llm_engine import LocalLlmEngine
from .llm_engine.mistral_llm_engine import MistralLlmEngine
from .llm_engine.openai_llm_engine import OpenAILlmEngine
from .response_parser import ResponseParser
from .system_metrics import SystemMetrics
from .web.api import Api
from .web.static import Static
from .web.websockts import Websockets
from .web_agent import WebAgent
from .web_io_buffer import WebIOBuffer
from .working_memory import WorkingMemory
from .xml_validator import XMLValidator

class Main:
    @classmethod
    async def create(cls, config: Config):
        self = cls()
        self._config = config

        self._system_prompt = self._config.system_prompt.read_text()
        self._action_schema = self._config.action_schema.read_text()

        # Initialize LLM engines based on config
        self._llms = {}
        
        if config.local_enabled:
            self._llms['local'] = LocalLlmEngine(
                config.local_model,
                config.local_temperature,
                config.local_token_limit,
                config.local_api_key,
            )
            
        if config.openai_enabled:
            self._llms['openai'] = OpenAILlmEngine(
                config.openai_model,
                config.openai_temperature,
                config.openai_token_limit,
                config.openai_api_key,
            )
            
        if config.hf_enabled:
            self._llms['hf'] = HfLlmEngine(
                config.hf_model,
                config.hf_temperature,
                config.hf_api_key,
            )
            
        if config.mistral_enabled:
            self._llms['mistral'] = MistralLlmEngine(
                config.mistral_model,
                config.mistral_temperature,
                config.mistral_token_limit,
                config.mistral_api_key,
            )

        if config.deepseek_enabled:
            self._llms['deepseek'] = DeepSeekLlmEngine(
                config.deepseek_model,
                config.deepseek_temperature,
                config.deepseek_token_limit,
                config.hf_api_key,  # Use the existing HF API key
            )

        if not self._llms:
            raise ValueError("No LLM engines enabled in configuration")

        self._io_buffer = WebIOBuffer()
        self._working_memory = WorkingMemory()
        self._agent = WebAgent(
            system_prompt=self._system_prompt,
            action_schema=self._action_schema,
            working_memory=self._working_memory,
            metrics=SystemMetrics(),
            llms=self._llms,
            validator=XMLValidator(self._action_schema),
            parser=ResponseParser(config.work_dir, self._io_buffer),
            iteration_logger=IterationLogger(self._config.iterations_dir, self._system_prompt, self._action_schema),
        )
        self._auto_approver = AutoApprover(self._agent)

        self._app = web.Application()
        self._api = Api(config.work_dir, self._app, self._agent, self._io_buffer, self._working_memory, self._auto_approver)
        self._websockets = Websockets(self._app, self._agent, self._io_buffer, self._auto_approver, self._working_memory)
        self._static = Static(self._app, self._config)

        return self

    @property
    def app(self):
        return self._app

    async def _serve_index(self, request: web.Request) -> web.Response:
        """Serve the React application HTML for any unmatched routes."""
        index_path = self._config.static_files / "index.html"
        if not index_path.exists():
            raise web.HTTPNotFound()

        with open(index_path, "r") as f:
            html_content = f.read()

        return web.Response(
            text=html_content,
            content_type="text/html"
        )

def main():
    loop = asyncio.new_event_loop()
    config = Config()
    main_instance = loop.run_until_complete(Main.create(config))
    print(f"Web server started at http://localhost:{config.port}")
    web.run_app(main_instance.app, loop=loop, host=config.host, port=config.port)
    return 0
===== FILE: sia/auto_approver.py =====
from threading import Thread, Event
from typing import Callable, TypeAlias, TypedDict
from .web_agent import WebAgent, LlmState

class AutoApproverConfig(TypedDict):
    context_enabled: bool
    response_enabled: bool
    context_timeout: float
    response_timeout: float
    llm_name: str

ConfigChangeHandler: TypeAlias = Callable[[AutoApproverConfig], None]

class AutoApprover:
    """
    Handles automatic approvals for WebAgent states after configurable timeouts.
    """
    
    def __init__(self, agent: WebAgent):
        """
        Initialize auto approver with a WebAgent instance.
        """
        self.agent = agent
        self._llm_name = next(iter(agent.llms.keys()))
        self._context_timeout = 5.0
        self._response_timeout = 10.0
        self._context_enabled = False
        self._response_enabled = False
        self._stop_event = Event()
        self._context_thread: Thread | None = None
        self._response_thread: Thread | None = None
        self._config_change_handlers: list[ConfigChangeHandler] = []
        
        self.agent.add_llm_change_handler(self._handle_llm_state_change)
        self.agent.add_context_change_handler(self._handle_context_change)

    @property
    def config(self) -> AutoApproverConfig:
        return AutoApproverConfig(
            context_enabled=self._context_enabled,
            response_enabled=self._response_enabled,
            context_timeout=self._context_timeout,
            response_timeout=self._response_timeout,
            llm_name=self._llm_name
        )

    def set_config(self, config: AutoApproverConfig) -> None:
        if config['llm_name'] not in self.agent.llms:
            raise ValueError(f"Unknown LLM: {config['llm_name']}")
        
        notify_config_change = self._notify_config_change
        self._notify_config_change = lambda: None
            
        try:
            self.context_enabled = False
            self.response_enabled = False

            self.context_timeout = config['context_timeout']
            self.response_timeout = config['response_timeout']
            self.llm_name = config['llm_name']

            self.context_enabled = config['context_enabled']
            self.response_enabled = config['response_enabled'] 
        finally:
            self._notify_config_change = notify_config_change
        self._notify_config_change()

    def add_config_change_handler(self, handler: ConfigChangeHandler) -> None:
        self._config_change_handlers.append(handler)

    def _notify_config_change(self) -> None:
        current_config = self.config
        for handler in self._config_change_handlers:
            handler(current_config)

    @property
    def context_timeout(self) -> float:
        return self._context_timeout
    
    @context_timeout.setter 
    def context_timeout(self, timeout: float) -> None:
        if self._context_enabled:
            raise ValueError("Cannot change timeout while auto-approval is enabled")
        self._context_timeout = timeout
        self._notify_config_change()

    @property
    def response_timeout(self) -> float:
        return self._response_timeout
    
    @response_timeout.setter
    def response_timeout(self, timeout: float) -> None:
        if self._response_enabled:
            raise ValueError("Cannot change timeout while auto-approval is enabled")
        self._response_timeout = timeout
        self._notify_config_change()

    @property
    def context_enabled(self) -> bool:
        return self._context_enabled

    @context_enabled.setter
    def context_enabled(self, enabled: bool) -> None:
        if enabled == self._context_enabled:
            return
        self._context_enabled = enabled
        self._stop_context_thread()
        if enabled and self.agent.llms[self._llm_name] == LlmState.NO_OUTPUT:
            self._start_context_thread()
        self._notify_config_change()

    @property 
    def response_enabled(self) -> bool:
        return self._response_enabled

    @response_enabled.setter
    def response_enabled(self, enabled: bool) -> None:
        if enabled == self._response_enabled:
            return
        self._response_enabled = enabled
        self._stop_response_thread()
        if enabled and self.agent.llms[self._llm_name] == LlmState.OUTPUT:
            self._start_response_thread()
        self._notify_config_change()

    @property
    def llm_name(self) -> str:
        return self._llm_name
    
    @llm_name.setter
    def llm_name(self, name: str) -> None:
        if name not in self.agent.llms:
            raise ValueError(f"Unknown LLM: {name}")
        self._llm_name = name
        self._notify_config_change()

    def _handle_llm_state_change(self, llm_name: str, state: LlmState) -> None:
        if llm_name != self._llm_name:
            return
            
        if state == LlmState.OUTPUT and self._response_enabled:
            self._start_response_thread()
        else:
            self._stop_response_thread()

    def _handle_context_change(self, context: str, generated: bool) -> None:
        if generated and self._context_enabled:
            self._start_context_thread()
        else:
            self._stop_context_thread()

    def _stop_context_thread(self) -> None:
        if self._context_thread:
            self._stop_event.set()
            self._context_thread = None
            self._stop_event.clear()

    def _stop_response_thread(self) -> None:
        if self._response_thread:
            self._stop_event.set()
            self._response_thread = None
            self._stop_event.clear()

    def _start_context_thread(self) -> None:
        self._context_thread = Thread(target=self._context_approval_thread)
        self._context_thread.start()

    def _start_response_thread(self) -> None:
        self._response_thread = Thread(target=self._response_approval_thread)
        self._response_thread.start()

    def _context_approval_thread(self) -> None:
        if self._stop_event.wait(self._context_timeout):
            return
        if self._context_enabled:
            self.agent.run_inference(self._llm_name)

    def _response_approval_thread(self) -> None:
        if self._stop_event.wait(self._response_timeout):
            return
        if (self._response_enabled and 
            self.agent.llms[self._llm_name] == LlmState.OUTPUT):
            self.agent.approve_response(self._llm_name, self.agent.get_output(self._llm_name))

===== FILE: sia/base_agent.py =====
from abc import ABC
import xml.etree.ElementTree as ET

from .llm_engine import LlmEngine
from .response_parser import ResponseParser
from .system_metrics import SystemMetrics
from .util import pretty_print_element
from .working_memory import WorkingMemory
from .xml_validator import XMLValidator

class BaseAgent(ABC):
    """
    Abstract base class for SIA agents.
    
    Provides core functionality for maintaining working memory, system metrics,
    and coordinating components for LLM inference.
    """
    
    def __init__(
            self,
            system_prompt: str,
            action_schema: str, 
            working_memory: WorkingMemory,
            metrics: SystemMetrics,
            validator: XMLValidator,
            parser: ResponseParser,
        ):
        """
        Initialize agent with required components.
        """
        self._system_prompt = system_prompt
        self._action_schema = action_schema
        self._working_memory = working_memory
        self._metrics = metrics
        self._validator = validator
        self._parser = parser
    
    @property
    def system_prompt(self) -> str:
        """Get the system prompt."""
        return f"{self._system_prompt}\n{self._action_schema}"
            
    def _compile_context(self, llmEngine: LlmEngine) -> str:
        """
        Compile the current context for LLM inference.
        Includes system metrics and working memory entries.
        
        Returns:
            str: Complete context as XML string
        """
        memory_context = self._working_memory.generate_context()
        metrics_data = self._metrics.get_metrics()
        
        # Create context element
        context = ET.Element("context")
        context.set("time", metrics_data["timestamp"])
        context.set("memory_used", str(metrics_data["memory_used"]))
        context.set("memory_total", str(metrics_data["memory_total"]))
        context.set("disk_used", str(metrics_data["disk_used"]))
        context.set("disk_total", str(metrics_data["disk_total"]))
        context.set("stdin", str(self._parser.io_buffer.buffer_length()))
        context.set("context", "100%")

        for entry in memory_context:
            context.append(entry)

        context_str = pretty_print_element(context)
        
        # Calculate token usage percentage
        token_count = llmEngine.token_count(self.system_prompt, context_str)
        token_limit = llmEngine.token_limit()
        context_usage = (float(token_count) / float(token_limit)) * 100.0
        
        # Update context usage metric
        context.set("context", f"{str(round(context_usage, 2))}%")
        
        return pretty_print_element(context)
===== FILE: sia/command.py =====
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING

from .working_memory import WorkingMemory
from .command_result import CommandResult

class Command(ABC):
    """
    Abstract base class for all commands that can be executed on working memory.
    Commands represent immediate actions that modify the system state.
    """
    
    @abstractmethod
    def execute(self, memory: WorkingMemory) -> CommandResult:
        """
        Execute the command on the given working memory.
        
        Args:
            memory: WorkingMemory instance to execute command on
            
        Returns:
            CommandResult: Result of command execution
        """
        pass

===== FILE: sia/command_result.py =====
from dataclasses import dataclass

@dataclass
class CommandResult:
    """
    Result of a command execution.
    
    Attributes:
        message: Optional message describing the result
        success: Whether the command executed successfully
        should_stop: Whether the agent should stop after this command
    """
    message: str
    success: bool = True
    should_stop: bool = False
    
    @staticmethod
    def success() -> 'CommandResult':
        """Create a successful command result."""
        return CommandResult(message="", success=True, should_stop=False)
        
    @staticmethod
    def failure(message: str) -> 'CommandResult':
        """
        Create a failed command result.
        
        Args:
            message: Description of the failure
        """
        return CommandResult(message=message, success=False, should_stop=False)
        
    @staticmethod
    def stop() -> 'CommandResult':
        """Create a command result indicating the agent should stop."""
        return CommandResult(message="", success=True, should_stop=True)
===== FILE: sia/config.py =====
from dataclasses import dataclass
from python_dotenv import load_dotenv
from pathlib import Path
from typing import Optional
import argparse
import os

@dataclass
class Config:
    def __init__(self):
        load_dotenv()
        parser = argparse.ArgumentParser(description='SIA - Self Improving Agent')
        
        # Core configuration
        parser.add_argument(
            '--system-prompt',
            type=Path,
            default=os.getenv('SIA_SYSTEM_PROMPT', '/root/sia/system_prompt.md'),
            help='Path to the system prompt file (default: /root/sia/system_prompt.md, env: SIA_SYSTEM_PROMPT)'
        )
        parser.add_argument(
            '--action-schema',
            type=Path,
            default=os.getenv('SIA_ACTION_SCHEMA', '/root/sia/action_schema.xsd'),
            help='Path to the action schema file (default: /root/sia/action_schema.xsd, env: SIA_ACTION_SCHEMA)'
        )
        parser.add_argument(
            '--iterations-dir',
            type=Path,
            default=os.getenv('SIA_ITERATIONS_DIR', '/root/data/iterations'),
            help='Path to the directory for storing iterations (default: /root/data/iterations, env: SIA_ITERATIONS_DIR)'
        )
        parser.add_argument(
            '--work-dir',
            type=Path,
            default=os.getenv('SIA_WORK_DIR', '/root/desktop'),
            help='Path to the working directory (default: /root/desktop, env: SIA_WORK_DIR)'
        )

        # Web server configuration
        parser.add_argument(
            '--server',
            action='store_true',
            default=self._parse_bool_env('SIA_SERVER_ENABLED', False),
            help='Enable web server for debugging and human feedback (env: SIA_SERVER_ENABLED)'
        )
        parser.add_argument(
            '--host',
            type=str,
            default=os.getenv('SIA_SERVER_HOST', '0.0.0.0'),
            help='Web server host (default: 0.0.0.0, env: SIA_SERVER_HOST)'
        )
        parser.add_argument(
            '--port',
            type=int,
            default=int(os.getenv('SIA_SERVER_PORT', '8080')),
            help='Web server port (default: 8080, env: SIA_SERVER_PORT)'
        )
        parser.add_argument(
            '--static-files',
            type=Path,
            default=self._parse_optional_path('SIA_STATIC_FILES', '/root/static/'),
            help='Path to static web files (default: /root/static/, env: SIA_STATIC_FILES)'
        )

        # Local LLM configuration
        parser.add_argument(
            '--local-enable',
            action='store_true',
            default=self._parse_bool_env('SIA_LOCAL_ENABLED', False),
            help='Enable local LLM engine (env: SIA_LOCAL_ENABLED)'
        )
        parser.add_argument(
            '--local-model',
            type=str,
            default=os.getenv('SIA_LOCAL_MODEL', '/root/models/current'),
            help='Path to local model directory (default: /root/models/current, env: SIA_LOCAL_MODEL)'
        )
        parser.add_argument(
            '--local-temperature',
            type=float,
            default=float(os.getenv('SIA_LOCAL_TEMPERATURE', '0.7')),
            help='Local LLM temperature (default: 0.7, env: SIA_LOCAL_TEMPERATURE)'
        )
        parser.add_argument(
            '--local-token-limit',
            type=int,
            default=int(os.getenv('SIA_LOCAL_TOKEN_LIMIT', '2048')),
            help='Local LLM token limit (env: SIA_LOCAL_TOKEN_LIMIT)'
        )
        parser.add_argument(
            '--local-api-key',
            type=str,
            default=os.getenv('SIA_LOCAL_API_KEY'),
            help='API key for local models (env: SIA_LOCAL_API_KEY)'
        )

        # OpenAI configuration
        parser.add_argument(
            '--openai-enable',
            action='store_true',
            default=self._parse_bool_env('SIA_OPENAI_ENABLED', False),
            help='Enable OpenAI LLM engine (env: SIA_OPENAI_ENABLED)'
        )
        parser.add_argument(
            '--openai-model',
            type=str,
            default=os.getenv('SIA_OPENAI_MODEL', 'gpt-3.5-turbo'),
            help='OpenAI model name (default: gpt-3.5-turbo, env: SIA_OPENAI_MODEL)'
        )
        parser.add_argument(
            '--openai-temperature',
            type=float,
            default=float(os.getenv('SIA_OPENAI_TEMPERATURE', '0.7')),
            help='OpenAI temperature (default: 0.7, env: SIA_OPENAI_TEMPERATURE)'
        )
        parser.add_argument(
            '--openai-token-limit',
            type=int,
            default=int(os.getenv('SIA_OPENAI_TOKEN_LIMIT', '4096')),
            help='OpenAI token limit (env: SIA_OPENAI_TOKEN_LIMIT)'
        )
        parser.add_argument(
            '--openai-api-key',
            type=str,
            default=os.getenv('SIA_OPENAI_API_KEY'),
            help='OpenAI API key (env: SIA_OPENAI_API_KEY)'
        )

        # Hugging Face configuration
        parser.add_argument(
            '--hf-enable',
            action='store_true',
            default=self._parse_bool_env('SIA_HF_ENABLED', False),
            help='Enable Hugging Face LLM engine (env: SIA_HF_ENABLED)'
        )
        parser.add_argument(
            '--hf-model',
            type=str,
            default=os.getenv('SIA_HF_MODEL'),
            help='Hugging Face model name (env: SIA_HF_MODEL)'
        )
        parser.add_argument(
            '--hf-temperature',
            type=float,
            default=float(os.getenv('SIA_HF_TEMPERATURE', '0.7')),
            help='Hugging Face temperature (default: 0.7, env: SIA_HF_TEMPERATURE)'
        )
        parser.add_argument(
            '--hf-api-key',
            type=str,
            default=os.getenv('SIA_HF_API_KEY'),
            help='Hugging Face API key (env: SIA_HF_API_KEY)'
        )

        # Mistral configuration
        parser.add_argument(
            '--mistral-enable',
            action='store_true',
            default=self._parse_bool_env('SIA_MISTRAL_ENABLED', False),
            help='Enable Mistral LLM engine (env: SIA_MISTRAL_ENABLED)'
        )
        parser.add_argument(
            '--mistral-model',
            type=str,
            default=os.getenv('SIA_MISTRAL_MODEL'),
            help='Mistral model name (env: SIA_MISTRAL_MODEL)'
        )
        parser.add_argument(
            '--mistral-temperature',
            type=float,
            default=float(os.getenv('SIA_MISTRAL_TEMPERATURE', '0.7')),
            help='Mistral temperature (default: 0.7, env: SIA_MISTRAL_TEMPERATURE)'
        )
        parser.add_argument(
            '--mistral-token-limit',
            type=int,
            default=int(os.getenv('SIA_MISTRAL_TOKEN_LIMIT', '4096')),
            help='Mistral token limit (env: SIA_MISTRAL_TOKEN_LIMIT)'
        )
        parser.add_argument(
            '--mistral-api-key',
            type=str,
            default=os.getenv('SIA_MISTRAL_API_KEY'),
            help='Mistral API key (env: SIA_MISTRAL_API_KEY)'
        )
        parser.add_argument(
            '--deepseek-enable',
            action='store_true',
            default=self._parse_bool_env('SIA_DEEPSEEK_ENABLED', False),
            help='Enable DeepSeek LLM engine (env: SIA_DEEPSEEK_ENABLED)'
        )
        parser.add_argument(
            '--deepseek-model',
            type=str,
            default=os.getenv('SIA_DEEPSEEK_MODEL', '/root/models/current'),
            help='Path to fine-tuned DeepSeek model (env: SIA_DEEPSEEK_MODEL)'
        )
        parser.add_argument(
            '--deepseek-temperature',
            type=float,
            default=float(os.getenv('SIA_DEEPSEEK_TEMPERATURE', '0.6')),
            help='DeepSeek temperature (default: 0.6, env: SIA_DEEPSEEK_TEMPERATURE)'
        )
        parser.add_argument(
            '--deepseek-token-limit',
            type=int,
            default=int(os.getenv('SIA_DEEPSEEK_TOKEN_LIMIT', '0')),
            help='DeepSeek token limit (0 for model default, env: SIA_DEEPSEEK_TOKEN_LIMIT)'
        )

        self.args = parser.parse_args()

    def _parse_bool_env(self, env_var: str, default: bool) -> bool:
        val = os.getenv(env_var)
        if val is None:
            return default
        return val.lower() in ('true', '1', 'yes', 'on')
    
    def _parse_optional_path(self, env_var: str, default: Optional[Path]) -> Optional[Path]:
        val = os.getenv(env_var)
        if val is None:
            return default
        return Path(val)

    # Core properties
    @property
    def system_prompt(self) -> Path:
        return self.args.system_prompt
    
    @property
    def action_schema(self) -> Path:
        return self.args.action_schema
    
    @property
    def iterations_dir(self) -> Path:
        return self.args.iterations_dir
    
    @property
    def work_dir(self) -> Path:
        return self.args.work_dir

    # Server properties
    @property
    def server(self) -> bool:
        return self.args.server

    @property
    def host(self) -> str:
        return self.args.host
    
    @property
    def port(self) -> int:
        return self.args.port
    
    @property
    def static_files(self) -> Path:
        return self.args.static_files

    # Local LLM properties
    @property
    def local_enabled(self) -> bool:
        return self.args.local_enable

    @property
    def local_model(self) -> str:
        return self.args.local_model

    @property
    def local_temperature(self) -> float:
        return self.args.local_temperature

    @property
    def local_token_limit(self) -> int:
        return self.args.local_token_limit
    
    @property
    def local_api_key(self) -> Optional[str]:
        return self.args.local_api_key

    # OpenAI properties
    @property
    def openai_enabled(self) -> bool:
        return self.args.openai_enable

    @property
    def openai_model(self) -> str:
        return self.args.openai_model

    @property
    def openai_temperature(self) -> float:
        return self.args.openai_temperature

    @property
    def openai_token_limit(self) -> int:
        return self.args.openai_token_limit

    @property
    def openai_api_key(self) -> Optional[str]:
        return self.args.openai_api_key

    # Hugging Face properties
    @property
    def hf_enabled(self) -> bool:
        return self.args.hf_enable

    @property
    def hf_model(self) -> str:
        return self.args.hf_model

    @property
    def hf_temperature(self) -> float:
        return self.args.hf_temperature

    @property
    def hf_api_key(self) -> Optional[str]:
        return self.args.hf_api_key

    # Mistral properties
    @property
    def mistral_enabled(self) -> bool:
        return self.args.mistral_enable

    @property
    def mistral_model(self) -> str:
        return self.args.mistral_model

    @property
    def mistral_temperature(self) -> float:
        return self.args.mistral_temperature

    @property
    def mistral_token_limit(self) -> int:
        return self.args.mistral_token_limit

    @property
    def mistral_api_key(self) -> Optional[str]:
        return self.args.mistral_api_key

    @property
    def deepseek_enabled(self) -> bool:
        return self.args.deepseek_enable

    @property
    def deepseek_model(self) -> str:
        return self.args.deepseek_model

    @property
    def deepseek_temperature(self) -> float:
        return self.args.deepseek_temperature

    @property
    def deepseek_token_limit(self) -> Optional[int]:
        # Return None if 0 to use model default
        return self.args.deepseek_token_limit if self.args.deepseek_token_limit > 0 else None
===== FILE: sia/delete_command.py =====
from .command import Command
from .command_result import CommandResult
from .working_memory import WorkingMemory

class DeleteCommand(Command):
    """
    Command to delete an entry from working memory.
    Ensures proper cleanup of entry resources before removal.
    
    Attributes:
        id: Unique identifier of entry to delete
    """
    
    def __init__(self, id: str):
        """
        Initialize delete command.
        
        Args:
            id: Unique identifier of entry to delete
        """
        self.id = id
        
    def execute(self, memory: WorkingMemory) -> CommandResult:
        """
        Delete the specified entry from working memory.
        Performs cleanup on the entry before removal.
        
        Args:
            memory: WorkingMemory instance to delete entry from
            
        Returns:
            CommandResult: Success if entry was found and deleted,
                         failure with message if entry not found
        """
        entry = memory.get_entry(self.id)
        if entry is None:
            return CommandResult.failure(f"Entry with id '{self.id}' not found")
            
        # Perform cleanup before removing
        entry.cleanup()
        memory.remove_entry(self.id)
        return CommandResult.success()
===== FILE: sia/entry/__init__.py =====
from abc import ABC, abstractmethod
import xml.etree.ElementTree as ET
from typing import Callable, List

class Entry(ABC):
    """
    Abstract base class for all entry types in the working memory.
    Provides observable pattern functionality.
    """
    
    def __init__(self, id: str):
        """
        Initialize a new entry with provided id and timestamp.
        
        Args:
            id: Unique identifier for this entry
        """
        self.id = id
        self._change_handlers: List[Callable[['Entry'], None]] = []

    def add_change_handler(self, handler: Callable[['Entry'], None]) -> None:
        """Add a callback for entry changes."""
        if handler not in self._change_handlers:
            self._change_handlers.append(handler)

    def notify_change(self) -> None:
        """Notify all handlers of entry state change."""
        for handler in self._change_handlers:
            handler(self)

    @abstractmethod
    def update(self) -> None:
        """
        Update the entry's state.
        Must be implemented by concrete classes.
        """
        pass
    
    @abstractmethod
    def generate_context(self) -> ET.Element:
        """
        Generate an XML Element representing this entry's context.
        Must be implemented by concrete classes.
        
        Returns:
            ET.Element: XML element containing the entry's data
        """
        pass

    def serialize(self) -> dict[str, str]:
        """
        Collect all public attributes of the entry into a dictionary.
        """
        pass

    def cleanup(self) -> None:
        """
        Clean up any resources used by this entry.
        Should be overridden by classes that need cleanup.
        Default implementation does nothing.
        """
        pass

    def reset(self) -> None:
        """
        Reset the entry as if update was never called.
        Default implementation does nothing.
        """
        pass
===== FILE: sia/entry/background_entry.py =====
import subprocess
import xml.etree.ElementTree as ET 
from typing import Optional

from . import Entry

class BackgroundEntry(Entry):
    """
    Entry type for long-running background processes.
    
    Attributes:
        script: The script/command to execute
        stdout: Captured standard output 
        stderr: Captured standard error
        process: The running subprocess.Popen instance
        exit_code: Exit code when process completes
    """
    
    def __init__(
            self,
            id: str,
            work_dir: str,
            script: str,
        ):
        """
        Initialize a new background entry.
        
        Args:
            id: Unique identifier for this entry
            work_dir: Working directory for the process
            script: The script/command to execute
        """
        super().__init__(id)
        self.work_dir = work_dir
        self.script = script
        self._process: Optional[subprocess.Popen] = None
        self.stdout = ""
        self.stderr = ""
        self.exit_code = None

    @property
    def pid(self) -> Optional[int]:
        """Get the process ID (None if not running)."""
        return self._process.pid if self._process is not None else None

    def cleanup(self) -> None:
        """
        Clean up the background process if it's still running.
        Ensures process is terminated and file handles are closed.
        """
        if self._process is not None:
            try:
                if self._process.stdout:
                    self._process.stdout.close()
                if self._process.stderr:
                    self._process.stderr.close()
                if self._process.poll() is None:
                    self._process.terminate()
                    try:
                        self._process.wait(timeout=1.0)
                    except subprocess.TimeoutExpired:
                        self._process.kill()
                        self._process.wait()
            except:
                pass  # Ignore cleanup errors
            finally:
                self._process = None

    def reset(self) -> None:
        """
        Cleans up existing process and resets output buffers.
        """
        self.cleanup()
        self.stdout = ""
        self.stderr = ""
        self.exit_code = None
        self.notify_change()
        
    def update(self) -> None:
        """
        Start the process if not running and collect any new output.
        Updates stdout and stderr with any new output.
        """
        if self._process is None and self.exit_code is None:
            self._process = subprocess.Popen(
                self.script,
                cwd=self.work_dir,
                shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                text=True,
                bufsize=1  # Line buffered
            )
            self.notify_change()
            return
            
        if self._process is None:
            return
            
        exit_code = self._process.poll()
        if exit_code is not None:
            try:
                remaining_out, remaining_err = self._process.communicate(timeout=0.1)
                self.stdout += remaining_out
                self.stderr += remaining_err
            except subprocess.TimeoutExpired:
                pass  # Process didn't finish communicating, try again next update
            
            self.exit_code = exit_code
            self.cleanup()
            self.notify_change()
            return
            
        if self._process.stdout:
            while True:
                try:
                    line = self._process.stdout.readline()
                    if not line:
                        break
                    self.stdout += line
                except:
                    break
                    
        if self._process.stderr:
            while True:
                try:
                    line = self._process.stderr.readline()
                    if not line:
                        break
                    self.stderr += line
                except:
                    break
        self.notify_change()

    def generate_context(self) -> ET.Element:
        """
        Generate an XML Element representing this background entry.
        
        Returns:
            ET.Element: XML element containing the entry's data
        """
        element = ET.Element("background", {"id": self.id})
        if self._process is not None:
            element.set("pid", str(self._process.pid))
        elif self.exit_code is not None:
            element.set("exit_code", str(self.exit_code))
        element.text = self.script
        stdout_elem = ET.SubElement(element, "stdout")
        stdout_elem.text = self.stdout
        stderr_elem = ET.SubElement(element, "stderr")
        stderr_elem.text = self.stderr
        
        return element

    def serialize(self):
        """
        Collect all public attributes of the entry into a dictionary.
        """
        return {
            "type": "background",
            "id": self.id,
            "work_dir": str(self.work_dir),
            "script": self.script,
            "stdout": self.stdout,
            "stderr": self.stderr,
            "exit_code": self.exit_code,
        }
===== FILE: sia/entry/entry_factory.py =====
from pathlib import Path

from . import Entry
from ..io_buffer import IOBuffer
from .background_entry import BackgroundEntry
from .parse_error_entry import ParseErrorEntry
from .read_entry import ReadEntry
from .reasoning_entry import ReasoningEntry
from .repeat_entry import RepeatEntry
from .single_entry import SingleEntry
from .write_entry import WriteEntry

class EntryFactory:
    def create_entry(data: dict, work_dir: Path, io_buffer: IOBuffer) -> Entry:
        entry_type = data.get("type")
        entry_id = data.get("id")
        
        if entry_type == "background":
            return BackgroundEntry(entry_id, work_dir, data["script"])
            
        elif entry_type == "read_stdin":
            return ReadEntry(entry_id, io_buffer)
            
        elif entry_type == "reasoning":
            return ReasoningEntry(entry_id, data["content"])
            
        elif entry_type == "repeat":
            return RepeatEntry(
                entry_id,
                work_dir,
                data["script"],
                data.get("timeout"),
                data.get("limit")
            )
            
        elif entry_type == "single":
            return SingleEntry(
                entry_id,
                work_dir,
                data["script"],
                data.get("timeout"),
                data.get("limit")
            )
            
        elif entry_type == "write":
            return WriteEntry(entry_id, data["content"], io_buffer)
            
        raise ValueError(f"Unknown entry type: {entry_type}")

    def update_entry(entry: Entry, data: dict):
        if isinstance(entry, Entry):
            if "id" in data:
                entry.id = data["id"]

        if isinstance(entry, SingleEntry):
            if "work_dir" in data:
                entry.work_dir = Path(data["work_dir"])
            if "script" in data:
                entry.script = data["script"]
            if "timeout" in data:
                entry.timeout = float(data["timeout"]) if data["timeout"] is not None else None
            if "limit" in data:
                entry.limit = int(data["limit"]) if data["limit"] else None
            if "stdout" in data:
                entry.stdout = data["stdout"]
            if "stderr" in data:
                entry.stderr = data["stderr"]
            if "exit_code" in data:
                entry.exit_code = int(data["exit_code"]) if data["exit_code"] is not None else None
            if "executed" in data:
                entry.executed = bool(data["executed"])
            if "timed_out" in data:
                entry.timed_out = bool(data["timed_out"])

        if isinstance(entry, RepeatEntry):
            if "work_dir" in data:
                entry.work_dir = Path(data["work_dir"])
            if "script" in data:
                entry.script = data["script"]
            if "timeout" in data:
                entry.timeout = float(data["timeout"]) if data["timeout"] is not None else None
            if "limit" in data:
                entry.limit = int(data["limit"]) if data["limit"] else None
            if "stdout" in data:
                entry.stdout = data["stdout"]
            if "stderr" in data:
                entry.stderr = data["stderr"]
            if "exit_code" in data:
                entry.exit_code = int(data["exit_code"]) if data["exit_code"] is not None else None
            if "executed" in data:
                entry.executed = bool(data["executed"])
            if "timed_out" in data:
                entry.timed_out = bool(data["timed_out"])

        elif isinstance(entry, BackgroundEntry):
            if "work_dir" in data:
                entry.work_dir = Path(data["work_dir"])
            if "script" in data:
                entry.script = data["script"]
            if "stdout" in data:
                entry.stdout = data["stdout"]
            if "stderr" in data:
                entry.stderr = data["stderr"]
            if "exit_code" in data:
                entry.exit_code = int(data["exit_code"]) if data["exit_code"] is not None else None

        elif isinstance(entry, ParseErrorEntry):
            if "content" in data:
                entry.content = data["content"]
            if "error" in data:
                entry.error = data["error"]

        elif isinstance(entry, ReadEntry):
            if "content" in data:
                entry.content = data["content"]
            if "read" in data:
                entry.read = bool(data["read"])

        elif isinstance(entry, ReasoningEntry):
            if "content" in data:
                entry.content = data["content"]

        elif isinstance(entry, WriteEntry):
            if "content" in data:
                entry.content = data["content"]
            if "written" in data:
                entry.written = bool(data["written"])
===== FILE: sia/entry/parse_error_entry.py =====
import xml.etree.ElementTree as ET

from . import Entry

class ParseErrorEntry(Entry):
    """
    Entry type for parse and validation errors.
    """
    
    def __init__(
            self,
            id: str,
            content: str,
            error: str,
        ):
        """
        Initialize a new parse error entry.
        
        Args:
            id: Unique identifier for this entry
            content: Original content that failed to parse
            error: Error message describing the failure
        """
        super().__init__(id)
        self.content = content
        self.error = error

    def update(self) -> None:
        pass

    def generate_context(self) -> ET.Element:
        """
        Generate an XML Element representing this parse error entry.
        
        Returns:
            ET.Element: XML element containing the entry's data
        """
        element = ET.Element("parse_error", {"id": self.id})
        error_elem = ET.SubElement(element, "error")
        error_elem.text = self.error
        content_elem = ET.SubElement(element, "content")
        content_elem.text = self.content
        
        return element

    def serialize(self):
        """
        Collect all public attributes of the entry into a dictionary.
        """
        return {
            "type": "parse_error",
            "id": self.id,
            "content": self.content,
            "error": self.error,
        }
===== FILE: sia/entry/read_entry.py =====
import xml.etree.ElementTree as ET

from . import Entry
from ..io_buffer import IOBuffer

class ReadEntry(Entry):
    """
    Entry type for reading content from standard input.
    """
    
    def __init__(
            self,
            id: str,
            io_buffer: IOBuffer,
        ):
        """
        Initialize a new read entry.
        
        Args:
            id: Unique identifier for this entry
            io_buffer: Buffer to use for IO operations
        """
        super().__init__(id)
        self.content: str = ""
        self.read = False
        self._io_buffer = io_buffer
        
    def update(self) -> None:
        """
        Read from stdin if not already read.
        Uses the provided IO buffer for the actual read operation.
        """
        if not self.read:
            self.content = self._io_buffer.read()
            self.read = True
            self.notify_change()
    
    def reset(self) -> None:
        """
        Reset the entry state to its initial state.
        """
        self.read = False
        self.content = ""
        self.notify_change()

    def generate_context(self) -> ET.Element:
        """
        Generate an XML Element representing this read entry.
        
        Returns:
            ET.Element: XML element containing the entry's data
        """
        element = ET.Element("read_stdin", {"id": self.id})
        if self.read:
            element.text = self.content
            
        return element

    def serialize(self):
        """
        Collect all public attributes of the entry into a dictionary.
        """
        return {
            "type": "read_stdin",
            "id": self.id,
            "content": self.content,
            "read": self.read,
        }
===== FILE: sia/entry/reasoning_entry.py =====
import xml.etree.ElementTree as ET

from . import Entry

class ReasoningEntry(Entry):
    """
    Entry type for agent reasoning steps.
    """
    
    def __init__(
            self,
            id: str,
            content: str,
        ):
        """
        Initialize a new reasoning entry.
        
        Args:
            id: Unique identifier for this entry
            timestamp: Creation timestamp for this entry
            content: The reasoning text
        """
        super().__init__(id)
        self.content = content
        
    def update(self) -> None:
        """No update needed for reasoning entries."""
        pass

    def generate_context(self) -> ET.Element:
        """
        Generate an XML Element representing this reasoning entry.
        
        Returns:
            ET.Element: XML element containing the entry's data
        """
        element = ET.Element("reasoning", {"id": self.id})
        element.text = self.content
        
        return element

    def serialize(self):
        """
        Collect all public attributes of the entry into a dictionary.
        """
        return {
            "type": "reasoning",
            "id": self.id,
            "content": self.content,
        }
===== FILE: sia/entry/repeat_entry.py =====
from pathlib import Path
import subprocess
import xml.etree.ElementTree as ET
from typing import Optional

from . import Entry

class RepeatEntry(Entry):
    """
    Entry type for scripts that are executed on every update.
    """

    default_timeout = 1
    default_limit = 1024
    
    def __init__(
            self,
            id: str,
            work_dir: Path,
            script: str,
            timeout: Optional[float] = None,
            limit: Optional[int] = None,
        ):
        """
        Initialize a new repeat entry.
        
        Args:
            id: Unique identifier for this entry
            script: The script/command to execute
            timeout: Maximum time to wait for script execution
            limit: Maximum number of characters to capture from stdout/stderr
        """
        super().__init__(id)
        self.work_dir = work_dir
        self.script = script
        self.timeout = timeout
        self.limit = limit
        self.stdout: str = ""
        self.stderr: str = ""
        self.exit_code: Optional[int] = None
        self.timed_out: bool = False
        
    def update(self) -> None:
        """
        Execute the script and update the output.
        Captures stdout, stderr and exit code from each execution.
        """
        try:
            process = subprocess.run(
                self.script,
                cwd=self.work_dir,
                timeout=(self.timeout or self.default_timeout),
                shell=True,
                capture_output=True,
                text=True
            )
            self.stdout = process.stdout
            self.stderr = process.stderr
            self.exit_code = process.returncode
            self.timed_out = False
        except subprocess.TimeoutExpired as e:
            self.timed_out = True
        self.notify_change()

    def generate_context(self) -> ET.Element:
        """
        Generate an XML Element representing this repeat entry.
        
        Returns:
            ET.Element: XML element containing the entry's data
        """
        element = ET.Element("repeat", {"id": self.id})
        if self.timeout:
            element.set("timeout", str(self.timeout))
        element.text = self.script
        if self.timed_out:
            element.set("timed_out", "true")
        elif self.exit_code is not None:
            element.set("exit_code", str(self.exit_code))
            if self.limit:
                element.set("limit", str(self.limit))
            if len(self.stdout) > (self.limit or self.default_limit):
                element.set("stdout_truncated", "true")
                element.set("stdout_length", str(len(self.stdout)))
            stdout_elem = ET.SubElement(element, "stdout")
            stdout_elem.text = self.stdout[:(self.limit or self.default_limit)]
            if len(self.stderr) > (self.limit or self.default_limit):
                element.set("stderr_truncated", "true")
                element.set("stderr_length", str(len(self.stderr)))
            stderr_elem = ET.SubElement(element, "stderr")
            stderr_elem.text = self.stderr[:(self.limit or self.default_limit)]
        
        return element

    def serialize(self):
        """
        Collect all public attributes of the entry into a dictionary.
        """
        return {
            "type": "repeat",
            "id": self.id,
            "work_dir": str(self.work_dir),
            "script": self.script,
            "timeout": self.timeout,
            "limit": self.limit,
            "stdout": self.stdout,
            "stderr": self.stderr,
            "exit_code": self.exit_code,
            "timed_out": self.timed_out
        }

===== FILE: sia/entry/single_entry.py =====
from pathlib import Path
import subprocess
import xml.etree.ElementTree as ET
from typing import Optional

from . import Entry

class SingleEntry(Entry):
    """
    Entry type for one-time script executions.
    """

    default_timeout = 1
    default_limit = 1024
    
    def __init__(
            self,
            id: str,
            work_dir: Path,
            script: str,
            timeout: Optional[float] = None,
            limit: Optional[int] = None,
        ):
        """
        Initialize a new single shot entry.
        
        Args:
            id: Unique identifier for this entry
            script: The script/command to execute
            timeout: Maximum time to wait for script execution
            limit: Maximum number of characters to capture from stdout/stderr
        """
        super().__init__(id)
        self.work_dir = work_dir
        self.script = script
        self.timeout = timeout
        self.limit = limit
        self.stdout: str = ""
        self.stderr: str = ""
        self.exit_code: Optional[int] = None
        self.executed: bool = False
        self.timed_out: bool = False

    def reset(self) -> None:
        """Reset execution state to allow running again."""
        self.executed = False
        self.timed_out = False
        self.stdout = ""
        self.stderr = ""
        self.exit_code = None
        self.notify_change()
        
    def update(self) -> None:
        """
        Execute the script if not already executed.
        Captures stdout, stderr and exit code.
        """
        if self.executed:
            return
        self.executed = True
        try:
            process = subprocess.run(
                self.script,
                cwd=self.work_dir,
                timeout=(self.timeout or self.default_timeout),
                shell=True,
                capture_output=True,
                text=True
            )
            self.stdout = process.stdout
            self.stderr = process.stderr
            self.exit_code = process.returncode
        except subprocess.TimeoutExpired as e:
            self.timed_out = True
        self.notify_change()

    def generate_context(self) -> ET.Element:
        """
        Generate an XML Element representing this single shot entry.
        
        Returns:
            ET.Element: XML element containing the entry's data
        """
        element = ET.Element("single", {"id": self.id})
        if self.timeout:
            element.set("timeout", str(self.timeout))
        element.text = self.script
        if self.timed_out:
            element.set("timed_out", "true")
        elif self.executed:
            element.set("exit_code", str(self.exit_code))
            if self.limit:
                element.set("limit", str(self.limit))
            if len(self.stdout) > (self.limit or self.default_limit):
                element.set("stdout_truncated", "true")
                element.set("stdout_length", str(len(self.stdout)))
            stdout_elem = ET.SubElement(element, "stdout")
            stdout_elem.text = self.stdout[:(self.limit or self.default_limit)]
            if len(self.stderr) > (self.limit or self.default_limit):
                element.set("stderr_truncated", "true")
                element.set("stderr_length", str(len(self.stderr)))
            stderr_elem = ET.SubElement(element, "stderr")
            stderr_elem.text = self.stderr[:(self.limit or self.default_limit)]
        return element

    def serialize(self):
        """
        Collect all public attributes of the entry into a dictionary.
        """
        return {
            "type": "single",
            "id": self.id,
            "work_dir": str(self.work_dir),
            "script": self.script,
            "timeout": self.timeout,
            "limit": self.limit,
            "stdout": self.stdout,
            "stderr": self.stderr,
            "exit_code": self.exit_code,
            "executed": self.executed,
            "timed_out": self.timed_out
        }
===== FILE: sia/entry/write_entry.py =====
import xml.etree.ElementTree as ET

from . import Entry
from ..io_buffer import IOBuffer

class WriteEntry(Entry):
    """
    Entry type for writing content to standard output.
    """
    
    def __init__(
            self,
            id: str,
            content: str,
            io_buffer: IOBuffer,
        ):
        """
        Initialize a new write entry.
        
        Args:
            id: Unique identifier for this entry
            content: Text to write to stdout
            io_buffer: Buffer to use for IO operations
        """
        super().__init__(id)
        self.content = content
        self.written: bool = False
        self._io_buffer = io_buffer
        
    def update(self) -> None:
        """
        Write the content to stdout if not already written.
        Uses the provided IO buffer for the actual write operation.
        """
        if not self.written:
            self._io_buffer.write(self.content)
            self.written = True
            self.notify_change()
    
    def reset(self) -> None:
        """
        Reset the entry state to its initial state.
        """
        self.written = False
        self.notify_change()

    def generate_context(self) -> ET.Element:
        """
        Generate an XML Element representing this write entry.
        
        Returns:
            ET.Element: XML element containing the entry's data
        """
        element = ET.Element("write_stdout", {"id": self.id})
        element.text = self.content
        return element

    def serialize(self):
        """
        Collect all public attributes of the entry into a dictionary.
        """
        return {
            "type": "write",
            "id": self.id,
            "content": self.content,
            "written": self.written,
        }

===== FILE: sia/io_buffer.py =====
from abc import ABC, abstractmethod

class IOBuffer(ABC):
    """
    Abstract base class defining the interface for input/output operations.
    
    This interface allows for different implementations of IO handling,
    such as direct system IO or buffered web interface communication.
    """
    
    @abstractmethod
    def read(self) -> str:
        """
        Read and return available input.
        
        Should clear the input buffer after reading.
        
        Returns:
            str: Content from input buffer, or empty string if no input available
        """
        pass
        
    @abstractmethod
    def write(self, content: str) -> None:
        """
        Write content to output.
        
        Args:
            content: String content to write
        """
        pass
        
    @abstractmethod
    def buffer_length(self) -> int:
        """
        Get the current length of buffered input.
        
        Returns:
            int: Number of characters in the input buffer
        """
        pass
===== FILE: sia/iteration_logger.py =====
from datetime import datetime
from pathlib import Path
import xml.etree.ElementTree as ET
import hashlib

from .util import format_timestamp

class IterationLogger:
    """Logs agent iterations to XML files"""
    
    def __init__(
            self,
            iterations_dir: Path,
            system_prompt: str,
            action_schema: str,
        ):
        """Initialize with directory for storing iteration files"""
        self.iterations_dir = iterations_dir
        self.iterations_dir.mkdir(parents=True, exist_ok=True)
        self._system_prompt_hash = hashlib.sha256(system_prompt.encode()).hexdigest()
        self._action_schema_hash = hashlib.sha256(action_schema.encode()).hexdigest()
        
    def log_iteration(
            self,
            timestamp: datetime,
            context: str,
            response: str,
        ):
        """
        Save an iteration to an XML file
        
        Args:
            context: The context as ElementTree
            response: Raw response from LLM
        """
        filename = f"iteration_{format_timestamp(timestamp)}.xml"
        filepath = self.iterations_dir / filename
        
        root = ET.Element("iteration")
        
        root.set("system_prompt_hash", self._system_prompt_hash)
        root.set("action_schema_hash", self._action_schema_hash)
        
        context_elem = ET.SubElement(root, "context")
        context_elem.text = context
        
        response_elem = ET.SubElement(root, "response")
        response_elem.text = response
        
        tree = ET.ElementTree(root)
        tree.write(filepath, encoding="utf-8", xml_declaration=True)
===== FILE: sia/iteration_parser.py =====
from pathlib import Path
from typing import List
import xml.etree.ElementTree as ET

from .entry import Entry
from .entry.background_entry import BackgroundEntry
from .entry.parse_error_entry import ParseErrorEntry
from .entry.read_entry import ReadEntry
from .entry.reasoning_entry import ReasoningEntry
from .entry.repeat_entry import RepeatEntry
from .entry.single_entry import SingleEntry
from .entry.write_entry import WriteEntry

class IterationParser:
    """Parses iteration XML files into entries"""

    @staticmethod
    def parse_iteration(content: str, work_dir: Path, io_buffer) -> tuple[str, str, List[Entry]]:
        """
        Parse iteration XML content into context, response and entries
        
        Args:
            content: Raw XML content as string
            io_buffer: Buffer to use for IO operations
            
        Returns:
            Tuple of (context_str, response_str, entry_list)
        """
        root = ET.fromstring(content)
        context_elem = root.find("context")
        
        if context_elem is None:
            raise ValueError("Invalid iteration file - missing context")
        
        context = ET.fromstring(context_elem.text)

        entries = []
        for elem in context:
            if elem.tag == "background":
                entries.append(IterationParser._parse_background(elem, work_dir))
            elif elem.tag == "parse_error": 
                entries.append(IterationParser._parse_parse_error(elem))
            elif elem.tag == "read_stdin":
                entries.append(IterationParser._parse_read(elem, io_buffer))
            elif elem.tag == "reasoning":
                entries.append(IterationParser._parse_reasoning(elem))
            elif elem.tag == "repeat":
                entries.append(IterationParser._parse_repeat(elem, work_dir))
            elif elem.tag == "single":
                entries.append(IterationParser._parse_single(elem, work_dir))
            elif elem.tag == "write_stdout":
                entries.append(IterationParser._parse_write(elem, io_buffer))

        return entries

    @staticmethod
    def _parse_background(elem: ET.Element, work_dir: Path) -> BackgroundEntry:
        entry = BackgroundEntry(
            id=elem.get("id"),
            script=elem.text,
            work_dir=work_dir
        )
        if elem.get("exit_code"):
            entry.exit_code = int(elem.get("exit_code"))
        stdout = elem.find("stdout")
        if stdout is not None:
            entry.stdout = stdout.text or ""
        stderr = elem.find("stderr")
        if stderr is not None:
            entry.stderr = stderr.text or ""
        return entry

    @staticmethod
    def _parse_parse_error(elem: ET.Element) -> ParseErrorEntry:
        error = elem.find("error")
        content = elem.find("content")
        return ParseErrorEntry(
            id=elem.get("id"),
            content=content.text if content is not None else "",
            error=error.text if error is not None else ""
        )

    @staticmethod 
    def _parse_read(elem: ET.Element, io_buffer) -> ReadEntry:
        entry = ReadEntry(
            id=elem.get("id"),
            io_buffer=io_buffer
        )
        entry.content = elem.text or ""
        entry.read = True
        return entry

    @staticmethod
    def _parse_reasoning(elem: ET.Element) -> ReasoningEntry:
        return ReasoningEntry(
            id=elem.get("id"),
            content=elem.text or ""
        )

    @staticmethod
    def _parse_repeat(elem: ET.Element, work_dir: Path) -> RepeatEntry:
        entry = RepeatEntry(
            id=elem.get("id"),
            work_dir=work_dir,
            script=elem.text,
            timeout=float(elem.get("timeout")) if elem.get("timeout") else None,
            limit=int(elem.get("limit")) if elem.get("limit") else None
        )
        if elem.get("exit_code"):
            entry.exit_code = int(elem.get("exit_code"))
        entry.timed_out = elem.get("timed_out") == "true"
        stdout = elem.find("stdout")
        if stdout is not None:
            entry.stdout = stdout.text or ""
        stderr = elem.find("stderr")
        if stderr is not None:
            entry.stderr = stderr.text or ""
        return entry

    @staticmethod
    def _parse_single(elem: ET.Element, work_dir: Path) -> SingleEntry:
        entry = SingleEntry(
            id=elem.get("id"),
            work_dir=work_dir,
            script=elem.text,
            timeout=float(elem.get("timeout")) if elem.get("timeout") else None,
            limit=int(elem.get("limit")) if elem.get("limit") else None
        )
        if elem.get("exit_code"):
            entry.exit_code = int(elem.get("exit_code"))
            entry.executed = True
        entry.timed_out = elem.get("timed_out") == "true"
        stdout = elem.find("stdout")
        if stdout is not None:
            entry.stdout = stdout.text or ""
        stderr = elem.find("stderr")
        if stderr is not None:
            entry.stderr = stderr.text or ""
        return entry

    @staticmethod
    def _parse_write(elem: ET.Element, io_buffer) -> WriteEntry:
        entry = WriteEntry(
            id=elem.get("id"),
            content=elem.text or "",
            io_buffer=io_buffer
        )
        entry.written = True
        return entry
===== FILE: sia/llm_engine/__init__.py =====
from typing import Callable, Iterator
from abc import ABC, abstractmethod

class LlmEngine(ABC):
    @abstractmethod
    def infer(self, system_prompt: str, main_context: str, should_stop: Callable[[], bool] = lambda: False) -> Iterator[str]:
        pass

    @abstractmethod
    def token_count(self, system_prompt: str, main_context: str) -> int:
        pass

    @abstractmethod
    def token_limit(self) -> int:
        pass
===== FILE: sia/llm_engine/deepseek_llm_engine.py =====
from typing import Callable, Iterator, Optional
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
from threading import Thread
from pathlib import Path

from . import LlmEngine
from .. import util

class DeepSeekLlmEngine(LlmEngine):
    """
    LLM Engine implementation for DeepSeek models.
    Supports fine-tuned DeepSeek-R1 and its distilled versions.
    """
    
    def __init__(
        self,
        model_path: str,
        temperature: float = 0.6,
        token_limit: Optional[int] = None,
        api_key: Optional[str] = None,
    ):
        """
        Initialize the DeepSeek LLM Engine.

        Args:
            model_path: Local path to the fine-tuned model
            temperature: Sampling temperature (0.6 default as recommended)
            token_limit: Maximum tokens to generate or context length override
            api_key: HuggingFace API token if needed
        """
        self._model_path = Path(model_path)
        self._temperature = temperature
        self._token_limit = token_limit
        
        # Load tokenizer with trust_remote_code for DeepSeek models
        self._tokenizer = AutoTokenizer.from_pretrained(
            self._model_path,
            token=api_key,
            trust_remote_code=True,
        )
        
        # Set padding token to avoid warnings
        if self._tokenizer.pad_token is None:
            self._tokenizer.pad_token = self._tokenizer.eos_token
            
        # Load model with 4-bit quantization by default
        self._device_map = "auto"
        
        self._model = AutoModelForCausalLM.from_pretrained(
            self._model_path,
            return_dict=True,
            low_cpu_mem_usage=True,
            trust_remote_code=True,
            device_map=self._device_map,
            load_in_4bit=True, 
            torch_dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16,
            token=api_key,
        )
        
        # Ensure model is in evaluation mode
        self._model.eval()
                        
    def infer(self, system_prompt: str, main_context: str, should_stop: Callable[[], bool] = lambda: False) -> Iterator[str]:
        """
        Run inference using the system prompt and main context.

        Args:
            system_prompt: The system prompt string
            main_context: The main context string after templating
            should_stop: Callback that returns True when inference should stop

        Returns:
            Iterator[str]: An iterator that yields the generated text.
        """
        # Tokenize input
        inputs = self._tokenizer(system_prompt + "\n\n" + main_context, return_tensors="pt").to(self._device_map)
        
        # Create streamer for token-by-token generation
        streamer = TextIteratorStreamer(
            self._tokenizer,
            skip_prompt=True,
            timeout=15.0
        )
        
        # Generate in a separate thread to enable streaming
        generation_kwargs = {
            "input_ids": inputs.input_ids,
            "attention_mask": inputs.attention_mask,
            "max_new_tokens": self.token_limit() if self._token_limit else 2048,
            "temperature": self._temperature,
            "do_sample": True,
            "streamer": streamer,
            "repetition_penalty": 1.1,
            "pad_token_id": self._tokenizer.pad_token_id,
        }
        
        generation_thread = Thread(target=self._model.generate, kwargs=generation_kwargs)
        generation_thread.start()
        
        # Yield tokens as they become available
        try:
            for text in streamer:
                yield text
                if should_stop():
                    break
        finally:
            # Ensure thread is properly joined even if iteration is interrupted
            generation_thread.join()

    def token_count(self, system_prompt: str, main_context: str) -> int:
        """
        Count tokens for the given system prompt and main context.
        
        Args:
            system_prompt: The system prompt string
            main_context: The main context string
            
        Returns:
            int: Total number of tokens
        """
        combined_prompt = f"{system_prompt}\n\n{main_context}"
        return len(self._tokenizer.encode(combined_prompt))

    def token_limit(self) -> int:
        """
        Get the model's context window size.
        
        Returns:
            int: Maximum number of tokens the model can process
        """
        if self._token_limit is not None:
            return self._token_limit
        
        # Try to detect model size from config
        try:
            config_file = self._model_path / "config.json"
            if config_file.exists():
                import json
                with open(config_file, 'r') as f:
                    config = json.load(f)
                if 'max_position_embeddings' in config:
                    return config['max_position_embeddings']
                if 'model_max_length' in config:
                    return config['model_max_length']
        except Exception:
            pass
            
        # Default to 8k if we can't determine
        return 8192
===== FILE: sia/llm_engine/hf_llm_engine.py =====
from huggingface_hub import InferenceClient
from transformers import AutoTokenizer, AutoConfig
from typing import Iterator, Optional, Callable

from . import LlmEngine

class HfLlmEngine(LlmEngine):
    """
    LLM Engine implementation using HuggingFace's InferenceClient.
    """
    
    def __init__(
        self,
        model: str,
        temperature: float,
        api_token: Optional[str],
    ):
        """
        Initialize the HuggingFace Inference API LLM Engine.

        Args:
            model: HuggingFace model ID to use
            temperature: Sampling temperature
            api_token: HuggingFace API token
        """
        self._model = model
        self._temperature = temperature

        self._tokenizer = AutoTokenizer.from_pretrained(model, token=api_token)
        self._config = AutoConfig.from_pretrained(model, token=api_token)
        self._client = InferenceClient(token=api_token)
                        
    def infer(self, system_prompt: str, main_context: str, should_stop: Callable[[], bool] = lambda: False) -> Iterator[str]:
        """
        Run inference using the system prompt and main context.

        Args:
            system_prompt: The system prompt string
            main_context: The main context string after templating
            should_stop: Callback that returns True when inference should stop

        Returns:
            Iterator[str]: An iterator that yields the generated text.
        """
        messages = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": main_context}
        ]
        
        stream = self._client.chat_completion(
            model=self._model,
            messages=messages,
            temperature=self._temperature,
            stream=True
        )
        
        try:
            for response in stream:
                if should_stop():
                    stream.close()
                    break
                if content := response.choices[0].delta.content:
                    yield content
        finally:
            stream.close()

    def token_count(self, system_prompt: str, main_context: str) -> int:
        messages = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": main_context}
        ]
        prompt = self._tokenizer.apply_chat_template(
            messages, tokenize=False, add_generation_prompt=True
        )
        return len(self._tokenizer.encode(prompt))

    def token_limit(self) -> int:
        """
        Get the model's context window size.
        
        Returns:
            int: Maximum number of tokens the model can process
        """
        return self._config.max_position_embeddings

===== FILE: sia/llm_engine/local_llm_engine.py =====
from threading import Thread
from typing import Iterator, Optional, Callable

from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline, TextIteratorStreamer
import torch

from . import LlmEngine
from .. import util

class LocalLlmEngine(LlmEngine):
    def __init__(
        self,
        model_path: str,
        temperature: float,
        token_limit: int,
        api_token: Optional[str],
    ):
        """
        Initialize the LLM Engine with a model path.

        Args:
            model_path: Path to the model weights to be used.
            temperature: Temperature for sampling
            api_token: Huggingface API key
            token_limit: Maximum number of tokens to generate
        """
        self._temperature = temperature
        self._token_limit = token_limit
        self._tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token)
        model = AutoModelForCausalLM.from_pretrained(
            model_path,
            return_dict=True,
            low_cpu_mem_usage=True,
            torch_dtype=torch.bfloat16,
            device_map="auto",
            trust_remote_code=True,
            token=api_token,
        )
        if self._tokenizer.pad_token_id is None:
            self._tokenizer.pad_token_id = self._tokenizer.eos_token_id
        if model.config.pad_token_id is None:
            model.config.pad_token_id = model.config.eos_token_id
        self._pipeline = pipeline(
            "text-generation",
            model=model,
            tokenizer=self._tokenizer,
            torch_dtype=torch.bfloat16,
            device_map="auto",
            return_full_text=False,
        )

    def infer(self, system_prompt: str, main_context: str, should_stop: Callable[[], bool] = lambda: False) -> Iterator[str]:
        """
        Run inference using the system prompt and main context.

        Args:
            system_prompt: The system prompt string
            main_context: The main context string after templating
            should_stop: Callback that returns True when inference should stop

        Returns:
            Iterator[str]: An iterator that yields the generated text.
        """
        messages = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": main_context}
        ]
        prompt = self._tokenizer.apply_chat_template(
            messages, tokenize=False, add_generation_prompt=True
        )
        streamer = TextIteratorStreamer(
            self._tokenizer,
            skip_prompt=True
        )
        generation_thread = Thread(target=self._pipeline, kwargs=dict(
            text_inputs=prompt,
            do_sample=True,
            temperature=self._temperature,
            max_new_tokens=self.token_limit(),
            streamer=streamer
        ))
        generation_thread.start()

        for text in util.stop_before_value(streamer, '<|eot_id|>'):
            yield text
            if should_stop():
                break

        generation_thread.join()

    def token_count(self, system_prompt: str, main_context: str) -> int:
        """
        Count tokens for the given system prompt and main context.
        
        Args:
            system_prompt: The system prompt string
            main_context: The main context string
            
        Returns:
            int: Total number of tokens
        """
        messages = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": main_context}
        ]
        prompt = self._tokenizer.apply_chat_template(
            messages, tokenize=False, add_generation_prompt=True
        )
        return len(self._tokenizer.encode(prompt))

    def token_limit(self) -> int:
        """
        Get the model's context window size.
        
        Returns:
            int: Maximum number of tokens the model can process
        """
        if self._token_limit is not None:
            return self._token_limit
        else:
            return self._pipeline.model.config.max_position_embeddings

===== FILE: sia/llm_engine/mistral_llm_engine.py =====
from typing import Iterator, Optional, Callable
from mistralai import Mistral
from mistral_common.protocol.instruct.messages import SystemMessage, UserMessage
from mistral_common.protocol.instruct.request import ChatCompletionRequest
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer

from . import LlmEngine

class MistralLlmEngine(LlmEngine):
    def __init__(
            self,
            model: str,
            temperature: float,
            token_limit: int,
            api_key: str,
        ):
        self._model = model
        self._temperature = temperature
        self._token_limit = token_limit
        self._api_key = api_key
        self._client = Mistral(api_key=api_key)
        self._tokenizer = MistralTokenizer.v3()

    def infer(self, system_prompt: str, main_context: str, should_stop: Callable[[], bool] = lambda: False) -> Iterator[str]:
        messages = [
            {
                "role": "system",
                "content": system_prompt,
            },
            {
                "role": "user",
                "content": main_context,
            },
            {
                "role": "assistant",
                "content": "<",
                "prefix": True,
            },
        ]
        stream_response = self._client.chat.stream(
            model=self._model,
            messages=messages,
            temperature=self._temperature,
        )
        
        try:
            for chunk in stream_response:
                if should_stop():
                    stream_response.response.close()
                    break
                if content := chunk.data.choices[0].delta.content:
                    yield content
        finally:
            stream_response.response.close()

    def token_count(self, system_prompt: str, main_context: str) -> int:
        messages = [
            SystemMessage(content=system_prompt),
            UserMessage(content=main_context),
        ]

        tokenized = self._tokenizer.encode_chat_completion(
            ChatCompletionRequest(
                messages=messages,
                model=self._model
            )
        )
        return len(tokenized.tokens)

    def token_limit(self) -> int:
        return self._token_limit

===== FILE: sia/llm_engine/openai_llm_engine.py =====
from typing import Callable, Iterator
import openai
import tiktoken

from . import LlmEngine

class OpenAILlmEngine(LlmEngine):
    """
    LLM Engine implementation using OpenAI's API.
    Supports streaming responses from chat completion models.
    """
    
    def __init__(
        self,
        model: str,
        temperature: float,
        token_limit: int,
        api_key: str,
    ):
        """
        Initialize the OpenAI LLM Engine.

        Args:
            model: OpenAI model to use
            temperature: Temperature for sampling
            api_key: OpenAI API key
            token_limit: Maximum number of tokens to generate
        """
        self._model = model
        self._temperature = temperature
        self._token_limit = token_limit

        self._client = openai.Client(
            api_key=api_key,
        )
                        
    def infer(self, system_prompt: str, main_context: str, should_stop: Callable[[], bool] = lambda: False) -> Iterator[str]:
        messages = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": main_context}
        ]
        
        stream = self._client.chat.completions.create(
            model=self._model,
            messages=messages,
            temperature=self._temperature,
            stream=True,
        )

        try:
            for chunk in stream:
                if should_stop():
                    break
                if content := chunk.choices[0].delta.content:
                    yield content
        finally:
            stream.close()
            #stream.response.close()
    
    def token_count(self, system_prompt: str, main_context: str) -> int:
        """
        Calculate the total token count for the system prompt and context.
        
        Args:
            system_prompt: The system prompt string
            main_context: The main context string
            
        Returns:
            int: Total number of tokens
        """
        encoding = tiktoken.encoding_for_model(self._model)
        return len(encoding.encode(system_prompt)) + len(encoding.encode(main_context))
    
    def token_limit(self) -> int:
        return self._token_limit
===== FILE: sia/response_parser.py =====
from datetime import datetime
from pathlib import Path
import xml.etree.ElementTree as ET
from typing import Union

from .util import format_timestamp

from .command import Command 
from .delete_command import DeleteCommand
from .stop_command import StopCommand

from .entry.background_entry import BackgroundEntry
from .entry import Entry
from .io_buffer import IOBuffer
from .entry.parse_error_entry import ParseErrorEntry
from .entry.read_entry import ReadEntry
from .entry.reasoning_entry import ReasoningEntry
from .entry.repeat_entry import RepeatEntry
from .entry.single_entry import SingleEntry
from .entry.write_entry import WriteEntry

class ResponseParser:
    """
    Parses XML responses from the LLM into commands or entries.
    
    The parser validates the XML structure and converts it into the appropriate
    Command or Entry object based on the root element tag. Invalid input results
    in ParseErrorEntry objects rather than raising exceptions.
    
    Attributes:
        io_buffer: Buffer to use for IO operations
    """
    
    def __init__(self, work_dir: Path, io_buffer: IOBuffer):
        """
        Initialize parser with IO buffer.
        
        Args:
            work_dir: Workdir for the scripts
            io_buffer: Buffer to use for IO operations
        """
        self._work_dir = work_dir
        self._io_buffer = io_buffer
    
    @property
    def io_buffer(self) -> IOBuffer:
        """
        Get the IO buffer used by the parser.

        Returns:
            IOBuffer: Buffer used for IO operations
        """
        return self._io_buffer
        
    def parse(self, timestamp: datetime, xml: str) -> Union[Command, Entry]:
        """
        Parse XML response into a Command or Entry.
        
        Args:
            xml: XML string to parse
            
        Returns:
            Command or Entry based on the XML content
            ParseErrorEntry for any parsing or validation errors
        """
        entry_id = format_timestamp(timestamp)
        
        parser = ET.XMLPullParser(events=("start", "end"))
        parser.feed(xml)
        root = None
        try:
            for event, root in parser.read_events():
                if event == "start":
                    break
        except ET.ParseError as e:
            return ParseErrorEntry(entry_id, xml, f"Invalid XML: {str(e)}")
        
        try:
            if root.tag == 'delete':
                target_id = root.get('id')
                if not target_id:
                    return ParseErrorEntry(entry_id, xml, "Delete command missing required 'id' attribute")
                if len(root.attrib) > 1:
                    return ParseErrorEntry(entry_id, xml, "Delete command should only have 'id' attribute")
                return DeleteCommand(target_id)

            elif root.tag == 'stop':
                return StopCommand()
                
            elif root.tag == 'background':
                if len(root) != 0 or root.attrib or root.text is None or root.text.strip() == '':
                    return ParseErrorEntry(entry_id, xml, "Background entry requires (only) script content")
                return BackgroundEntry(entry_id, self._work_dir, root.text)
                
            elif root.tag == 'repeat':
                if len(root) != 0 or root.text is None or root.text.strip() == '':
                    return ParseErrorEntry(entry_id, xml, "Repeat entry requires (only) script content")
                if len(root.attrib) > 2 or any(k not in ('timeout', 'limit') for k in root.attrib):
                    return ParseErrorEntry(entry_id, xml, "Repeat entry only accepts 'timeout' and 'limit' attributes")
                timeout = root.get('timeout')
                limit = root.get('limit')
                timeout = float(timeout) if timeout is not None else None
                limit = int(limit) if limit is not None else None
                return RepeatEntry(entry_id, self._work_dir, root.text, timeout, limit)
                
            elif root.tag == 'single':
                if len(root) != 0 or root.text is None or root.text.strip() == '':
                    return ParseErrorEntry(entry_id, xml, "Single entry requires (only) script content")
                if len(root.attrib) > 2 or any(k not in ('timeout', 'limit') for k in root.attrib):
                    return ParseErrorEntry(entry_id, xml, "Single entry only accepts 'timeout' and 'limit' attributes")
                timeout = root.get('timeout')
                limit = root.get('limit')
                timeout = float(timeout) if timeout is not None else None
                limit = int(limit) if limit is not None else None
                return SingleEntry(entry_id, self._work_dir, root.text, timeout, limit)

            elif root.tag == 'reasoning':
                if len(root) != 0 or root.attrib or root.text is None or root.text.strip() == '':
                    return ParseErrorEntry(entry_id, xml, "Reasoning entry requires (only) text content")
                return ReasoningEntry(entry_id, root.text)
                
            elif root.tag == 'read_stdin':
                return ReadEntry(entry_id, self._io_buffer)
                
            elif root.tag == 'write_stdout':
                if len(root) != 0 or root.attrib or root.text is None or root.text.strip() == '':
                    return ParseErrorEntry(entry_id, xml, "Write stdout entry requires (only) text content")
                return WriteEntry(entry_id, root.text, self._io_buffer)
                
            else:
                return ParseErrorEntry(entry_id, xml, f"Unknown root element: {root.tag}")
                
        except Exception as e:
            return ParseErrorEntry(entry_id, xml, f"Error parsing response: {str(e)}")

===== FILE: sia/standard_io_buffer.py =====
import sys

from .io_buffer import IOBuffer

class StandardIOBuffer(IOBuffer):
    """
    IOBuffer implementation that uses system standard input/output.
    
    This class provides direct access to stdin/stdout for IO operations.
    It implements a basic line-based input buffer to handle cases where
    multiple reads are needed to process all available input.
    """
    
    def __init__(self):
        """Initialize the standard IO buffer."""
        self._input_buffer: str = ""

    def read(self) -> str:
        """
        Read available input from stdin.
        
        If there is buffered input from a previous read, return that first.
        Otherwise, try to read new input from stdin if available.
        
        Returns:
            str: Content read from stdin, or empty string if no input available
        """
        # Return and clear any existing buffered input
        if self._input_buffer:
            content = self._input_buffer
            self._input_buffer = ""
            return content
            
        # Check if there's input available
        if not sys.stdin.isatty() and sys.stdin.readable():
            try:
                content = sys.stdin.read()
                if content:
                    return content
            except Exception:
                pass  # Ignore any read errors
                
        return ""
        
    def write(self, content: str) -> None:
        """
        Write content to stdout.
        
        Args:
            content: String content to write
        """
        if not content:
            return
            
        try:
            sys.stdout.write(content)
            sys.stdout.flush()
        except Exception:
            pass  # Ignore write errors
            
    def buffer_length(self) -> int:
        """
        Get the current length of buffered input.
        
        Returns:
            int: Number of characters in the input buffer
        """
        return len(self._input_buffer)
===== FILE: sia/stop_command.py =====
from .command import Command
from .command_result import CommandResult
from .working_memory import WorkingMemory

class StopCommand(Command):
    """
    Command to stop the agent.
    Performs cleanup on all entries and clears working memory.
    """
    
    def execute(self, memory: WorkingMemory) -> CommandResult:
        """
        Signal that the agent should stop.
        Cleans up all entries and clears the working memory.
        
        Args:
            memory: WorkingMemory instance to clear
            
        Returns:
            CommandResult: Stop result
        """
        # Get a copy of entries to iterate over
        entries = memory.get_entries()
        
        # Clean up each entry individually
        for entry in entries:
            entry.cleanup()
            memory.remove_entry(entry.id)
            
        return CommandResult.stop()
===== FILE: sia/system_metrics.py =====
import time
from typing import Dict
import psutil
from .util import format_timestamp

class SystemMetrics:
    """
    Tracks system metrics including memory and disk usage.
    
    """

    def get_metrics(self) -> Dict:
        """
        Get current system metrics.
        Clears usage samples after calculating averages.
        
        Returns:
            Dict containing:
                - timestamp: Current timestamp
                - memory_used: Used memory in bytes
                - memory_total: Total memory in bytes
                - disk_used: Used disk space in bytes
                - disk_total: Total disk space in bytes
        """
        metrics = {}
        
        # Add timestamp
        metrics["timestamp"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
        
        # Memory usage in bytes
        memory = psutil.virtual_memory()
        metrics["memory_used"] = memory.used
        metrics["memory_total"] = memory.total
        
        # Disk usage in bytes (root partition)
        disk = psutil.disk_usage('/')
        metrics["disk_used"] = disk.used
        metrics["disk_total"] = disk.total
        
        return metrics
===== FILE: sia/util.py =====
import datetime
import xml.dom.minidom
import xml.etree.ElementTree as ET
from typing import Iterator, Optional

def stop_before_value(iterator: Iterator[str], stop_value: str) -> Iterator[str]:
    """
    Creates an iterator that yields values from the input iterator
    until it encounters the stop_value (exclusive).
    
    Args:
        iterator: The source iterator
        stop_value: The value to stop before
        
    Yields:
        Values from the iterator until stop_value is encountered
        If stop_value is part of an item, yields the part before stop_value
    """
    for item in iterator:
        if stop_value in item:
            split_point = item.index(stop_value)
            if split_point > 0:
                yield item[:split_point]
            break
        yield item

def pretty_print_element(elem: ET.Element, level: int = 0, max_line: int = 80) -> str:
    """Convert ElementTree element to pretty-printed string with custom formatting."""
    indent = '    ' * level
    
    # Handle empty elements
    if not elem.text and not elem.tail and not elem.attrib and len(elem) == 0:
        return f'{indent}<{elem.tag}/>'
        
    # Build opening tag with attributes
    tag = elem.tag
    attrs = elem.attrib
    if attrs:
        attr_strings = [f'{k}="{v}"' for k, v in sorted(attrs.items())]
        attr_str = ' ' + ' '.join(attr_strings)
        
        if len(indent) + len(tag) + len(attr_str) + 2 > max_line:
            attr_indent = indent + '    '
            attr_str = '\n' + '\n'.join(f'{attr_indent}{a}' for a in attr_strings)
    else:
        attr_str = ''
    
    parts = [f'{indent}<{tag}{attr_str}>']
    
    # Handle text content
    if elem.text and isinstance(elem.text, str) and elem.text.strip():
        text = elem.text
        if ']]>' in text:
            escaped = text.replace('&', '&amp;') \
                        .replace('<', '&lt;') \
                        .replace('>', '&gt;') \
                        .replace('"', '&quot;') \
                        .replace("'", '&apos;')
            parts.append(f'{indent}    {escaped}')
        else:
            parts.append(f'{indent}    <![CDATA[{text}]]>')
    
    # Handle children
    for child in elem:
        parts.append(pretty_print_element(child, level + 1, max_line))
        if child.tail and child.tail.strip():
            parts.append(f'{indent}    {child.tail}')
            
    parts.append(f'{indent}</{tag}>')
    return '\n'.join(parts)

def format_timestamp(timestamp: datetime) -> str:
        return timestamp.strftime("%Y%m%d_%H%M%S_%f")[:-3]
===== FILE: sia/web/api.py =====
from pathlib import Path
from aiohttp import web
import json
import asyncio


from ..auto_approver import AutoApprover
from ..entry.entry_factory import EntryFactory
from ..iteration_parser import IterationParser
from ..web_agent import WebAgent
from ..web_io_buffer import WebIOBuffer
from ..working_memory import WorkingMemory

class Api:
    def __init__(
            self,
            work_dir: Path,
            app: web.Application,
            agent: WebAgent,
            io_buffer: WebIOBuffer,
            working_memory: WorkingMemory,
            auto_approver: AutoApprover
        ):
        self._work_dir = work_dir
        self._app = app
        self._agent = agent
        self._working_memory = working_memory
        self._io_buffer = io_buffer
        self._auto_approver = auto_approver
        
        self._init_routes()
        
    def _init_routes(self):
        """Initialize REST API and WebSocket routes."""
        self._app.router.add_post("/api/inference/{llm}", self._run_inference)
        self._app.router.add_post("/api/inference/{llm}/stop", self._stop_inference)
        self._app.router.add_post("/api/approve/{llm}", self._approve_response)
        self._app.router.add_post("/api/context", self._modify_context)
        self._app.router.add_post("/api/input", self._send_input)
        self._app.router.add_post("/api/clear", self._clear_output)
        self._app.router.add_get("/api/output/{llm}", self._get_output)
        self._app.router.add_get("/api/llms", self._get_llms)
        self._app.router.add_get("/api/auto_approver/config", self._get_auto_approver_config)
        self._app.router.add_post("/api/auto_approver/config", self._set_auto_approver_config)
        self._app.router.add_post("/api/auto_approver/context_enabled", self._set_context_enabled)
        self._app.router.add_post("/api/auto_approver/response_enabled", self._set_response_enabled) 
        self._app.router.add_post("/api/auto_approver/context_timeout", self._set_context_timeout)
        self._app.router.add_post("/api/auto_approver/response_timeout", self._set_response_timeout)
        self._app.router.add_post("/api/auto_approver/llm", self._set_llm_name)
        self._app.router.add_get("/api/memory", self._get_memory)
        self._app.router.add_post("/api/memory/entry", self._create_entry)
        self._app.router.add_put("/api/memory/entry/{id}", self._save_entry)
        self._app.router.add_delete("/api/memory/entry/{id}", self._delete_entry)
        self._app.router.add_post("/api/memory/entry/{id}/reset", self._reset_entry)
        self._app.router.add_post("/api/memory/entry/{id}/update", self._update_entry)
        self._app.router.add_post("/api/memory/load_iteration", self._load_iteration)

    async def _run_inference(self, request: web.Request) -> web.Response:
        """Start inference on specified LLM."""
        llm_name = request.match_info["llm"]
        try:
            await asyncio.get_event_loop().run_in_executor(None, self._agent.run_inference, llm_name)
            return web.Response(status=200)
        except (ValueError, RuntimeError) as e:
            return web.Response(status=400, text=str(e))

    async def _stop_inference(self, request: web.Request) -> web.Response:
        """Stop inference on specified LLM."""
        llm_name = request.match_info["llm"]
        try:
            self._agent.stop_inference(llm_name)
            return web.Response(status=200)
        except ValueError as e:
            return web.Response(status=400, text=str(e))

    async def _approve_response(self, request: web.Request) -> web.Response:
        """Approve response from specified LLM."""
        llm_name = request.match_info["llm"]
        data = await request.json()
        response = data.get("response")
        if not response:
            return web.Response(status=400, text="Missing response in request body")
        try:
            self._agent.approve_response(llm_name, response)
            return web.Response(status=200)
        except ValueError as e:
            return web.Response(status=400, text=str(e))

    async def _modify_context(self, request: web.Request) -> web.Response:
        """Modify the current context."""
        data = await request.json()
        context = data.get("context")
        if not context:
            return web.Response(status=400, text="Missing context in request body")
        self._agent.modify_context(context)
        return web.Response(status=200)

    async def _send_input(self, request: web.Request) -> web.Response:
        """Send input to the IO buffer."""
        data = await request.json()
        input_text = data.get("input")
        if not input_text:
            return web.Response(status=400, text="Missing input in request body")
        self._io_buffer.append_stdin(input_text)
        return web.Response(status=200)

    async def _clear_output(self, request: web.Request) -> web.Response:
        """Clear the stdout buffer."""
        self._io_buffer.clear_stdout()
        return web.Response(status=200)

    async def _get_output(self, request: web.Request) -> web.Response:
        """Get complete output for specified LLM."""
        llm_name = request.match_info["llm"]
        try:
            output = self._agent.get_output(llm_name)
            return web.Response(
                text=json.dumps({"output": output}),
                content_type="application/json"
            )
        except ValueError as e:
            return web.Response(status=400, text=str(e))

    async def _get_llms(self, request: web.Request) -> web.Response:
        """Get all LLMs and their current states."""
        states = self._agent.llms
        return web.Response(
            text=json.dumps({
                name: state.name 
                for name, state in states.items()
            }),
            content_type="application/json"
        )

    async def _get_auto_approver_config(self, request: web.Request) -> web.Response:
        """Get current auto approver configuration."""
        return web.Response(
            text=json.dumps(self._auto_approver.config),
            content_type="application/json"
        )

    async def _set_auto_approver_config(self, request: web.Request) -> web.Response:
        """Update auto approver configuration."""
        data = await request.json()
        try:
            self._auto_approver.set_config(data)
            return web.Response(status=200)
        except (ValueError, KeyError) as e:
            return web.Response(status=400, text=str(e))

    async def _set_context_enabled(self, request: web.Request) -> web.Response:
        """Set context auto-approval enabled state."""
        data = await request.json()
        enabled = data.get("enabled")
        if enabled is None:
            return web.Response(status=400, text="Missing enabled parameter")
        try:
            self._auto_approver.context_enabled = enabled
            return web.Response(status=200)
        except ValueError as e:
            return web.Response(status=400, text=str(e))

    async def _set_response_enabled(self, request: web.Request) -> web.Response:
        """Set response auto-approval enabled state."""
        data = await request.json()
        enabled = data.get("enabled")
        if enabled is None:
            return web.Response(status=400, text="Missing enabled parameter")
        try:
            self._auto_approver.response_enabled = enabled
            return web.Response(status=200)
        except ValueError as e:
            return web.Response(status=400, text=str(e))

    async def _set_context_timeout(self, request: web.Request) -> web.Response:
        """Set context auto-approval timeout."""
        data = await request.json()
        timeout = data.get("timeout")
        if timeout is None:
            return web.Response(status=400, text="Missing timeout parameter")
        try:
            self._auto_approver.context_timeout = float(timeout)
            return web.Response(status=200)
        except ValueError as e:
            return web.Response(status=400, text=str(e))

    async def _set_response_timeout(self, request: web.Request) -> web.Response:
        """Set response auto-approval timeout."""
        data = await request.json()
        timeout = data.get("timeout")
        if timeout is None:
            return web.Response(status=400, text="Missing timeout parameter")
        try:
            self._auto_approver.response_timeout = float(timeout)
            return web.Response(status=200)
        except ValueError as e:
            return web.Response(status=400, text=str(e))

    async def _set_llm_name(self, request: web.Request) -> web.Response:
        """Set LLM name for auto-approval."""
        data = await request.json()
        name = data.get("name")
        if name is None:
            return web.Response(status=400, text="Missing name parameter")
        try:
            self._auto_approver.llm_name = name
            return web.Response(status=200)
        except ValueError as e:
            return web.Response(status=400, text=str(e))

    async def _get_memory(self, request: web.Request) -> web.Response:
        """Get complete working memory state."""
        entries = self._working_memory.get_entries()
        return web.Response(
            text=json.dumps([e.serialize() for e in entries]),
            content_type="application/json"
        )

    async def _create_entry(self, request: web.Request) -> web.Response:
        """Create a new entry in working memory."""
        data = await request.json()
        try:
            entry = EntryFactory.create_entry(data, self._work_dir, self._io_buffer)
            self._working_memory.add_entry(entry)
            return web.Response(
                text=json.dumps({"id": entry.id}),
                content_type="application/json"
            )
        except (ValueError, TypeError) as e:
            return web.Response(status=400, text=str(e))

    async def _save_entry(self, request: web.Request) -> web.Response:
        """Update properties of an existing entry."""
        entry_id = request.match_info["id"]
        data = await request.json()
        entry = self._working_memory.get_entry(entry_id)
        if not entry:
            return web.Response(status=404, text="Entry not found")
        try:
            EntryFactory.update_entry(entry, data)
            entry.notify_change()
            return web.Response(status=200)
        except ValueError as e:
            return web.Response(status=400, text=str(e))

    async def _delete_entry(self, request: web.Request) -> web.Response:
        """Delete an entry from working memory."""
        entry_id = request.match_info["id"]
        self._working_memory.remove_entry(entry_id)
        return web.Response(status=200)

    async def _reset_entry(self, request: web.Request) -> web.Response:
        """Reset an entry's state."""
        entry_id = request.match_info["id"]
        entry = self._working_memory.get_entry(entry_id)
        if not entry:
            return web.Response(status=404, text="Entry not found")
        entry.reset()
        return web.Response(status=200)
    
    async def _update_entry(self, request: web.Request) -> web.Response:
        """Update an entry's state."""
        entry_id = request.match_info["id"]
        entry = self._working_memory.get_entry(entry_id)
        if not entry:
            return web.Response(status=404, text="Entry not found")
        try:
            entry.update()
            entry.notify_change()
            return web.Response(status=200)
        except ValueError as e:
            return web.Response(status=400, text=str(e))

    async def _load_iteration(self, request: web.Request) -> web.Response:
        """Load entries from iteration XML content into working memory"""
        data = await request.json()
        content = data.get("content")
        if not content:
            return web.Response(status=400, text="Missing content in request body")
            
        entries = IterationParser.parse_iteration(content, self._work_dir, self._io_buffer)
        
        for entry in entries:
            self._working_memory.add_entry(entry)
            
        return web.Response(status=200)

===== FILE: sia/web/auto_approver_websocket.py =====
from aiohttp import web, WSMsgType
from typing import Dict, Set

from ..auto_approver import AutoApprover, AutoApproverConfig
from .util import wrap_async

class AutoApproverWebSocket:
    """
    WebSocket handler for AutoApprover configuration.
    Broadcasts config updates to all connected clients.
    """
    
    def __init__(self, auto_approver: AutoApprover):
        self._auto_approver = auto_approver
        self._clients: Set[web.WebSocketResponse] = set()
        self._auto_approver.add_config_change_handler(wrap_async(self._handle_config_change))
        
    async def _broadcast_message(self, message: Dict):
        """Broadcast message to all connected clients."""
        disconnected = set()
        for ws in self._clients:
            try:
                await ws.send_json(message)
            except ConnectionResetError:
                disconnected.add(ws)
        self._clients -= disconnected

    async def _handle_config_change(self, config: AutoApproverConfig):
        """Handle config changes from the AutoApprover."""
        await self._broadcast_message({
            "config": config
        })

    async def handle_connection(self, request: web.Request) -> web.WebSocketResponse:
        """Handle new WebSocket connections."""
        ws = web.WebSocketResponse(heartbeat=30)
        await ws.prepare(request)
        
        self._clients.add(ws)
        
        try:
            # Send initial config
            await ws.send_json({
                "config": self._auto_approver.config
            })
                
            async for msg in ws:
                if msg.type == WSMsgType.ERROR:
                    print(f"WebSocket connection closed with error: {ws.exception()}")
        finally:
            self._clients.remove(ws)
            
        return ws

===== FILE: sia/web/context_websocket.py =====
from aiohttp import web, WSMsgType
from typing import Dict, Set

from .util import wrap_async
from ..web_agent import WebAgent

class ContextWebSocket:
    """
    WebSocket handler for context changes.
    Broadcasts context updates to all connected clients.
    """
    
    def __init__(self, agent: WebAgent):
        self._agent = agent
        self._clients: Set[web.WebSocketResponse] = set()
        self._agent.add_context_change_handler(wrap_async(self._handle_context_change))
        
    async def _broadcast_message(self, message: Dict):
        """Broadcast message to all connected clients."""
        disconnected = set()
        for ws in self._clients:
            try:
                await ws.send_json(message)
            except ConnectionResetError:
                disconnected.add(ws)
        self._clients -= disconnected

    async def _handle_context_change(self, context: str, generated: bool):
        """Handle context changes from the WebAgent."""
        await self._broadcast_message({
            "context": context,
            "generated": generated
        })

    async def handle_connection(self, request: web.Request) -> web.WebSocketResponse:
        """Handle new WebSocket connections."""
        ws = web.WebSocketResponse(heartbeat=30)
        await ws.prepare(request)
        
        self._clients.add(ws)
        
        try:
            # Send initial context
            await ws.send_json({
                "context": self._agent.context,
                "generated": True
            })
                
            async for msg in ws:
                if msg.type == WSMsgType.ERROR:
                    print(f"WebSocket connection closed with error: {ws.exception()}")
        finally:
            self._clients.remove(ws)
            
        return ws

===== FILE: sia/web/llm_websocket.py =====
from aiohttp import web, WSMsgType
from typing import Dict, Set

from .util import wrap_async
from ..web_agent import WebAgent, LlmState

class LlmWebSocket:
    """
    WebSocket handler for LLM state changes.
    Broadcasts state updates to all connected clients.
    """
    
    def __init__(self, agent: WebAgent):
        self._agent = agent
        self._clients: Set[web.WebSocketResponse] = set()
        self._agent.add_llm_change_handler(wrap_async(self._handle_state_change))
        
    async def _broadcast_message(self, message: Dict):
        """Broadcast message to all connected clients."""
        disconnected = set()
        for ws in self._clients:
            try:
                await ws.send_json(message)
            except ConnectionResetError:
                disconnected.add(ws)
        self._clients -= disconnected

    async def _handle_state_change(self, llm_name: str, new_state: LlmState):
        """Handle state changes from the WebAgent."""
        await self._broadcast_message({
            "llm": llm_name,
            "state": new_state.name
        })

    async def handle_connection(self, request: web.Request) -> web.WebSocketResponse:
        """Handle new WebSocket connections."""
        ws = web.WebSocketResponse(heartbeat=30)
        await ws.prepare(request)
        
        try:
            # Send initial states for all LLMs
            states = self._agent.llms
            for llm_name, state in states.items():
                await ws.send_json({
                    "llm": llm_name,
                    "state": state.name
                })
        
            self._clients.add(ws)
                
            async for msg in ws:
                if msg.type == WSMsgType.ERROR:
                    print(f"WebSocket connection closed with error: {ws.exception()}")
        finally:
            self._clients.remove(ws)
            
        return ws
===== FILE: sia/web/memory_websocket.py =====
from aiohttp import web, WSMsgType
from typing import Dict, Set

from ..entry import Entry
from ..web_agent import WebAgent
from ..working_memory import WorkingMemory
from .util import wrap_async

class MemoryWebSocket:
    """
    WebSocket handler for working memory changes.
    Broadcasts memory updates to all connected clients.
    """
    
    def __init__(self, working_memory: WorkingMemory):
        self._working_memory = working_memory
        self._clients: Set[web.WebSocketResponse] = set()
        self._working_memory.add_change_handler(wrap_async(self._handle_memory_change))
        
    async def _broadcast_message(self, message: Dict):
        """Broadcast message to all connected clients."""
        disconnected = set()
        for ws in self._clients:
            try:
                await ws.send_json(message)
            except ConnectionResetError:
                disconnected.add(ws)
        self._clients -= disconnected

    async def _handle_memory_change(self):
        """Handle working memory changes."""
        entries = self._working_memory.get_entries()
        await self._broadcast_message({
            "type": "memory_state",
            "entries": [e.serialize() for e in entries]
        })

    async def handle_connection(self, request: web.Request) -> web.WebSocketResponse:
        """Handle new WebSocket connections."""
        ws = web.WebSocketResponse(heartbeat=30)
        await ws.prepare(request)
        
        try:
            # Send initial memory state
            entries = self._working_memory.get_entries()
            await ws.send_json({
                "type": "memory_state",
                "entries": [e.serialize() for e in entries]
            })
        
            self._clients.add(ws)
                
            async for msg in ws:
                if msg.type == WSMsgType.ERROR:
                    print(f"WebSocket connection closed with error: {ws.exception()}")
        finally:
            self._clients.remove(ws)
            
        return ws

===== FILE: sia/web/static.py =====
from aiohttp import web
import mimetypes

from ..config import Config

mimetypes.add_type("application/javascript", ".js")
mimetypes.add_type("application/javascript", ".jsx")
mimetypes.add_type("text/javascript", ".js")
mimetypes.add_type("text/javascript", ".jsx")

class Static:
    def __init__(self, app: web.Application, config: Config):
        self._config = config

        app.router.add_get("/", self._serve_index)
        app.router.add_static("/static/", self._config.static_files, show_index=False)
        app.router.add_static("/assets/", self._config.static_files / "assets", show_index=False)
        app.router.add_get("/{path:.*}", self._serve_index)

    async def _serve_index(self, request: web.Request) -> web.Response:
        """Serve the React application HTML for any unmatched routes."""
        index_path = self._config.static_files / "index.html"
        if not index_path.exists():
            raise web.HTTPNotFound()

        with open(index_path, "r") as f:
            html_content = f.read()

        return web.Response(
            text=html_content,
            content_type="text/html"
        )
===== FILE: sia/web/stdout_websocket.py =====
from aiohttp import web, WSMsgType
from typing import Dict, Set
import asyncio

from .util import wrap_async
from ..web_io_buffer import WebIOBuffer

class StdoutWebSocket:
    """
    WebSocket handler for stdout changes.
    Broadcasts stdout updates to all connected clients.
    """
    
    def __init__(self, io_buffer: WebIOBuffer):
        self._io_buffer = io_buffer
        self._clients: Set[web.WebSocketResponse] = set()
        self._io_buffer.add_stdout_change_handler(wrap_async(self._handle_stdout_change))
        
    async def _broadcast_message(self, message: Dict):
        """Broadcast message to all connected clients."""
        disconnected = set()
        for ws in self._clients:
            try:
                await ws.send_json(message)
            except ConnectionResetError:
                disconnected.add(ws)
        self._clients -= disconnected

    async def _handle_stdout_change(self, output: str):
        """Handle stdout changes from the WebIOBuffer."""
        await self._broadcast_message({
            "output": output
        })

    async def handle_connection(self, request: web.Request) -> web.WebSocketResponse:
        """Handle new WebSocket connections."""
        ws = web.WebSocketResponse(heartbeat=30)
        await ws.prepare(request)
        
        self._clients.add(ws)
        
        try:
            # Send initial stdout content
            await ws.send_json({
                "output": self._io_buffer.get_stdout()
            })
                
            async for msg in ws:
                if msg.type == WSMsgType.ERROR:
                    print(f"WebSocket connection closed with error: {ws.exception()}")
        finally:
            self._clients.remove(ws)
            
        return ws

===== FILE: sia/web/token_websocket.py =====
from aiohttp import web, WSMsgType
from typing import Dict, Set
import asyncio
import json

from .util import wrap_async
from ..web_agent import WebAgent

class TokenWebSocket:
    """
    WebSocket handler for LLM token streaming.
    Broadcasts new tokens to all connected clients.
    """
    
    def __init__(self, agent: WebAgent):
        self._agent = agent
        self._clients: Set[web.WebSocketResponse] = set()
        self._agent.add_token_handler(wrap_async(self._handle_new_token))
        
    async def _broadcast_message(self, message: Dict):
        """Broadcast message to all connected clients."""
        disconnected = set()
        for ws in self._clients:
            try:
                await ws.send_json(message)
            except ConnectionResetError:
                disconnected.add(ws)
        self._clients -= disconnected

    async def _handle_new_token(self, llm_name: str, token: str):
        """Handle new tokens from the WebAgent."""
        await self._broadcast_message({
            "llm": llm_name,
            "token": token
        })

    async def handle_connection(self, request: web.Request) -> web.WebSocketResponse:
        """Handle new WebSocket connections."""
        ws = web.WebSocketResponse(heartbeat=30)
        await ws.prepare(request)
        
        self._clients.add(ws)
        
        try:
            async for msg in ws:
                if msg.type == WSMsgType.ERROR:
                    print(f"WebSocket connection closed with error: {ws.exception()}")
        finally:
            self._clients.remove(ws)
            
        return ws

===== FILE: sia/web/util.py =====
import asyncio

def wrap_async(coro_func):
    """Wraps an async callback to be safely called from another thread."""
    loop = asyncio.get_event_loop()
    def wrapper(*args, **kwargs):
        loop.call_soon_threadsafe(lambda: asyncio.create_task(coro_func(*args, **kwargs)))
    return wrapper
===== FILE: sia/web/websockts.py =====
from aiohttp import web

from ..auto_approver import AutoApprover
from ..web_agent import WebAgent
from ..web_io_buffer import WebIOBuffer
from ..working_memory import WorkingMemory
from .auto_approver_websocket import AutoApproverWebSocket
from .context_websocket import ContextWebSocket 
from .llm_websocket import LlmWebSocket
from .memory_websocket import MemoryWebSocket
from .stdout_websocket import StdoutWebSocket
from .token_websocket import TokenWebSocket

class Websockets:
    def __init__(self, app: web.Application, agent: WebAgent, io_buffer: WebIOBuffer, auto_approver: AutoApprover, working_memory: WorkingMemory):
        self._llm_ws = LlmWebSocket(agent)
        self._context_ws = ContextWebSocket(agent)
        self._token_ws = TokenWebSocket(agent)
        self._stdout_ws = StdoutWebSocket(io_buffer)
        self._auto_approver_ws = AutoApproverWebSocket(auto_approver)
        self._memory_ws = MemoryWebSocket(working_memory)

        app.router.add_get("/ws/llm", self._llm_ws.handle_connection)
        app.router.add_get("/ws/context", self._context_ws.handle_connection)
        app.router.add_get("/ws/token", self._token_ws.handle_connection)
        app.router.add_get("/ws/stdout", self._stdout_ws.handle_connection)
        app.router.add_get("/ws/auto_approver", self._auto_approver_ws.handle_connection)
        app.router.add_get("/ws/memory", self._memory_ws.handle_connection)

===== FILE: sia/web_agent.py =====
from collections import defaultdict
from datetime import datetime, timezone
from enum import Enum, auto
from sys import exit
from threading import Lock
from typing import Callable, Dict, List, Optional

from .base_agent import BaseAgent
from .command import Command
from .command_result import CommandResult
from .iteration_logger import IterationLogger
from .llm_engine import LlmEngine
from .response_parser import ResponseParser
from .system_metrics import SystemMetrics
from .working_memory import WorkingMemory
from .xml_validator import XMLValidator

class LlmState(Enum):
    NO_OUTPUT = auto()
    INFERENCE = auto()
    OUTPUT = auto()

class WebAgent(BaseAgent):
    def __init__(
            self,
            system_prompt: str,
            action_schema: str, 
            working_memory: WorkingMemory,
            metrics: SystemMetrics,
            llms: Dict[str, LlmEngine],
            validator: XMLValidator,
            parser: ResponseParser,
            iteration_logger: IterationLogger,
        ):
        super().__init__(
            system_prompt,
            action_schema,
            working_memory,
            metrics,
            validator,
            parser
        )
        self._llms = llms
        self._iteration_logger = iteration_logger
        self._llm_states: Dict[str, LlmState] = {name: LlmState.NO_OUTPUT for name in llms}
        self._llm_outputs: Dict[str, str] = defaultdict(str)
        self._validation_error: Optional[str] = None
        self._command_result: Optional[CommandResult] = None
        self._context = self._compile_context(next(iter(self._llms.values())))
        self._stop_flags: Dict[str, bool] = {name: False for name in llms}
        
        # Locks
        self._llm_lock = Lock()
        self._output_lock = Lock()
        
        # Event handlers
        self._llm_change_handlers: List[Callable[[str, LlmState], None]] = []
        self._token_handlers: List[Callable[[str, str], None]] = []
        self._context_change_handlers: List[Callable[[str, bool], None]] = []

        # Working memory change handler
        self._working_memory.add_change_handler(self._handle_memory_update)

    @property
    def llms(self) -> Dict[str, LlmState]:
        """Get current state of all LLMs"""
        with self._llm_lock:
            return self._llm_states.copy()

    @property
    def context(self) -> str:
        return self._context

    @property
    def command_result(self) -> Optional[CommandResult]:
        return self._command_result

    @property
    def validation_error(self) -> Optional[str]:
        return self._validation_error

    def add_llm_change_handler(self, handler: Callable[[str, LlmState], None]) -> None:
        """Add handler for LLM state changes"""
        if handler not in self._llm_change_handlers:
            self._llm_change_handlers.append(handler)

    def add_token_handler(self, handler: Callable[[str, str], None]) -> None:
        """Add handler for new tokens"""
        if handler not in self._token_handlers:
            self._token_handlers.append(handler)

    def add_context_change_handler(self, handler: Callable[[str, bool], None]) -> None:
        """Add handler for context changes"""
        if handler not in self._context_change_handlers:
            self._context_change_handlers.append(handler)

    def modify_context(self, context: str, generated: bool = False) -> None:
        """Update context and reset all LLM states"""
        with self._llm_lock:
            self._context = context
            self._llm_outputs.clear()
            for llm_name in self._llms:
                self._set_llm_state(llm_name, LlmState.NO_OUTPUT)
            
            for handler in self._context_change_handlers:
                handler(context, generated)

    def run_inference(self, llm_name: str) -> None:
        """Start inference on specified LLM"""
        if llm_name not in self._llms:
            raise ValueError(f"Unknown LLM: {llm_name}")

        with self._llm_lock:
            if self._llm_states[llm_name] != LlmState.NO_OUTPUT:
                raise RuntimeError(f"LLM {llm_name} is not ready for inference")
            self._set_llm_state(llm_name, LlmState.INFERENCE)
            self._stop_flags[llm_name] = False

        llm = self._llms[llm_name]
        
        def should_stop() -> bool:
            return self._stop_flags[llm_name]
        
        response_token_iter = llm.infer(self.system_prompt, self.context, should_stop)
        
        with self._output_lock:
            self._llm_outputs[llm_name] = ""

        for token in response_token_iter:
            with self._output_lock:
                self._llm_outputs[llm_name] += token
            
            for handler in self._token_handlers:
                handler(llm_name, token)

        with self._llm_lock:
            self._set_llm_state(llm_name, LlmState.OUTPUT)

    def stop_inference(self, llm_name: str) -> None:
        """Stop ongoing inference for specified LLM"""
        if llm_name not in self._llms:
            raise ValueError(f"Unknown LLM: {llm_name}")
        self._stop_flags[llm_name] = True

    def get_output(self, llm_name: str) -> str:
        """Get complete output for specified LLM"""
        if llm_name not in self._llms:
            raise ValueError(f"Unknown LLM: {llm_name}")
            
        with self._output_lock:
            return self._llm_outputs[llm_name]

    def approve_response(self, llm_name: str, response: str) -> None:
        """Process approved response from specified LLM"""
        if llm_name not in self._llms:
            raise ValueError(f"Unknown LLM: {llm_name}")

        timestamp = datetime.now(timezone.utc)
        self._iteration_logger.log_iteration(timestamp, self._context, response)
        parse_result = self._parser.parse(timestamp, response)
        if isinstance(parse_result, Command):
            result = parse_result.execute(self._working_memory)
            self._command_result = result
            if result.should_stop:
                exit(42)
            else:
                self._working_memory.update()
        else:
            parse_result.update()
            self._working_memory.update()
            self._working_memory.add_entry(parse_result)

    def _set_llm_state(self, llm_name: str, state: LlmState) -> None:
        """Update LLM state and notify handlers"""
        self._llm_states[llm_name] = state
        for handler in self._llm_change_handlers:
            handler(llm_name, state)

    def _handle_memory_update(self) -> None:
        """Handle memory updates and update context"""
        context = self._compile_context(next(iter(self._llms.values())))
        self.modify_context(context, True)

===== FILE: sia/web_io_buffer.py =====
from threading import Lock
from typing import Callable, List

from .io_buffer import IOBuffer

class WebIOBuffer(IOBuffer):
    """
    Thread-safe WebIOBuffer that maintains the synchronous IOBuffer interface.
    Uses threading primitives instead of asyncio for synchronization.
    """
    def __init__(self):
        self._stdin_buffer = ""
        self._stdout_buffer = ""
        self._stdout_change_handlers: List[Callable[[str], None]] = []
    
    def add_stdout_change_handler(self, handler: Callable[[str], None]) -> None:
        """
        Add a callback for stdout changes.
        """
        if handler not in self._stdout_change_handlers:
            self._stdout_change_handlers.append(handler)
    
    def read(self) -> str:
        """Thread-safe read from stdin buffer."""
        content = self._stdin_buffer
        self._stdin_buffer = ""
        return content
            
    def write(self, content: str) -> None:
        """Thread-safe write to stdout buffer."""
        self._stdout_buffer += content
        for handler in self._stdout_change_handlers:
            handler(self._stdout_buffer)
            
    def buffer_length(self) -> int:
        """
        Get the current length of the stdin buffer.
        
        Returns:
            int: Number of characters in the stdin buffer
        """
        return len(self._stdin_buffer)
        
    def append_stdin(self, content: str) -> None:
        """
        Append content to the stdin buffer.
        
        Args:
            content: String content to append to the stdin buffer
        """
        self._stdin_buffer += content
            
    def get_stdout(self) -> str:
        """Thread-safe get stdout content."""
        return self._stdout_buffer
            
    def clear_stdout(self) -> None:
        """Thread-safe clear stdout buffer."""
        self._stdout_buffer = ""
        for handler in self._stdout_change_handlers:
            handler(self._stdout_buffer)
===== FILE: sia/working_memory.py =====
from typing import List, Optional, Callable, Set
import xml.etree.ElementTree as ET

from .entry import Entry

class WorkingMemory:
    """
    Manages a collection of entries that represent the current state of the system.
    
    The working memory stores different types of entries (scripts, reasoning, errors, etc.)
    and provides methods to add, remove, and update them. It also generates XML context
    representing the current state.
    
    Notifies observers of entry additions, deletions and changes. During update(),
    changes are batched and a single notification is sent after completion.
    """
    
    def __init__(self):
        """Initialize an empty working memory."""
        self._entries: List[Entry] = []
        self._change_handlers: List[Callable[[], None]] = []
        self._updating = False
        self._changed_during_update: Set[Entry] = set()
        
    def add_change_handler(self, handler: Callable[[], None]) -> None:
        """Add a callback for working memory changes."""
        if handler not in self._change_handlers:
            self._change_handlers.append(handler)

    def _notify_change(self) -> None:
        """Notify all handlers of working memory changes."""
        self._sort_entries()
        for handler in self._change_handlers:
            handler()
            
    def _handle_entry_change(self, entry: Entry) -> None:
        """Handle changes from individual entries."""
        if self._updating:
            self._changed_during_update.add(entry)
        else:
            self._notify_change()

    def _sort_entries(self) -> None:
        """Sort entries by ID in chronological order."""
        self._entries.sort(key=lambda x: x.id)
        
    def add_entry(self, entry: Entry) -> None:
        """
        Add a new entry to working memory.
        
        Args:
            entry: Entry object to add
        """
        if not isinstance(entry, Entry):
            raise TypeError("Entry must be an instance of Entry class")
        entry.add_change_handler(self._handle_entry_change)
        self._entries.append(entry)
        self._sort_entries()
        self._notify_change()
        
    def remove_entry(self, id: str) -> None:
        """
        Remove an entry from working memory by its ID.
        Ensures cleanup is performed before removal.
        
        Args:
            id: Unique identifier of entry to remove
        """
        entry = self.get_entry(id)
        if entry is not None:
            entry.cleanup()
            self._entries = [e for e in self._entries if e.id != id]
            self._sort_entries()
            self._notify_change()
            
    def clear(self) -> None:
        """
        Remove all entries from working memory.
        Ensures cleanup is performed on all entries.
        """
        for entry in self._entries:
            entry.cleanup()
        self._entries.clear()
        self._notify_change()
        
    def __del__(self):
        """Clean up all entries when memory is deleted."""
        if hasattr(self, '_entries'):
            self.clear()
            
    def get_entry(self, id: str) -> Optional[Entry]:
        """
        Get an entry by its ID.
        
        Args:
            id: Unique identifier of entry to retrieve
            
        Returns:
            Entry if found, None otherwise
        """
        for entry in self._entries:
            if entry.id == id:
                return entry
        return None

    def get_entries(self) -> List[Entry]:
        """
        Get all entries in working memory.
        
        Returns:
            List[Entry]: List of all entries in chronological order
        """
        return self._entries.copy()
            
    def get_entries_count(self) -> int:
        """
        Get the total number of entries.
        
        Returns:
            int: Number of entries in working memory
        """
        return len(self._entries)
        
    def update(self) -> None:
        """
        Update all entries in working memory.
        Batches change notifications and sends single update after completion.
        """
        self._updating = True
        self._changed_during_update.clear()
        
        for entry in self._entries:
            entry.update()
            
        self._updating = False
        if self._changed_during_update:
            self._notify_change()
            
    def generate_context(self) -> List[ET.Element]:
        """
        Generate XML Elements representing all entries.
        
        Returns:
            List[ET.Element]: List of XML elements for each entry
        """
        return [entry.generate_context() for entry in self._entries]

===== FILE: sia/xml_validator.py =====
import xml.etree.ElementTree as ET
from typing import Optional, Set

class XMLValidator:
    """
    Validates XML content against a schema.
    
    Attributes:
        _schema: The parsed XML schema to validate against
        _valid_root_elements: Set of valid root element names from schema
    """
    
    def __init__(self, schema: str):
        """
        Initialize validator with XML schema.
        
        Args:
            schema: XML schema string
        """
        # Register namespace used in schema
        ET.register_namespace('xs', 'http://www.w3.org/2001/XMLSchema')
        
        try:
            # Parse schema
            self._schema = ET.fromstring(schema.strip())
            
            # Extract valid root elements
            ns = {'xs': 'http://www.w3.org/2001/XMLSchema'}
            elements = self._schema.findall(".//xs:element", ns)
            self._valid_root_elements = {elem.get('name') for elem in elements if elem.get('name')}
            
        except ET.ParseError as e:
            raise ValueError(f"Invalid schema: {e}")
            
    def validate(self, xml: str) -> Optional[str]:
        """
        Validate XML content against the schema.
        
        Args:
            xml: XML string to validate
            
        Returns:
            str: Error message if validation fails, None if validation succeeds
        """
        try:
            # Parse XML
            root = ET.fromstring(xml.strip())
            
            # Check root element is valid
            if root.tag not in self._valid_root_elements:
                return f"Invalid root element: {root.tag}. Expected one of: {sorted(self._valid_root_elements)}"
                
            # Get schema definition for this element
            ns = {'xs': 'http://www.w3.org/2001/XMLSchema'}
            element_schema = self._schema.find(f".//xs:element[@name='{root.tag}']", ns)
            if element_schema is None:
                return f"Schema definition not found for element: {root.tag}"
                
            # Validate attributes if complex type defined
            complex_type = element_schema.find('xs:complexType', ns)
            if complex_type is not None:
                # Check required attributes
                for attr in complex_type.findall('.//xs:attribute[@use="required"]', ns):
                    attr_name = attr.get('name')
                    if attr_name not in root.attrib:
                        return f"Missing required attribute '{attr_name}' on element '{root.tag}'"
                        
                # Check attribute types
                for attr_name, attr_value in root.attrib.items():
                    attr_schema = complex_type.find(f'.//xs:attribute[@name="{attr_name}"]', ns)
                    if attr_schema is None:
                        return f"Unexpected attribute '{attr_name}' on element '{root.tag}'"
                        
                    attr_type = attr_schema.get('type')
                    if attr_type == 'xs:string':
                        continue  # All string values are valid
                    elif attr_type == 'xs:integer':
                        try:
                            int(attr_value)
                        except ValueError:
                            return f"Invalid integer value '{attr_value}' for attribute '{attr_name}'"
                            
            return None  # Validation successful
            
        except ET.ParseError as e:
            return f"Invalid XML: {e}"
        except Exception as e:
            return f"Validation error: {e}"
            
    def get_valid_root_elements(self) -> Set[str]:
        """
        Get set of valid root element names from schema.
        
        Returns:
            Set[str]: Set of valid root element names
        """
        return self._valid_root_elements.copy()
===== FILE: tools/itb/requirements.txt =====
selenium>=4.0.0
webdriver-manager>=3.8.0
click>=8.0.0
beautifulsoup4>=4.9.0
pytest>=7.0.0
pytest-cov>=4.0.0
black>=22.0.0
flake8>=4.0.0

===== FILE: tools/itb/setup.py =====
from setuptools import setup, find_packages

setup(
    name="itb",
    version="0.1.0",
    packages=find_packages(),
    scripts=[
        'bin/itb_click',
        'bin/itb_cursor',
        'bin/itb_input',
        'bin/itb_navigate',
        'bin/itb_refresh',
        'bin/itb_screenshot',
        'bin/itb_scroll',
        'bin/itb_start'
    ],
    install_requires=[
        'selenium>=4.0.0',
        'webdriver-manager>=3.8.0',
        'click>=8.0.0',
        'beautifulsoup4>=4.9.0',
        'pytest>=7.0.0',
        'pytest-cov>=4.0.0',
        'black>=22.0.0',
        'flake8>=4.0.0'
    ],
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Intended Audience :: Developers',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.10',
    ],
    python_requires='>=3.10',
)
===== FILE: tools/train/requirements.txt =====
pyyaml>=6.0
requests>=2.28.0
torch>=2.0.0
transformers>=4.30.0
# DeepSeek support
accelerate>=0.25.0
bitsandbytes>=0.41.1
einops>=0.7.0
sentencepiece>=0.1.99
unsloth>=2024.3
trl>=0.7.8
datasets>=2.14.6
peft>=0.8.0
===== FILE: tools/train/setup.py =====
from setuptools import setup, find_packages

setup(
    name="train",
    version="0.1.0",
    packages=find_packages(),
    scripts=[
        'bin/train_deepseek',
        'bin/train_mistral'
    ],
    install_requires=[
        'pyyaml>=6.0',
        'requests>=2.28.0',
        'torch>=2.0.0',
        'transformers>=4.30.0',
        'accelerate>=0.25.0',
        'bitsandbytes>=0.41.1',
        'einops>=0.7.0',
        'sentencepiece>=0.1.99',
        'unsloth>=2024.3',
        'trl>=0.7.8',
        'datasets>=2.14.6',
        'peft>=0.8.0',
        'pytest>=7.0.0',
        'pytest-cov>=4.0.0',
        'black>=22.0.0',
        'flake8>=4.0.0'
    ],
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Intended Audience :: Developers',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.10',
    ],
    python_requires='>=3.10',
)
===== FILE: tools/train/train.sh =====
#!/bin/bash

set -e

SIA_DIR="/root/sia"
OUTPUT_DIR="${1:-/root/models/$(cd "$SIA_DIR" && git rev-parse HEAD)}"

if [ -n "$(cd "$SIA_DIR" && git status --porcelain)" ]; then
    echo "Uncommitted changes in SIA directory"
    #exit 1
fi

mkdir -p "$OUTPUT_DIR"

train_deepseek --output-dir "$OUTPUT_DIR"
===== FILE: tools/train/train/__init__.py =====
"""
SIA Training Tool

This package provides utilities for fine-tuning language models used by SIA.
Supports DeepSeek and Mistral models.
"""

__version__ = "0.1.0"
===== FILE: tools/train/train/mistral_api.py =====
#!/root/venvs/train/bin/python
"""
Script for fine-tuning Mistral models for SIA using the Mistral API.
"""
from dataclasses import dataclass
from pathlib import Path
import argparse
import json
import os
import sys
import tempfile
import requests

# Import from our shared library
from .util import TrainingParams, DatasetCreator

@dataclass
class Config:
    def __init__(self):
        parser = argparse.ArgumentParser(description='Train SIA model using Mistral API')
        parser.add_argument(
            '--config',
            type=Path,
            default=Path('/root/sia/training/config.yaml'),
            help='Path to config file'
        )
        parser.add_argument(
            '--model',
            type=str,
            default='mistral-large-latest',
            help='Base model for fine-tuning'
        )
        parser.add_argument(
            '--api-key',
            type=str,
            default=os.environ.get('SIA_MISTRAL_API_KEY'),
            help='Mistral API key'
        )
        self.args = parser.parse_args()

    @property
    def config_path(self) -> Path:
        return self.args.config

    @property
    def model(self) -> str:
        return self.args.model
    
    @property
    def api_key(self) -> str:
        return self.args.api_key

def upload_file(api_key: str, file_path: Path) -> str:
    """Upload a file to the Mistral API and return the file ID"""
    url = "https://api.mistral.ai/v1/files"
    headers = {
        "Authorization": f"Bearer {api_key}"
    }
    files = {
        "file": ("dataset.jsonl", open(file_path, "rb"), "application/jsonl"),
        "purpose": (None, "fine-tune")
    }
    
    response = requests.post(url, headers=headers, files=files)
    if response.status_code != 200:
        print(f"Error uploading file: {response.text}")
        sys.exit(1)
    
    return response.json()["id"]

def start_finetune_job(api_key: str, model: str, file_id: str, params: sia_train_lib.TrainingParams):
    """Start a fine-tuning job on the Mistral API"""
    headers = {
        "Authorization": f"Bearer {api_key}",
        "Content-Type": "application/json"
    }
    data = {
        "model": model,
        "training_files": [{"file_id": file_id, "weight": 1}],
        "hyperparameters": {
            "learning_rate": params.learning_rate,
            "epochs": params.epochs
        }
    }
    
    response = requests.post(
        "https://api.mistral.ai/v1/fine_tuning/jobs",
        headers=headers,
        json=data
    )
    
    if response.status_code != 200:
        print(f"Error creating fine-tuning job: {response.text}")
        return None
        
    return response.json()["id"]

def main():
    config = Config()
    if not config.api_key:
        print("Error: Mistral API key not found. Set SIA_MISTRAL_API_KEY environment variable.")
        return 1
    
    training_data, train_params, commit_hash = sia_train_lib.prepare_training_data(config.config_path)
    
    if not training_data:
        print("No valid training data found. Exiting.")
        return 1
    
    model_name = f"sia_{commit_hash}"
    
    # Create temp file and upload
    with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f:
        for sample in training_data:
            json.dump(sample, f, ensure_ascii=False)
            f.write('\n')
    
    try:
        file_id = upload_file(config.api_key, Path(f.name))
        
        # Start fine-tuning job
        job_id = start_finetune_job(
            api_key=config.api_key,
            model=config.model,
            file_id=file_id,
            params=train_params
        )
        
        if not job_id:
            return 1
            
        print(f"Started fine-tuning job: {model_name}")
        print(f"Job ID: {job_id}")
        print(f"Check status: curl -H 'Authorization: Bearer {config.api_key}' https://api.mistral.ai/v1/fine_tuning/jobs/{job_id}")
    finally:
        os.unlink(f.name)
    
    return 0

if __name__ == "__main__":
    exit(main())

===== FILE: tools/train/train/unsloth_deepseek.py =====
#!/root/venvs/train/bin/python
"""
Script for fine-tuning DeepSeek models for SIA using Unsloth.
Training always starts from a base model and creates a new fine-tuned model.
"""
import argparse
import os
import sys
import torch
from dataclasses import dataclass
from pathlib import Path
import json

# Import from shared library
from .util import prepare_training_data

@dataclass
class Config:
    def __init__(self):
        parser = argparse.ArgumentParser(description='Train SIA model using Unsloth')
        parser.add_argument(
            '--config',
            type=Path,
            default=Path('/root/sia/training/config.yaml'),
            help='Path to config file'
        )
        parser.add_argument(
            '--base-model',
            type=str,
            default='deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B',
            help='HuggingFace model ID for base model'
        )
        parser.add_argument(
            '--output-dir',
            type=Path,
            required=True,
            help='Directory to save the trained model'
        )
        parser.add_argument(
            '--api-key',
            type=str,
            default=os.environ.get('SIA_HF_API_KEY'),
            help='HuggingFace API key'
        )
        self.args = parser.parse_args()

    @property
    def config_path(self) -> Path:
        return self.args.config

    @property
    def base_model(self) -> str:
        return self.args.base_model
    
    @property
    def output_dir(self) -> Path:
        return self.args.output_dir
    
    @property
    def api_key(self) -> str:
        return self.args.api_key

def train_model(config: Config, training_data, train_params, commit_hash):
    """Train the model using Unsloth"""
    try:
        from unsloth import FastLanguageModel
        from transformers import TrainingArguments, DataCollatorForSeq2Seq
        from trl import SFTTrainer
        from datasets import Dataset
        from unsloth.chat_templates import get_chat_template, train_on_responses_only
    except ImportError as e:
        print(f"Error importing required libraries: {e}")
        print("Please ensure Unsloth and its dependencies are installed.")
        sys.exit(1)

    print(f"Starting training from base model: {config.base_model}")
    
    # Convert to datasets format
    dataset = Dataset.from_list(training_data)
    
    # Determine if bfloat16 is supported
    dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
    
    # Load the model - always from a base model (no incremental updates)
    try:
        model, tokenizer = FastLanguageModel.from_pretrained(
            model_name=config.base_model,
            max_seq_length=2048,
            dtype=dtype,
            load_in_4bit=True,
            token=config.api_key,
        )
    except Exception as e:
        print(f"Error loading base model: {e}")
        sys.exit(1)
    
    # Apply LoRA
    target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
    model = FastLanguageModel.get_peft_model(
        model,
        r=16,
        target_modules=target_modules,
        lora_alpha=16,
        lora_dropout=0,
        bias="none",
        use_gradient_checkpointing="unsloth",
        random_state=3407,
    )
    
    # Apply chat template
    tokenizer = get_chat_template(
        tokenizer, 
        chat_template="llama-3.1",  # Compatible with DeepSeek
    )
    
    # Function to format conversations
    def formatting_prompts_func(examples):
        convos = examples["conversations"]
        texts = [tokenizer.apply_chat_template(convo, tokenize=False, add_generation_prompt=False) for convo in convos]
        return {"text": texts}
    
    # Standarize dataset and format
    from unsloth.chat_templates import standardize_sharegpt
    
    # Add conversations field if not present
    if "conversations" not in dataset.column_names:
        if "messages" in dataset.column_names:
            dataset = dataset.rename_column("messages", "conversations")
        else:
            dataset = dataset.map(lambda x: {"conversations": [{"role": "system", "content": x.get("system_prompt", "")}, 
                                                            {"role": "user", "content": x.get("prompt", "")}, 
                                                            {"role": "assistant", "content": x.get("response", "")}]})
    
    # Standardize format
    dataset = standardize_sharegpt(dataset)
    
    # Apply formatting
    dataset = dataset.map(formatting_prompts_func, batched=True)
    
    # Configure the trainer
    output_dir = config.output_dir / commit_hash
    output_dir.mkdir(parents=True, exist_ok=True)
    
    # Determine steps or epochs based on dataset size
    max_steps = None
    num_train_epochs = train_params.epochs
    if len(dataset) < 100:  # Small dataset
        # Aim for at least 500 steps for small datasets
        max_steps = 500
        num_train_epochs = None
    
    trainer = SFTTrainer(
        model=model,
        tokenizer=tokenizer,
        train_dataset=dataset,
        dataset_text_field="text",
        max_seq_length=2048,
        data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer),
        dataset_num_proc=2,
        packing=False,
        args=TrainingArguments(
            per_device_train_batch_size=2,
            gradient_accumulation_steps=4,
            warmup_steps=5,
            max_steps=max_steps,
            num_train_epochs=num_train_epochs,
            learning_rate=train_params.learning_rate,
            fp16=not torch.cuda.is_bf16_supported(),
            bf16=torch.cuda.is_bf16_supported(),
            logging_steps=10,
            optim="adamw_8bit",
            weight_decay=0.01,
            lr_scheduler_type="linear",
            seed=3407,
            output_dir=str(output_dir),
            report_to="none",
        ),
    )
    
    # Train only on responses
    trainer = train_on_responses_only(
        trainer,
        instruction_part="<|start_header_id|>user<|end_header_id|>\n\n",
        response_part="<|start_header_id|>assistant<|end_header_id|>\n\n",
    )
    
    # Train the model
    trainer.train()
    
    # Enable inference mode for the model
    model = FastLanguageModel.for_inference(model)
    
    # Save the model
    model.save_pretrained(output_dir)
    tokenizer.save_pretrained(output_dir)
    
    # Create a metadata file with training information
    with open(output_dir / "training_info.json", "w") as f:
        json.dump({
            "base_model": config.base_model,
            "commit_hash": commit_hash,
            "learning_rate": train_params.learning_rate,
            "epochs": train_params.epochs,
            "dataset_size": len(dataset),
            "training_method": "unsloth",
        }, f, indent=2)
    
    return output_dir

def main():
    config = Config()
    
    # Prepare training data
    training_data, train_params, commit_hash = prepare_training_data(config.config_path)
    
    if not training_data:
        print("No valid training data found. Exiting.")
        return 1
    
    # Train the model
    try:
        model_dir = train_model(config, training_data, train_params, commit_hash)
        
        # Create symlink to current
        current_link = config.output_dir / "current"
        if current_link.exists() or current_link.is_symlink():
            current_link.unlink()
        os.symlink(model_dir, current_link, target_is_directory=True)
        
        print(f"Training complete. Model saved to {model_dir}")
        print(f"Symlink created at {current_link}")
        
        return 0
    except Exception as e:
        print(f"Error during training: {e}")
        return 1

if __name__ == "__main__":
    exit(main())
===== FILE: tools/train/train/util.py =====
"""
Shared library for SIA model training functionality.
Contains common code for both API-based and local training.
"""
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, List, Optional, Set, Tuple
import hashlib
import json
import subprocess
import sys
import xml.etree.ElementTree as ET
import yaml

@dataclass
class TrainingParams:
    """Parameters for model training"""
    learning_rate: float
    epochs: int
    batch_size: int = 1

class DatasetCreator:
    """Creates training datasets from XML iteration files"""
    
    def __init__(
            self,
            xml_files: Set[Path],
            system_prompt_file: Path,
            action_schema_file: Path
        ):
        self.xml_files = xml_files
        self.system_prompt_file = Path(system_prompt_file)
        self.action_schema_file = Path(action_schema_file)
        
        self.system_prompt = self.system_prompt_file.read_text()
        self.system_prompt_hash = self._calculate_hash(self.system_prompt)
        
        self.action_schema = self.action_schema_file.read_text()
        self.action_schema_hash = self._calculate_hash(self.action_schema)
        
    def _calculate_hash(self, content: str) -> str:
        """Calculate SHA-256 hash of content"""
        return hashlib.sha256(content.encode()).hexdigest()
    
    def _parse_iteration_file(self, file_path: Path) -> Optional[Dict]:
        """Parse a single iteration XML file into a training example"""
        try:
            tree = ET.parse(file_path)
            root = tree.getroot()
            
            # Check hashes to ensure compatibility
            if root.get('system_prompt_hash') != self.system_prompt_hash:
                print(f"System prompt hash mismatch in {file_path}")
                return None
            if root.get('action_schema_hash') != self.action_schema_hash:
                print(f"Action schema hash mismatch in {file_path}")
                return None
            
            context_elem = root.find('context')
            response_elem = root.find('response')
            
            if context_elem is None or response_elem is None:
                print(f"Missing context or response elements in {file_path}")
                return None
                
            context = context_elem.text
            response = response_elem.text
            
            if not context or not response:
                print(f"Empty context or response in {file_path}")
                return None
            
            return {
                "messages": [
                    {
                        "role": "system",
                        "content": self.system_prompt + "\n" + self.action_schema
                    },
                    {
                        "role": "user",
                        "content": context
                    },
                    {
                        "role": "assistant", 
                        "content": response
                    }
                ]
            }
            
        except Exception as e:
            print(f"Error processing {file_path}: {str(e)}")
            return None
    
    def create_dataset(self) -> List[Dict]:
        """Create a dataset from all valid XML files"""
        samples = []
        total_files = len(self.xml_files)
        print(f"Processing {total_files} XML files...")
        
        for i, xml_file in enumerate(sorted(self.xml_files)):
            if i % 10 == 0:
                print(f"Processed {i}/{total_files} files...")
                
            sample = self._parse_iteration_file(xml_file)
            if sample:
                samples.append(sample)
        
        print(f"Created dataset with {len(samples)} samples from {total_files} files")
        return samples

def find_xml_files(data_paths: List[Path]) -> Set[Path]:
    """Find all XML files in the given data paths"""
    xml_files = set()
    for path in data_paths:
        if not path.exists():
            print(f"Error: Data path not found: {path}")
            sys.exit(1)
        xml_files.update(path.rglob('*.xml'))
    return xml_files

def format_chat_for_mistral(messages):
    """Format messages for Mistral chat format"""
    # Mistral uses a specific chat format:
    # <s>[INST] {system + user content} [/INST] {assistant response} </s>
    
    system_content = ""
    user_content = ""
    assistant_content = ""
    
    for msg in messages:
        role = msg["role"]
        content = msg["content"]
        
        if role == "system":
            system_content = content
        elif role == "user":
            user_content = content
        elif role == "assistant":
            assistant_content = content
    
    # Combine system and user content for the instruction
    instruction = system_content
    if instruction and user_content:
        instruction += "\n\n"
    instruction += user_content
    
    # Format according to Mistral chat template
    return f"<s>[INST] {instruction} [/INST] {assistant_content} </s>"

def prepare_training_data(config_path: Path) -> Tuple[List[Dict], TrainingParams, str]:
    """Prepare training data from config and XML files"""
    with open(config_path) as f:
        config_data = yaml.safe_load(f)
    
    data_paths = [Path(p) for p in config_data['data']]
    xml_files = find_xml_files(data_paths)
    
    paths = list(xml_files)
    paths.append(config_path)
    paths.append(Path(config_data['model']['system_prompt_path']))
    paths.append(Path(config_data['model']['action_schema']))
    commit_hash = check_git_status(paths)
    
    creator = DatasetCreator(
        xml_files=xml_files,
        system_prompt_file=config_data['model']['system_prompt_path'],
        action_schema_file=config_data['model']['action_schema']
    )
    
    training_data = creator.create_dataset()
    
    train_params = TrainingParams(
        learning_rate=config_data['params'].get('learning_rate', 1e-5),
        epochs=config_data['params'].get('epochs', 3),
        batch_size=config_data['params'].get('batch_size', 1)
    )
    
    return training_data, train_params, commit_hash

def save_jsonl_dataset(data: List[Dict], output_path: Path) -> None:
    """Save dataset in JSONL format"""
    with open(output_path, 'w', encoding='utf-8') as f:
        for sample in data:
            json.dump(sample, f, ensure_ascii=False)
            f.write('\n')
    print(f"Saved dataset with {len(data)} samples to {output_path}")

===== FILE: training/config.yaml =====
model:
  system_prompt_path: "/root/sia/system_prompt.md"
  action_schema: "/root/sia/action_schema.xsd"
params:
  learning_rate: 1e-5
  epochs: 3
data:
  - "/root/sia/training/clean_start/"
  - "/root/sia/training/delete_indicated_entries/"
  - "/root/sia/training/list_entries_to_delete/"