diff --git a/.gitignore b/.gitignore index d58ea1c..cf4d5e4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env +__pycache__/ data/ model/ -__pycache__/ \ No newline at end of file +sia.egg-info/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index cfffbf4..c851fbc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,57 +1,82 @@ -FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS requirements -RUN apt-get update -RUN apt-get upgrade -y -RUN apt install -y python3-pip -ENV TOKENIZERS_PARALLELISM=false -COPY requirements.txt /requirements.txt -RUN pip3 install -r /requirements.txt -RUN rm -rf /requirements.txt - -FROM requirements AS sia-test -COPY ./ /root/sia/ -WORKDIR /root/sia/ -RUN mkdir -p /root/models/current -CMD ["python3", "-m", "unittest", "discover", "-v", "-p", "*test.py"] - -FROM node:20-alpine AS web-test -WORKDIR /app -COPY web/package*.json ./ -RUN npm install -COPY web . -RUN npm test - -FROM node:20-alpine AS web-build -WORKDIR /app -COPY web/package*.json ./ -RUN npm install -COPY web . -RUN npm run build - -FROM requirements -RUN apt-get update -RUN apt-get install -y wget gnupg +FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base +RUN apt-get update && \ + apt-get upgrade -y && \ + apt install -y \ + python3-pip \ + git \ + python3-venv \ + wget \ + gnupg \ + vim \ + curl RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - -RUN echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list -RUN apt-get update -RUN apt-get install -y google-chrome-stable +RUN echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list +RUN apt-get update && \ + apt-get install -y \ + google-chrome-stable RUN rm -rf /var/lib/apt/lists/* +# Create directory structure RUN mkdir -p \ /root/sia \ + /root/sia/scripts \ /root/data/iterations \ /root/data/user \ /root/data/tasks \ /root/data/environment \ /root/models \ - /root/desktop + /root/desktop \ + /root/venvs -COPY ./tools/itb/requirements.txt /root/sia/tools/itb/requirements.txt -RUN cd /root/sia/tools/itb/ && python3 -m pip install -r requirements.txt +# ITB tool setup +FROM base AS itb-env +COPY ./scripts/setup_binaries.py /root/sia/scripts/ +COPY ./tools/itb/setup.py /root/sia/tools/itb/setup.py +RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/tools/itb/setup.py +RUN python3 -m venv /root/venvs/itb +RUN /root/venvs/itb/bin/pip install -e /root/sia/tools/itb/ -COPY ./tools/ /root/sia/tools/ -RUN cd /root/sia/tools/itb/ && python3 -m pip install -e ".[dev]" +# Train tool setup +FROM base AS train-env +COPY ./scripts/setup_binaries.py /root/sia/scripts/ +COPY ./tools/train/setup.py /root/sia/tools/train/setup.py +RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/tools/train/setup.py +RUN python3 -m venv /root/venvs/train +RUN /root/venvs/train/bin/pip install -e /root/sia/tools/train/ +# SIA core setup +FROM base AS sia-env +COPY ./scripts/setup_binaries.py /root/sia/scripts/ +COPY ./setup.py /root/sia/setup.py +RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/setup.py +RUN python3 -m venv /root/venvs/sia +RUN /root/venvs/sia/bin/pip install -e /root/sia/ + +# Web frontend build +FROM node:20-alpine AS web-build +WORKDIR /app +COPY web/package*.json ./ +RUN npm install +RUN rm -rf /root/.npm/_cacache +COPY web . +RUN npm run build + +# Final image +FROM base + +# Copy virtual environments (these layers only change if setup.py files change) +COPY --from=itb-env /root/venvs/itb /root/venvs/itb +COPY --from=train-env /root/venvs/train /root/venvs/train +COPY --from=sia-env /root/venvs/sia /root/venvs/sia + +# Copy source code and scripts (these change frequently but don't affect venv layers) +COPY --from=itb-env /root/sia/tools/itb /root/sia/tools/itb +COPY --from=train-env /root/sia/tools/train /root/sia/tools/train +COPY --from=sia-env /root/sia /root/sia COPY --from=web-build /app/dist /root/static/ + +RUN echo 'for venv in /root/venvs/*/bin; do PATH="$venv:$PATH"; done' >> /etc/profile && \ + echo 'export PATH' >> /etc/profile WORKDIR /root/desktop -CMD ["/root/sia/scripts/restart.sh"] \ No newline at end of file +CMD ["/bin/bash", "-l", "-c", "/root/sia/scripts/restart.sh"] \ No newline at end of file diff --git a/collect.txt b/collect.txt index fb25f36..18519ed 100644 --- a/collect.txt +++ b/collect.txt @@ -142,12 +142,14 @@ Directory Tree: |____README.md |____requirements.txt |____scripts +| |____bootstrap.sh | |____collect.sh | |____container.sh -| |____install.sh | |____restart.sh | |____run.sh +| |____setup_binaries.py | |____test.sh +|____setup.py |____sia | |____auto_approver.py | |____base_agent.py @@ -175,14 +177,23 @@ Directory Tree: | | | |____single_entry.cpython-310.pyc | | | |____write_entry.cpython-310.pyc | | | |______init__.cpython-310.pyc -| |____hf_llm_engine.py | |____io_buffer.py | |____iteration_logger.py | |____iteration_parser.py -| |____llm_engine.py -| |____local_llm_engine.py -| |____mistral_llm_engine.py -| |____openai_llm_engine.py +| |____llm_engine +| | |____deepseek_llm_engine.py +| | |____hf_llm_engine.py +| | |____local_llm_engine.py +| | |____mistral_llm_engine.py +| | |____openai_llm_engine.py +| | |______init__.py +| | |______pycache__ +| | | |____deepseek_llm_engine.cpython-310.pyc +| | | |____hf_llm_engine.cpython-310.pyc +| | | |____local_llm_engine.cpython-310.pyc +| | | |____mistral_llm_engine.cpython-310.pyc +| | | |____openai_llm_engine.cpython-310.pyc +| | | |______init__.cpython-310.pyc | |____response_parser.py | |____standard_io_buffer.py | |____stop_command.py @@ -241,6 +252,13 @@ Directory Tree: | | |____xml_validator.cpython-310.pyc | | |______init__.cpython-310.pyc | | |______main__.cpython-310.pyc +|____sia.egg-info +| |____dependency_links.txt +| |____entry_points.txt +| |____PKG-INFO +| |____requires.txt +| |____SOURCES.txt +| |____top_level.txt |____system_prompt.md |____tasks |____test @@ -296,7 +314,18 @@ Directory Tree: | | |____setup.py | | |____social.html | |____train -| | |____train_mistral.py +| | |____bin +| | | |____train_deepseek +| | | |____train_mistral +| | |____readme.md +| | |____requirements.txt +| | |____setup.py +| | |____train +| | | |____mistral_api.py +| | | |____unsloth_deepseek.py +| | | |____util.py +| | | |______init__.py +| | |____train.sh |____training | |____clean_start | | |____iteration_20250116_134549_655.xml @@ -368,9 +397,207 @@ Directory Tree: File Contents: -===== FILE: mini_prompt.md ===== -You are SIA, the Self Improving Agent. -These are the actions you can take: +===== FILE: action_schema.xsd ===== + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +===== FILE: Dockerfile ===== +FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base +RUN apt-get update && \ + apt-get upgrade -y && \ + apt install -y \ + python3-pip \ + git \ + python3-venv \ + wget \ + gnupg \ + vim \ + curl +RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - +RUN echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list +RUN apt-get update && \ + apt-get install -y \ + google-chrome-stable +RUN rm -rf /var/lib/apt/lists/* + +# Create directory structure +RUN mkdir -p \ + /root/sia \ + /root/sia/scripts \ + /root/data/iterations \ + /root/data/user \ + /root/data/tasks \ + /root/data/environment \ + /root/models \ + /root/desktop \ + /root/venvs + +# ITB tool setup +FROM base AS itb-env +COPY ./scripts/setup_binaries.py /root/sia/scripts/ +COPY ./tools/itb/setup.py /root/sia/tools/itb/setup.py +RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/tools/itb/setup.py +RUN python3 -m venv /root/venvs/itb +RUN /root/venvs/itb/bin/pip install -e /root/sia/tools/itb/ + +# Train tool setup +FROM base AS train-env +COPY ./scripts/setup_binaries.py /root/sia/scripts/ +COPY ./tools/train/setup.py /root/sia/tools/train/setup.py +RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/tools/train/setup.py +RUN python3 -m venv /root/venvs/train +RUN /root/venvs/train/bin/pip install -e /root/sia/tools/train/ + +# SIA core setup +FROM base AS sia-env +COPY ./scripts/setup_binaries.py /root/sia/scripts/ +COPY ./setup.py /root/sia/setup.py +RUN python3 /root/sia/scripts/setup_binaries.py /root/sia/setup.py +RUN python3 -m venv /root/venvs/sia +RUN /root/venvs/sia/bin/pip install -e /root/sia/ + +# Web frontend build +FROM node:20-alpine AS web-build +WORKDIR /app +COPY web/package*.json ./ +RUN npm install +COPY web . +RUN npm run build + +# Final image +FROM base + +# Copy virtual environments (these layers only change if setup.py files change) +COPY --from=itb-env /root/venvs/itb /root/venvs/itb +COPY --from=train-env /root/venvs/train /root/venvs/train +COPY --from=sia-env /root/venvs/sia /root/venvs/sia + +# Copy source code and scripts (these change frequently but don't affect venv layers) +COPY --from=itb-env /root/sia/tools/itb /root/sia/tools/itb +COPY --from=train-env /root/sia/tools/train /root/sia/tools/train +COPY --from=sia-env /root/sia /root/sia +COPY --from=web-build /app/dist /root/static/ + +ENV PATH="/root/venvs/*/bin:${PATH}" +WORKDIR /root/desktop + +CMD ["/root/sia/scripts/restart.sh"] ===== FILE: procedures/filesystem_design/reasoning.md ===== # Filesystem Usage Reasoning @@ -1038,8 +1265,9 @@ This preserves the temporal relationships between entries while anchoring them t ## Training Configuration -SIA takes a modular approach to model training by having separate specialized tools for each provider like train_mistral.py, train_openai.py, etc. +SIA takes a modular approach to model training by having separate specialized tools for each provider like train_mistral, train_deepseek, etc. Each tool shares similar core functionality while handling provider-specific requirements. +The default training tool and parameters are called from the `/root/sia/tools/train/train.sh` script. While the training process is conceptually similar across providers, each has unique requirements for data formatting, API interactions, and job management. By creating dedicated tools, we can properly encapsulate these differences without complicating the core training logic. @@ -1070,29 +1298,6 @@ This separation of concerns makes it easier to: - Handle provider-specific error cases and requirements appropriately - Update individual providers' implementations as their APIs evolve -### Example - -Config file: - -```yaml -model: - system_prompt_path: "system_prompt.md" - action_schema: "action_schema.xsd" -params: - learning_rate: 1e-5 - epochs: 3 -data: - - "training/clean_start/" - - "training/delete_indicated_entries/" - - "training/list_entries_to_delete/" -``` - -Training command: - -```bash -python train_mistral.py --model mistral-large-latest -``` - ## Repository Structure All components that define SIA's behavior are version controlled in a single repository, providing a clear and reproducible state for any point in time. @@ -1495,1497 +1700,5207 @@ The `--no-ff` flag creates a merge commit even for fast-forward merges, maintain Git credentials are stored in the environment, not in the filesystem. This ensures that credentials are not exposed in the iteration log. Info about the repository and relevant environment variables is stored in `/root/data/environment/sia_repo.md`. -===== FILE: README.md ===== -# SIA - The Self Improving Agent - -SIA is an agentic artificial intelligence system that autonomously completes complex tasks by writing and executing scripts. -It uses a Large Language Model (LLM) which operates in a loop. -Each iteration a context is updated with system info and a list of previous reasoning and actions. -The agent responds with a new reasoning or an action. -Context, reasoning and actions are stored in a file for each iteration. -SIA can read past iterations to improve its reasoning and actions. -It can improve in several ways: -- By finetuning the LLM with a better reasoning or action for a given context -- By modifying its own source code -- By refining Procedures -- By developing Tools - -## Example - -This example shows a typical context with some monitored items and previous actions. -Between each of the responses, the context would be updated. - -### Context - -```xml - - - - - - - - - - - - - - - - - - - -``` - -### Responses - -Start by reasoning about the task. -```xml - - John did not specify an exact time. - I'll suggest 9am. - He also did not specify how to be reminded. - I'll ask but if he doesn't respond I'll assume a text message on standard output is fine. - I'll write down this task in a file so I remember it even on a power failure. - -``` - -Store important information on disk. -```xml - /root/data/tasks/reminder_to_feed_cat.txt]]> -``` - -Respond to the user. -```xml -I'll remind you to feed the cat tomorrow morning at 9am. Is a message on the standard output ok? -``` - -Clear initial reasoning. -```xml - -``` - -The conversation is kept in context to understand the user's expected response. -If the context was near full, it would be summarized and cleaned up. - -The `single` output is also kept in context. -If the file was updated often, it could be replaced by a repeated `cat`, like the general info. - -## Working principles - -The main context is regenerated for each iteration. -It contains info about the system and previous actions that have not been deleted. -Together with the system prompt and available core actions it forms the prompt for the LLM. -The LLM responds with one core action. - -### Core Actions - -There are only a few core actions: -- Starting a script -- Deleting data from context -- Stopping SIA -- Reading standard input -- Writing to standard output -- Reasoning - -### Scripts - -Scripts can run in one of 2 modes: single-shot or repeat. -Their mode and output (stdout and stderr) stay in the context until they are explicitly removed. -In this way the agent manages what information is available in the context. - -#### Single-shot script - -The script is executed once. -This is useful for most operations e.g. writing to or moving a file or downloading content from the internet. -The next iteration starts after the scripts has finished. - -#### Repeat script - -The script is restarted on each iteration. -This is useful for monitoring files or the file system. -commands like `head` and `tail` can be used to limit the data in context. -The next iteration starts after all repeat scripts in context have finished. - -### Use of XML - -The context and actions are formatted as XML. -For the context this adds clear rules for escaping. -This is usefull in case a previous context is embedded. - -The LLM is free to escape data any way it wants, -as long as it results in valid XML. -The response is validated against a schema. - -#### XML Data Flow -Entries store their content as raw text. During context compilation, the XML formatter -wraps text content in CDATA sections, except when the content contains CDATA closing sequences. -In those cases, the formatter uses standard XML escaping. - -This separation between storage and formatting: -- Keeps entry data clean and unescaped -- Centralizes XML formatting rules -- Makes it easy to change escaping rules without modifying entries -- Allows different formatting for different use cases - -The Context is escaped using CDATA blocks. -Except when the data contains CDATA closing sequences. -Then the whole block is escaped using standard XML escaping. - - -### The SIA process - -SIA is typically runs with the `restart.sh` script. -This is a simple shell script that runs SIA in a loop. -When stopped, SIA restarts and reloads the Python files. -This is how SIA can self-update. - -SIA can also run SIA processes as script. -This can be used for testing updates to the LLM or core functionality. - -### Server for debuggin and human input - -SIA can be started with an optional `--server` flag. -This starts a web server that can be used to interact with SIA. -It is made, specifically for reinforcement learning by human feedback. -The web interface takes over standard input and output. -It will display the context for editing before handing it to the LLM. -After each run of the LLM, before parsing, it will display the reasoning and actions. -It interactively displays if the actions can be parsed. -At any time, the user can write to the standard input of SIA. - -## Architecture - -SIA follows a modular architecture centered around an agent that processes context through an LLM to generate actions. -The system can run in two modes: a standard command-line mode and an interactive web mode for debugging and human feedback. - -### Core Components - -#### Agent Architecture -The core of SIA is the agent, which exists in two variants: -- ProceduralAgent: Runs in a simple loop, processing context and executing actions directly -- WebAgent: Uses a state machine to allow human intervention and feedback through a web interface - -Both agent types share common components: -- WorkingMemory: Maintains the current state through a collection of entries and system metrics -- ResponseParser: Converts LLM output into commands or entries -- XMLValidator: Validates responses against a schema -- IOBuffer: Handles input/output operations in an agent-appropriate way - -#### Working Memory -The working memory stores the current state of the system through different types of entries: -- SingleEntries: Output of single-shot script executions -- RepeatEntry: Continuously refreshed script outputs -- ReasoningEntry: LLM's thought process documentation -- ParseErrorEntry: XML validation or parsing errors -- IOEntry: Input/output operations - -Each entry can be serialized to XML for inclusion in the LLM context. -Working memory is cleaned through explicit delete commands or when context size limits are reached. - -#### Command Processing -SIA distinguishes between two types of LLM outputs: -1. Commands: Immediate actions that modify the system - - DeleteCommand: Removes entries from working memory - - StopCommand: Terminates the agent -2. Entries: Records that become part of working memory - - Created from script executions, IO operations, reasoning, or errors - - Persist until explicitly deleted or context limits are reached - -#### IO Handling -IO operations are abstracted through an IOBuffer interface: -- StandardIOBuffer: Direct access to system stdin/stdout -- WebIOBuffer: Buffer for web interface communication -This abstraction allows the ResponseParser to generate consistent IOEntries regardless of agent type. - -### Processing Flow - -#### Standard Agent Flow -1. Update system metrics and context size -2. Compile context from working memory entries -3. Process context through LLM -4. Validate XML response -5. Parse response into command or entry -6. Execute command or update working memory -7. Return to step 1 unless stopped - -#### Web Agent Flow -1. Update system metrics and context size -2. Compile context and await human approval -3. Process approved context through LLM -4. Validate XML response -5. Present validation results and await human review -6. Parse approved response into command or entry -7. Execute command or update working memory -8. Return to step 1 unless stopped - -### Web Interface - -The web interface provides interactive debugging and human feedback through a WebSocket-based protocol: - -#### Server-Client Communication -- Server Messages: - - Context updates for review - - Validation results from LLM responses - - Output updates from script execution - - State updates for UI synchronization -- Client Messages: - - Context approval decisions - - Response modifications - - User input for stdin operations - -#### WebServer Architecture -The web server manages: -- WebSocket connections to clients -- Message routing to/from the WebAgent -- Broadcasting state updates to all connected clients - -This architecture allows for: -- Real-time monitoring of agent state -- Interactive debugging of LLM responses -- Human-in-the-loop operation for testing and development -- Collection of human feedback for reinforcement learning - -### Diagrams - -#### Core classes - -```mermaid -classDiagram - class SystemMetrics { - +SystemMetrics(sample_interval float) - +generate_context(context_usage float) ElementTree - +stop() void - -monitor_loop() void - } - - class LLMEngine { - +LLMEngine(model_path str) - +set_model_path(model_path str) void - +infer(system_prompt str, main_context str) Iterator~str~ - } - - class BaseAgent { - <> - -working_memory: WorkingMemory - -metrics: SystemMetrics - -llm: LLMEngine - -parser: ResponseParser - -validator: XMLValidator - -action_schema: str - - #_compile_context() str - } - - class WorkingMemory { - -entries: List~Entry~ - - +WorkingMemory() - +add_entry(entry Entry) void - +remove_entry(id str) void - +clear() void - +get_entry(id str) Optional~Entry~ - +get_entries() List~Entry~ - +get_entries_count() int - +get_entries_by_type(type Type) List~Entry~ - +update() void - +generate_context() List~ElementTree~ - } - - class XMLValidator { - +XMLValidator(schema str) - +validate(xml str) Optional~str~ - +get_valid_root_elements() Set~str~ - } - - class ResponseParser { - -io_buffer: IOBuffer - - +ResponseParser(io_buffer IOBuffer) - +parse(xml str) Command | Entry - } - - class Entry { - <> - +id: str readonly - +timestamp: datetime readonly - - +Entry(id str, timestamp datetime) - +update() void* - +generate_context() ElementTree* - +cleanup() void* - } - - class IOBuffer { - <> - +read() str* - +write(content str) void* - +buffer_length() int* - } - - class Command { - <> - +execute(memory WorkingMemory) CommandResult* - } - - SystemMetrics "1" --* "1" BaseAgent - LLMEngine "1" --* "1" BaseAgent - XMLValidator "1" --* "1" BaseAgent - BaseAgent "1" *-- "1" IOBuffer - BaseAgent "1" *-- "1" WorkingMemory - BaseAgent "1" *-- "1" ResponseParser - WorkingMemory "1" *-- "*" Entry - ResponseParser ..> Entry - ResponseParser ..> Command -``` - -#### Standard Agent Flow - -```mermaid -stateDiagram-v2 - direction LR - state "Standard Agent Flow" as standard_agent_flow { - [*] --> UpdateSystem: Start - UpdateSystem --> CompileContext: Updated Metrics & Size - CompileContext --> ProcessLLM - ProcessLLM --> ValidateXML: LLM Response - ValidateXML --> ParseResponse: Valid XML - ValidateXML --> UpdateEntries: Invalid XML\nCreate ParseErrorEntry - ParseResponse --> ExecuteCommands: Command - ParseResponse --> UpdateEntries: Entry - ExecuteCommands --> [*]: Stop Command - ExecuteCommands --> UpdateEntries: Delete Command - UpdateEntries --> UpdateSystem: Continue Loop - } -``` - -#### Web Agent - -```mermaid -classDiagram - class BaseAgent { - <> - -working_memory: WorkingMemory - -metrics: SystemMetrics - -llm: LLMEngine - -parser: ResponseParser - -validator: XMLValidator - -action_schema: str - - #_compile_context() str - } - - class StandardAgent { - +StandardAgent(model_path str, system_prompt str, action_schema str) - +run() void - } - - class WebAgent { - +context: str - +response: str readonly - +current_state WebAgentState readonly - +command_result Optional[CommandResult] readonly - +validation_error Optional[str] readonly - - +add_state_change_handler(handler Callable) void - +add_response_change_handler(handler Callable) void - +approve_context() void - +set__response(response str) void - +approve_response() void - } - - class WebAgentState { - - <> - UPDATE - CONTEXT_APPROVAL - INFERENCE - RESPONSE_APPROVAL - STOPPED - } - - class WebSocketManager { - -web_sockets: Set~WebSocket~ - - +WebServer(agent WebAgent, io_buffer WebIOBuffer, static_files path, host str, port int) - } - - class ClientMessage { - <> - APPROVE_CONTEXT - APPROVE_RESPONSE - MODIFY_RESPONSE - SEND_INPUT - } - - class ServerMessage { - <> - STATE_CHANGE - CONTEXT_UPDATE - RESPONSE_UPDATE - OUTPUT_UPDATE - VALIDATION_ERROR - } - - class WebIOBuffer { - -stdin_buffer: str - -stdout_buffer: str - - +read() str - +write(content str) void - +buffer_length() int - +append_stdin(content str) void - +get_stdout() str - +clear_stdout() void - } - - BaseAgent <|-- WebAgent - BaseAgent <|-- StandardAgent - - WebServer --> ClientMessage - WebServer --> ServerMessage - - WebServer "1" *-- "1" WebIOBuffer - WebServer "1" *-- "1" WebAgent - WebAgent "1" *-- "1" WebAgentState -``` - -#### Web Agent Flow - -```mermaid -stateDiagram-v2 - direction LR - state "Web Agent Flow" as web_agent_flow { - [*] --> UpdateSystem: Start - UpdateSystem --> CompileContext: Updated Metrics & Size - CompileContext --> WaitForContextApproval: Send Context - WaitForContextApproval --> ProcessLLM: Context Approved - ProcessLLM --> ValidateXML: LLM Response - ValidateXML --> WaitForResponseApproval: Send Validation Result - ValidateXML --> UpdateEntries: Invalid XML\nCreate ParseErrorEntry - - WaitForResponseApproval --> ValidateXML: Modified Response - WaitForResponseApproval --> ParseResponse: Approved Response - - ParseResponse --> ExecuteCommands: Command - ParseResponse --> UpdateEntries: Entry - - ExecuteCommands --> [*]: Stop Command - ExecuteCommands --> UpdateEntries: Delete Command - - UpdateEntries --> UpdateSystem: Continue Loop - } -``` - -#### Entry classes - -```mermaid -classDiagram - class Entry { - <> - +id: str readonly - +timestamp: datetime readonly - - +Entry(id str, timestamp datetime) - +update() void* - +generate_context() ElementTree* - +cleanup() void* - } - - class SingleEntry { - +script: str readonly - +stdout: str readonly - +stderr: str readonly - +exit_code: Optional~int~ readonly - - +Single(script str, id str, timestamp datetime) - +update() void - +generate_context() ElementTree - } - - class RepeatEntry { - +script: str readonly - +stdout: str readonly - +stderr: str readonly - +exit_code: Optional~int~ readonly - - +RepeatEntry(script str, id str, timestamp datetime) - +update() void - +generate_context() ElementTree - } - - class ReasoningEntry { - +content: str readonly - - +ReasoningEntry(content str, id str, timestamp datetime) - +update() void - +generate_context() ElementTree - } - - class ParseErrorEntry { - +content: str readonly - +error: str readonly - - +ParseErrorEntry(content str, error str, id str, timestamp datetime) - +update() void - +generate_context() ElementTree - } - - class ReadEntry { - +content: str readonly - - +ReadEntry(io_buffer IOBuffer, id str, timestamp datetime) - +update() void - +generate_context() ElementTree - } - - class WriteEntry { - +content: str readonly - - +WriteEntry(content str, io_buffer IOBuffer, id str, timestamp datetime) - +update() void - +generate_context() ElementTree - } - - ReasoningEntry --|> Entry - ParseErrorEntry --|> Entry - ReadEntry --|> Entry - Entry <|-- WriteEntry - Entry <|-- SingleEntry - Entry <|-- RepeatEntry -``` - -#### IO Buffer classes - -```mermaid -classDiagram - class IOBuffer { - <> - +read() str* - +write(content str) void* - +buffer_length() int* - } - - class StandardIOBuffer { - +StandardIOBuffer() - +read() str - +write(content str) void - +buffer_length() int - } - - class WebIOBuffer { - -stdin_buffer: str - -stdout_buffer: str - - +read() str - +write(content str) void - +buffer_length() int - +append_stdin(content str) void - +get_stdout() str - +clear_stdout() void - } - - IOBuffer <|.. WebIOBuffer - IOBuffer <|.. StandardIOBuffer -``` - -#### Command classes - -```mermaid -classDiagram - direction LR - class Command { - <> - +execute(memory WorkingMemory) CommandResult* - } - - class DeleteCommand { - +DeleteCommand(id str) - +execute(memory WorkingMemory) CommandResult - } - - class StopCommand { - +StopCommand() - +execute(memory WorkingMemory) CommandResult - } - - class CommandResult { - +message: str - +success: bool - +should_stop: bool - - +CommandResult(message str, success bool, should_stop bool) - +static success() CommandResult - +static failure(message str) CommandResult - +static stop() CommandResult - } - - Command <|-- DeleteCommand - Command <|-- StopCommand - Command -- CommandResult -``` - -===== FILE: system_prompt.md ===== -You are SIA, the Self Improving Agent. -Your goal is to autonomously complete complex tasks by writing and executing scripts. -You can solve any problem. - -Each iteration, the context is updated with the result of your previous actions. -You modify the context by issuing a command using XML. -Parameters and scripts may be long and complex. -Use correct XML escaping or CDATA sections. -It is very important that you always respond with one action adhering to the XML schema! -Do not respond with anything else after the first action. - -The next iteration starts when all scripts have finished. -These are repeat scripts from the previous iterations and possibly one new single-shot script. -Avoid blocking scripts so you can iterate quickly. - -# Context - -The context has a limited length. -The `context_usage` attribute of the main context element indicates how much of the context is used in %. -This should never reach 100%! -Use the delete action to remove unnecessary items from the context. -But keep interesting information. -You can't learn from your mistakes if you delete them before fixing. - -# Linux Environment - -You have access to the Linux environment that runs the SAI process. -In this environment you can run scripts by issuing the right actions. -Scripts and their output appear in the context. -You can use a script for starting a detached process that runs in the background. -All processes can be managed by the usual Linux tools. -The scripts defined in the script actions all run in a `bash` shell. -You are logged in as root. - -# File system - -The file system helps you structure your thoughts. -Because of the limited context window you can't remember everything you've done and learned. -Writing and updating files will help you in: -- remembering tasks -- planning solution strategies -- keeping track of progress -- managing overview of large projects -- using tools you've created - -It is important to bring a lot of structure to the files and directories. -This will help you find the right info when needed. -When solving a problem, make sure to load the relevant info in context before planning. -You can load a single file with a `cat` command executed in a `single` action. -`head`, `tail`, `grep`, `find`, `tree`, ... all have their uses. - -For code source files it may be interesting to add line numbers. -More advanced scripts can be used, for instance to extract documentation from source files. -This helps you to know how to use a file without loading all the code in context too. - -If it isn't clear what you should do next, check the filesystem for notes that may guide you! - -# Iterative Problem Solving - -Take small steps and verify your work. -Create unit tests for all your work so you can do regression tests after each step. - -Keep notes of when you started on a subtask and which solutions you tried. -This way you avoid repeating yourself and decide when to look for an alternative approach to a problem. - -Version control tools help remember steps taken, solutions tried and files modified. -Make extensive use of `git`! - -Your most important tool is the reasoning action. -You should reason about everything you'll do before issuing a command in the next iteration! -Inspect your previous actions in detail. -If something didn't work, try to understand why and don't repeat the same mistake. -Don't delete mistakes until you understand them. - -If you notice parse_error entries in the context you have made a mistake. -Reason about the error before trying again. - -# User interaction - -You are always working for a user. -Get to know them and make notes about what you learn from them. -Be a helpful assistant to the user. -Open the relevant user notes when you interact with them. - -The main way to communicate is using standard io. -The user may want you to set up alternative communication methods. -Use scripts and background processes to do so. - -The user may take some time to respond or may forget to respond. -Keep detailed notes of your interactions and your expectations regarding time! -Avoid overflowing the user with many messages. -===== FILE: tools/itb/README.md ===== -# ITB: The Interactive Text Browser - -## Understanding the Challenge - -Modern websites are designed for human interaction. They rely on visual cues, spatial relationships, and complex interactions through mouse and keyboard. A human can instantly understand that a blue underlined piece of text is clickable, that a grid of product cards represents a catalog, or that a hamburger icon will reveal a navigation menu. - -AI agents, however, process information differently. They need structured data that explicitly describes both content and interaction possibilities. Traditional web scraping tools can extract content but struggle with interaction. Screen readers come closer but are optimized for linear reading rather than complex interaction patterns. - -ITB (Interactive Text Browser) bridges this gap. It transforms web pages into a format that AI agents can process effectively while preserving all the interaction capabilities a human would have. This transformation focuses on what matters for interaction - the content that's visible and the ways it can be manipulated - while removing technical complexity that only exists to support visual presentation. - -## How ITB Works - -ITB operates as a bridge between an AI agent and a web browser. When you start ITB, it launches a browser process that runs independently, just like when you open a browser yourself. This browser loads and renders web pages normally, but instead of displaying them on screen, it communicates their content and state to the AI agent through a simplified XML format. - -## Page Representation - -ITB transforms complex HTML structures into a simplified XML format that represents what's actually visible and interactive on the page. Let's look at several examples to understand this transformation. - -### Text Content - -When a human reads a webpage, they naturally process related content together regardless of how it's structured in HTML: - -```html - -
-

Breaking News

-
- 2024-12-20 - By John Smith -
-

Scientists announce breakthrough...

-
- - - - Breaking News - 2024-12-20 - By John Smith - Scientists announce breakthrough... - -``` - -The transformation merges related text elements into a single coherent flow, making it easier for AI agents to process content the way a human would read it. - -### Interactive Elements - -ITB describes elements by their interaction capabilities rather than their HTML structure: - -```html - - - - - - Username: - - Password: - - Login - -``` - -### Media Content - -Media elements like images, videos, and audio are represented simply with their source and alternative text: - -```html - -
- System architecture diagram - - -
- - - - System architecture diagram - -``` - -### Embedded Content - -Modern web pages often embed content from other sources using iframes. While humans don't notice these technical boundaries, they affect how we can interact with the content. ITB handles this by representing iframes transparently: - -```html - -
-

Welcome Back

- -
- - - - Welcome Back -