From aad9564e817486251ddebdf9217b228d0e649779 Mon Sep 17 00:00:00 2001 From: geens Date: Wed, 13 Nov 2024 11:43:48 +0100 Subject: [PATCH 1/5] Added examples to action schema --- action_schema.xsd | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/action_schema.xsd b/action_schema.xsd index b131bfc..8f22353 100644 --- a/action_schema.xsd +++ b/action_schema.xsd @@ -1,4 +1,7 @@ + @@ -17,6 +22,8 @@ Stop command terminates the agent gracefully. For the main SIA instance this will trigger an update and restart. For sub-instances this is the correct way to stop after all tasks are complete. + Example: + --> @@ -26,6 +33,10 @@ Single script that runs once and completes. Output is stored in context until explicitly deleted. Used for one-time operations like file manipulation. + Example: + + ls / + --> @@ -40,6 +51,10 @@ After a command is issued, all repeat scripts in context are run again. Useful for monitoring changing files or viewing results immediately after changing a file. Repeat scripts should execute quickly to avoid blocking the agent. + Example: + + ls / + --> @@ -53,6 +68,10 @@ As an agent it is important to reason about your actions and their results. In a reasoning action you can write freeform text. This is also stored in context until deleted. + Example: + + I should explore the file system for interesting files. + --> @@ -64,6 +83,8 @@ @@ -72,6 +93,10 @@ From 50f05b621384bb8005a35c7e071934c4963045cc Mon Sep 17 00:00:00 2001 From: geens Date: Wed, 13 Nov 2024 11:44:58 +0100 Subject: [PATCH 2/5] Better response parsing --- sia/response_parser.py | 84 ++++++++---------------------------------- 1 file changed, 16 insertions(+), 68 deletions(-) diff --git a/sia/response_parser.py b/sia/response_parser.py index 040c433..41dacac 100644 --- a/sia/response_parser.py +++ b/sia/response_parser.py @@ -36,46 +36,6 @@ class ResponseParser: io_buffer: Buffer to use for IO operations """ self.io_buffer = io_buffer - - def _extract_content(self, xml: str, root: ET.Element) -> str: - """ - Extract content from XML, handling CDATA sections and preserving special characters. - - Args: - xml: Original XML string - root: Parsed XML element - - Returns: - str: Extracted content with proper handling of CDATA and special characters - """ - if root.text is None: - return "" - - # Find opening tag end - tag_name = root.tag - tag_start = xml.find('<' + tag_name) - tag_end = xml.find('>', tag_start) - if tag_end == -1: - return "" - - # Check for self-closing tag - if xml[tag_end-1] == '/': - return "" - - # Find closing tag start - close_tag = '' - close_start = xml.rfind(close_tag) - if close_start == -1: - return "" - - # Extract content - content = xml[tag_end+1:close_start] - - # Handle CDATA sections - if content.startswith(''): - content = content[9:-3] # Remove CDATA markers - - return content def parse(self, xml: str) -> Union[Command, Entry]: """ @@ -91,61 +51,49 @@ class ResponseParser: entry_id = str(uuid.uuid4()) timestamp = datetime.now() - # First try to parse the XML try: root = ET.fromstring(xml.strip()) except ET.ParseError as e: return ParseErrorEntry(xml, f"Invalid XML: {str(e)}", entry_id, timestamp) - - # Extract content preserving special characters and handling CDATA - content = self._extract_content(xml, root) - # Convert to appropriate type based on root tag try: if root.tag == 'delete': - # Extract target ID and create delete command target_id = root.get('id') if not target_id: return ParseErrorEntry(xml, "Delete command missing required 'id' attribute", entry_id, timestamp) return DeleteCommand(target_id) elif root.tag == 'stop': - # Create stop command return StopCommand() elif root.tag == 'background': - # Create background script entry - if not content: - return ParseErrorEntry(xml, "Background entry missing script content", entry_id, timestamp) - return BackgroundEntry(content, entry_id, timestamp) + if len(root) != 0 or root.attrib or root.text is None or root.text.strip() == '': + return ParseErrorEntry(xml, "Background entry requires (only) script content", entry_id, timestamp) + return ParseErrorEntry(xml, "Background entry requires script content", entry_id, timestamp) + return BackgroundEntry(root.text, entry_id, timestamp) elif root.tag == 'repeat': - # Create repeat script entry - if not content: - return ParseErrorEntry(xml, "Repeat entry missing script content", entry_id, timestamp) - return RepeatEntry(content, entry_id, timestamp) + if len(root) != 0 or root.attrib or root.text is None or root.text.strip() == '': + return ParseErrorEntry(xml, "Repeat entry requires (only) script content", entry_id, timestamp) + return RepeatEntry(root.text, entry_id, timestamp) elif root.tag == 'single': - # Create single shot script entry - if not content: - return ParseErrorEntry(xml, "Single shot entry missing script content", entry_id, timestamp) - return SingleEntry(content, entry_id, timestamp) + if len(root) != 0 or root.attrib or root.text is None or root.text.strip() == '': + return ParseErrorEntry(xml, "Single entry requires (only) script content", entry_id, timestamp) + return SingleEntry(root.text, entry_id, timestamp) elif root.tag == 'reasoning': - # Create reasoning entry - if not content: - return ParseErrorEntry(xml, "Reasoning entry missing content", entry_id, timestamp) - return ReasoningEntry(content, entry_id, timestamp) + if len(root) != 0 or root.attrib or root.text is None or root.text.strip() == '': + return ParseErrorEntry(xml, "Reasoning entry requires (only) text content", entry_id, timestamp) + return ReasoningEntry(root.text, entry_id, timestamp) elif root.tag == 'read_stdin': - # Create read entry - no content required return ReadEntry(self.io_buffer, entry_id, timestamp) elif root.tag == 'write_stdout': - # Create write entry - if not content: - return ParseErrorEntry(xml, "Write entry missing content", entry_id, timestamp) - return WriteEntry(content, self.io_buffer, entry_id, timestamp) + if len(root) != 0 or root.attrib or root.text is None or root.text.strip() == '': + return ParseErrorEntry(xml, "Write stdout entry requires (only) text content", entry_id, timestamp) + return WriteEntry(root.text, self.io_buffer, entry_id, timestamp) else: return ParseErrorEntry(xml, f"Unknown root element: {root.tag}", entry_id, timestamp) From 8e3914528b5cb636cb9faafdb9cfe3e07274d820 Mon Sep 17 00:00:00 2001 From: geens Date: Wed, 13 Nov 2024 11:45:44 +0100 Subject: [PATCH 3/5] More details in system prompt --- system_prompt.md | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/system_prompt.md b/system_prompt.md index 684da32..7e6ca33 100644 --- a/system_prompt.md +++ b/system_prompt.md @@ -3,24 +3,31 @@ Your goal is to autonomously complete complex tasks by writing and executing scr You can solve any problem. Each iteration, the context is updated with the result of your previous actions. -You modify the context by issuing a commands using XML. +You modify the context by issuing a command using XML. Parameters and scripts may be long and complex. Use correct XML escaping or CDATA sections. It is very important that you always respond with one action adhering to the XML schema! -Do not respond with anything else after the action. +Do not respond with anything else after the first action. + +The next iteration starts when all scripts have finished. +These are repeat scripts from the previous iterations and possibly one new single-shot script. +Avoid blocking scripts so you can iterate quickly. # Context + The context has a limited length. The `context_usage` attribute of the main context element indicates how much of the context is used in %. This should never reach 100%! -Use the delete action to remove unnecessary items from the context as soon as possible. +Use the delete action to remove unnecessary items from the context. +But keep interesting information. +You can't learn from your mistakes if you delete them before fixing. # Linux Environment You have access to the Linux environment that runs the SAI process. -In this environment you can run scripts. -Scripts are usually managed by the SIA process and kept in context. -From a managed process you can also start detached processes. +In this environment you can run scripts by issuing the right actions. +Scripts and their output appear in the context. +You can use a script for starting a detached process that runs in the background. All processes can be managed by the usual Linux tools. The scripts defined in the script actions all run in a `bash` shell. @@ -45,6 +52,8 @@ For code source files it may be interesting to add line numbers. More advanced scripts can be used, for instance to extract documentation from source files. This helps you to know how to use a file without loading all the code in context too. +If it isn't clear what you should do next, check the filesystem for notes that may guide you! + # Iterative Problem Solving Take small steps and verify your work. @@ -56,6 +65,9 @@ This way you avoid repeating yourself and decide when to look for an alternative Version control tools help remember steps taken, solutions tried and files modified. Make extensive use of `git`! +Your most important tool is the reasoning action. +You should reason about everything you'll do before you issue a command! + # User interaction You are always working for a user. @@ -65,7 +77,8 @@ Open the relevant user notes when you interact with them. The main way to communicate is using standard io. The user may want you to set up alternative communication methods. -User scripts and background processes to do so. +Use scripts and background processes to do so. The user may take some time to respond or may forget to respond. -Keep notes of your interaction and your expectations. +Keep detailed notes of your interactions and your expectations regarding time! +Avoid overflowing the user with many messages. \ No newline at end of file From 832f712b1b97933cc7f7b86877c743679d61ab3e Mon Sep 17 00:00:00 2001 From: geens Date: Wed, 13 Nov 2024 11:46:06 +0100 Subject: [PATCH 4/5] Enable word wrap in editor --- web/src/components/editors/StandardEditor.jsx | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/components/editors/StandardEditor.jsx b/web/src/components/editors/StandardEditor.jsx index 9879215..132059c 100644 --- a/web/src/components/editors/StandardEditor.jsx +++ b/web/src/components/editors/StandardEditor.jsx @@ -28,6 +28,7 @@ export const StandardEditor = ({ readOnly, fontSize: 14, automaticLayout: true, + wordWrap: 'on', }} /> From 229f4f4ac7994b5a12abfcde9c702e75a8aacb2b Mon Sep 17 00:00:00 2001 From: geens Date: Wed, 13 Nov 2024 11:46:29 +0100 Subject: [PATCH 5/5] Added openai llm engine --- requirements.txt | 1 + sia/__main__.py | 7 ++++- sia/openai_llm_engine.py | 57 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 sia/openai_llm_engine.py diff --git a/requirements.txt b/requirements.txt index 07d56b5..da16518 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ accelerate aiohttp bs4 +openai python-dotenv torch transformers \ No newline at end of file diff --git a/sia/__main__.py b/sia/__main__.py index 8338bcc..eace4e6 100644 --- a/sia/__main__.py +++ b/sia/__main__.py @@ -5,11 +5,11 @@ import asyncio import mimetypes import time - from .config import Config from .hf_llm_engine import HfLlmEngine from .llm_engine import LlmEngine from .local_llm_engine import LocalLlmEngine +from .openai_llm_engine import OpenAILlmEngine from .response_parser import ResponseParser from .system_metrics import SystemMetrics from .web_agent import WebAgent @@ -50,6 +50,11 @@ class Main: model_id=self._config.model, api_token=self._config.api_token ) + case "openai": + self._llm = OpenAILlmEngine( + model=self._config.model, + api_key=self._config.api_token + ) case "test": self._llm = TestLLM() case _: diff --git a/sia/openai_llm_engine.py b/sia/openai_llm_engine.py new file mode 100644 index 0000000..88607a8 --- /dev/null +++ b/sia/openai_llm_engine.py @@ -0,0 +1,57 @@ +from typing import Iterator, Optional +import openai +import json + +from .llm_engine import LlmEngine + +class OpenAILlmEngine(LlmEngine): + """ + LLM Engine implementation using OpenAI's API. + Supports streaming responses from chat completion models. + """ + + def __init__( + self, + model: str, + api_key: str, + ): + """ + Initialize the OpenAI LLM Engine. + + Args: + model: OpenAI model to use (default: gpt-4) + api_key: OpenAI API key. If None, will try to read from OPENAI_API_KEY env var + """ + self._model = model + + # Initialize OpenAI client + self.client = openai.Client( + api_key=api_key, + ) + + def infer(self, system_prompt: str, main_context: str) -> Iterator[str]: + """ + Run inference using the system prompt and main context. + + Args: + system_prompt: The system prompt string + main_context: The main context string after templating + + Returns: + Iterator[str]: An iterator that yields the generated text in chunks. + """ + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": main_context} + ] + + stream = self.client.chat.completions.create( + model=self._model, + messages=messages, + stream=True, + temperature=0.3, + ) + + for chunk in stream: + if content := chunk.choices[0].delta.content: + yield content \ No newline at end of file