Response parser and split up class diagrams

readme.md

@@ -267,7 +267,100 @@ This architecture allows for:
 - Human-in-the-loop operation for testing and development
 - Collection of human feedback for reinforcement learning
 
-### Standard Agent Flow
+### Diagrams
+
+#### Core classes
+
+```mermaid
+classDiagram
+    class SystemMetrics {
+        +SystemMetrics(sample_interval float)
+        +generate_context(context_usage float) ElementTree
+        +stop() void
+        -monitor_loop() void
+    }
+
+    class LLMEngine {
+        +LLMEngine(model_path str)
+        +set_model_path(model_path str) void
+        +infer(system_prompt str, main_context str) Iterator~str~
+    }
+
+    class BaseAgent {
+        <<abstract>>
+        #working_memory: WorkingMemory
+        #metrics: SystemMetrics
+        #llm: LLMEngine
+        #parser: ResponseParser
+        #validator: XMLValidator
+        #io_buffer: IOBuffer
+
+        #compile_context() str
+    }
+
+    class WorkingMemory {
+        -entries: List~Entry~
+
+        +WorkingMemory()
+        +add_entry(entry Entry) void
+        +remove_entry(id str) void
+        +clear() void
+        +get_entry(id str) Optional~Entry~
+        +get_entries() List~Entry~
+        +get_entries_count() int
+        +get_entries_by_type(type Type) List~Entry~
+        +update() void
+        +generate_context() List~ElementTree~
+    }
+
+    class XMLValidator {
+        +XMLValidator(schema str)
+        +validate(xml str) Optional~str~
+        +get_valid_root_elements() Set~str~
+    }
+
+    class ResponseParser {
+        -io_buffer: IOBuffer
+
+        +ResponseParser(io_buffer IOBuffer)
+        +parse(xml str) Union~Command, Entry~
+    }
+
+    class Entry {
+        <<abstract>>
+        +id: str
+        +timestamp: datetime
+
+        +Entry(id str, timestamp datetime)
+        +update() void*
+        +generate_context() ElementTree*
+        +cleanup() void*
+    }
+
+    class IOBuffer {
+        <<interface>>
+        +read() str*
+        +write(content str) void*
+        +buffer_length() int*
+    }
+
+    class Command {
+        <<abstract>>
+        +execute(memory WorkingMemory) CommandResult*
+    }
+
+    SystemMetrics "1" --* "1" BaseAgent
+    LLMEngine "1" --* "1" BaseAgent
+    XMLValidator "1" --* "1" BaseAgent
+    BaseAgent "1" *-- "1" IOBuffer
+    BaseAgent "1" *-- "1" WorkingMemory
+    BaseAgent "1" *-- "1" ResponseParser
+    WorkingMemory "1" *-- "*" Entry
+    ResponseParser ..> Entry
+    ResponseParser ..> Command
+```
+
+#### Standard Agent Flow
 
 ```mermaid
 stateDiagram-v2
@@ -287,7 +380,74 @@ stateDiagram-v2
     }
 ```
 
-### Web Agent Flow
+#### Web Agent
+
+```mermaid
+classDiagram
+    class BaseAgent {
+        <<abstract>>
+        #working_memory: WorkingMemory
+        #metrics: SystemMetrics
+        #llm: LLMEngine
+        #parser: ResponseParser
+        #validator: XMLValidator
+        #io_buffer: IOBuffer
+
+        #compile_context() str
+    }
+
+    class StandardAgent {
+        +StandardAgent(model_path str, system_prompt str, action_schema str)
+        +run() void
+    }
+
+    class WebAgent {
+        -current_state: WebAgentState
+
+        +WebAgent(model_path str, system_prompt str, action_schema str)
+        +get_current_state() WebAgentState
+        +proceed() void
+        +add_state_change_handler(handler func) void
+    }
+
+    class WebAgentState {
+        <<enumeration>>
+        UPDATE
+        CONTEXT_APPROVAL
+        INFERENCE
+        RESPONSE_APPROVAL
+    }
+
+    class WebServer {
+        -agent: WebAgent
+        -clients: List~WebSocket~
+
+        +WebServer(agent WebAgent)
+        -broadcast_state_change() void
+    }
+
+    class WebIOBuffer {
+        -stdin_buffer: str
+        -stdout_buffer: str
+
+        +WebIOBuffer()
+        +read() str
+        +write(content str) void
+        +buffer_length() int
+        +append_stdin(content str) void
+        +get_stdout() str
+        +clear_stdout() void
+    }
+
+    BaseAgent <|-- WebAgent
+    BaseAgent <|-- StandardAgent
+
+    WebServer "1" *-- "1" WebIOBuffer
+    WebServer "1" *-- "1" WebAgent
+    WebAgent "1" *-- "1" WebAgentState
+```
+
+#### Web Agent Flow
 
 ```mermaid
 stateDiagram-v2
@@ -314,226 +474,169 @@ stateDiagram-v2
     }
 ```
 
-### Class Diagram
+#### Entry classes
 
 ```mermaid
 classDiagram
-    direction LR
-    class BaseAgent {
-        <<abstract>>
-        #working_memory WorkingMemory
-        #metrics SystemMetrics
-        #llm LLMEngine
-        #parser ResponseParser
-        #validator XMLValidator
-        #io_buffer IOBuffer
-        #compile_context() str
-    }
-
-    class LLMEngine {
-        +LLMEngine(model_path str)
-        +set_model_path(model_path str)
-        +inference(context str) Iterator~str~
-    }
-
-    class StandardAgent {
-        +run()
-    }
-
-    class WebAgent {
-        -current_state WebAgentState
-        +get_current_state() WebAgentState
-        +proceed()
-        +add_state_change_handler(handler func())
-    }
-
-    class WebAgentState {
-        <<enumeration>>
-        UPDATE
-        CONTEXT_APPROVAL
-        INFERENCE
-        RESPONSE_APPROVAL
-    }
-
-    class WorkingMemory {
-        +WorkingMemory()
-        -entries List~Entry~
-        +add_entry(entry Entry)
-        +remove_entry(id str)
-        +update()
-        +generate_context() List~ElementTree~
-    }
-
-    class SystemMetrics {
-        +generate_context(entries List~ElementTree~) ElementTree
-    }
-
-    class XMLValidator {
-        -schema ElementTree
-        +XMLValidator(schema str)
-        +validate(xml str) Optional~str~
-    }
-
-    class ResponseParser {
-        -io_buffer IOBuffer
-        +ResponseParser(io_buffer IOBuffer)
-        +parse(xml str) Command | Entry
-    }
-
     class Entry {
         <<abstract>>
-        +id str
-        +timestamp datetime
+        +id: str
+        +timestamp: datetime
+
         +Entry(id str, timestamp datetime)
-        +update()*
+        +update() void*
         +generate_context() ElementTree*
+        +cleanup() void*
     }
 
     class SingleShotEntry {
-        +script str
-        +stdout str
-        +stderr str
-        +exit_code int
+        +script: str
+        +stdout: str
+        +stderr: str
+        +exit_code: Optional~int~
+
         +SingleShotEntry(script str, id str, timestamp datetime)
-        +update()
+        +update() void
         +generate_context() ElementTree
     }
 
     class RepeatEntry {
-        +script str
-        +stdout str
-        +stderr str
-        +exit_code int
+        +script: str
+        +stdout: str
+        +stderr: str
+        +exit_code: Optional~int~
+
         +RepeatEntry(script str, id str, timestamp datetime)
-        +update()
+        +update() void
         +generate_context() ElementTree
     }
 
     class BackgroundEntry {
-        +script str
-        +stdout str
-        +stderr str
-        +process Process
+        +script: str
+        +stdout: str
+        +stderr: str
+        +process: Optional~Process~
+        -accumulated_stdout: str
+        -accumulated_stderr: str
+        -exit_code: Optional~int~
+
         +BackgroundEntry(script str, id str, timestamp datetime)
-        +update()
+        +update() void
         +generate_context() ElementTree
+        +cleanup() void
     }
 
     class ReasoningEntry {
-        +content str
+        +content: str
+
         +ReasoningEntry(content str, id str, timestamp datetime)
-        +update()
+        +update() void
         +generate_context() ElementTree
     }
 
     class ParseErrorEntry {
-        +content str
-        +error str
+        +content: str
+        +error: str
+
         +ParseErrorEntry(content str, error str, id str, timestamp datetime)
-        +update()
+        +update() void
         +generate_context() ElementTree
     }
 
     class ReadEntry {
-        +content str
+        +content: str
+        +io_buffer: IOBuffer
+
         +ReadEntry(io_buffer IOBuffer, id str, timestamp datetime)
-        +update()
+        +update() void
         +generate_context() ElementTree
     }
 
     class WriteEntry {
-        +content str
+        +content: str
+        +io_buffer: IOBuffer
+
         +WriteEntry(content str, io_buffer IOBuffer, id str, timestamp datetime)
-        +update()
+        +update() void
         +generate_context() ElementTree
     }
 
+    ReasoningEntry --|> Entry
+    ParseErrorEntry --|> Entry
+    ReadEntry --|> Entry
+    Entry <|-- WriteEntry
+    Entry <|-- SingleShotEntry
+    Entry <|-- RepeatEntry
+    Entry <|-- BackgroundEntry
+```
+
+#### IO Buffer classes
+
+```mermaid
+classDiagram
     class IOBuffer {
         <<interface>>
         +read() str*
-        +write(content str)*
+        +write(content str) void*
         +buffer_length() int*
     }
 
     class StandardIOBuffer {
         +StandardIOBuffer()
         +read() str
-        +write(content str)
+        +write(content str) void
         +buffer_length() int
     }
 
     class WebIOBuffer {
-        -stdin_buffer str
-        -stdout_buffer str
+        -stdin_buffer: str
+        -stdout_buffer: str
+
         +WebIOBuffer()
         +read() str
-        +write(content str)
+        +write(content str) void
         +buffer_length() int
-        +append_stdin(content str)
+        +append_stdin(content str) void
         +get_stdout() str
-        +clear_stdout()
+        +clear_stdout() void
+    }
+
+    IOBuffer <|.. WebIOBuffer
+    IOBuffer <|.. StandardIOBuffer
+```
+
+#### Command classes
+
+```mermaid
+classDiagram
+    direction LR
+    class Command {
+        <<abstract>>
+        +execute(memory WorkingMemory) CommandResult*
+    }
+
+    class DeleteCommand {
+        +DeleteCommand(id str)
+        +execute(memory WorkingMemory) CommandResult
+    }
+
+    class StopCommand {
+        +StopCommand()
+        +execute(memory WorkingMemory) CommandResult
     }
 
     class CommandResult {
-        +should_stop bool
-        +success bool
-        +message str
-        +CommandResult(should_stop bool, success bool, message str)
+        +message: str
+        +success: bool
+        +should_stop: bool
+
+        +CommandResult(message str, success bool, should_stop bool)
         +static success() CommandResult
         +static failure(message str) CommandResult
         +static stop() CommandResult
     }
 
-    class Command {
-        <<abstract>>
-        +execute(memory &WorkingMemory) CommandResult*
-    }
-
-    class DeleteCommand {
-        +id str
-        +DeleteCommand(id str)
-        +execute(memory &WorkingMemory) CommandResult
-    }
-
-    class StopCommand {
-        +execute(memory &WorkingMemory) CommandResult
-    }
-
-    class WebServer {
-        -agent WebAgent
-        -clients List~WebSocket~
-        +WebServer(agent WebAgent)
-        -broadcast_state_change()
-    }
-
-    BaseAgent <|-- WebAgent
-    BaseAgent <|-- StandardAgent
-    BaseAgent "1" *-- "1" IOBuffer
-    BaseAgent "1" *-- "1" WorkingMemory
-    BaseAgent "1" *-- "1" SystemMetrics
-    BaseAgent "1" *-- "1" XMLValidator
-    BaseAgent "1" *-- "1" LLMEngine
-    BaseAgent "1" *-- "1" ResponseParser
-
-    WebServer "1" *-- "1" WebIOBuffer
-    WebServer "1" *-- "1" WebAgent
-
-    WebAgent "1" *-- "1" WebAgentState
-
-    WorkingMemory "1" *-- "*" Entry
-
-    Entry <|-- ReasoningEntry
-    Entry <|-- ParseErrorEntry
-    Entry <|-- ReadEntry
-    Entry <|-- WriteEntry
-    Entry <|-- SingleShotEntry
-    Entry <|-- RepeatEntry
-    Entry <|-- BackgroundEntry
-
     Command <|-- DeleteCommand
     Command <|-- StopCommand
     Command -- CommandResult
-
-    IOBuffer <|.. WebIOBuffer
-    IOBuffer <|.. StandardIOBuffer
 ```

sia/response_parser.py

@@ -0,0 +1,154 @@
+from datetime import datetime
+import uuid
+import xml.etree.ElementTree as ET
+from typing import Union
+
+from .command import Command 
+from .delete_command import DeleteCommand
+from .stop_command import StopCommand
+
+from .entry import Entry
+from .background_entry import BackgroundEntry
+from .parse_error_entry import ParseErrorEntry
+from .read_entry import ReadEntry
+from .reasoning_entry import ReasoningEntry
+from .repeat_entry import RepeatEntry
+from .single_shot_entry import SingleShotEntry
+from .write_entry import WriteEntry
+
+class ResponseParser:
+    """
+    Parses XML responses from the LLM into commands or entries.
+    
+    The parser validates the XML structure and converts it into the appropriate
+    Command or Entry object based on the root element tag. Invalid input results
+    in ParseErrorEntry objects rather than raising exceptions.
+    
+    Attributes:
+        io_buffer: Buffer to use for IO operations
+    """
+    
+    def __init__(self, io_buffer):
+        """
+        Initialize parser with IO buffer.
+        
+        Args:
+            io_buffer: Buffer to use for IO operations
+        """
+        self.io_buffer = io_buffer
+
+    def _extract_content(self, xml: str, root: ET.Element) -> str:
+        """
+        Extract content from XML, handling CDATA sections and preserving special characters.
+        
+        Args:
+            xml: Original XML string
+            root: Parsed XML element
+            
+        Returns:
+            str: Extracted content with proper handling of CDATA and special characters
+        """
+        if root.text is None:
+            return ""
+            
+        # Find opening tag end
+        tag_name = root.tag
+        tag_start = xml.find('<' + tag_name)
+        tag_end = xml.find('>', tag_start)
+        if tag_end == -1:
+            return ""
+            
+        # Check for self-closing tag
+        if xml[tag_end-1] == '/':
+            return ""
+            
+        # Find closing tag start
+        close_tag = '</' + tag_name + '>'
+        close_start = xml.rfind(close_tag)
+        if close_start == -1:
+            return ""
+            
+        # Extract content
+        content = xml[tag_end+1:close_start]
+        
+        # Handle CDATA sections
+        if content.startswith('<![CDATA[') and content.endswith(']]>'):
+            content = content[9:-3]  # Remove CDATA markers
+            
+        return content
+        
+    def parse(self, xml: str) -> Union[Command, Entry]:
+        """
+        Parse XML response into a Command or Entry.
+        
+        Args:
+            xml: XML string to parse
+            
+        Returns:
+            Command or Entry based on the XML content
+            ParseErrorEntry for any parsing or validation errors
+        """
+        entry_id = str(uuid.uuid4())
+        timestamp = datetime.now()
+        
+        # First try to parse the XML
+        try:
+            root = ET.fromstring(xml.strip())
+        except ET.ParseError as e:
+            return ParseErrorEntry(xml, f"Invalid XML: {str(e)}", entry_id, timestamp)
+            
+        # Extract content preserving special characters and handling CDATA
+        content = self._extract_content(xml, root)
+        
+        # Convert to appropriate type based on root tag
+        try:
+            if root.tag == 'delete':
+                # Extract target ID and create delete command
+                target_id = root.get('id')
+                if not target_id:
+                    return ParseErrorEntry(xml, "Delete command missing required 'id' attribute", entry_id, timestamp)
+                return DeleteCommand(target_id)
+                
+            elif root.tag == 'stop':
+                # Create stop command
+                return StopCommand()
+                
+            elif root.tag == 'background':
+                # Create background script entry
+                if not content:
+                    return ParseErrorEntry(xml, "Background entry missing script content", entry_id, timestamp)
+                return BackgroundEntry(content, entry_id, timestamp)
+                
+            elif root.tag == 'repeat':
+                # Create repeat script entry
+                if not content:
+                    return ParseErrorEntry(xml, "Repeat entry missing script content", entry_id, timestamp)
+                return RepeatEntry(content, entry_id, timestamp)
+                
+            elif root.tag == 'single_shot':
+                # Create single shot script entry
+                if not content:
+                    return ParseErrorEntry(xml, "Single shot entry missing script content", entry_id, timestamp)
+                return SingleShotEntry(content, entry_id, timestamp)
+                
+            elif root.tag == 'reasoning':
+                # Create reasoning entry
+                if not content:
+                    return ParseErrorEntry(xml, "Reasoning entry missing content", entry_id, timestamp)
+                return ReasoningEntry(content, entry_id, timestamp)
+                
+            elif root.tag == 'read_stdin':
+                # Create read entry - no content required
+                return ReadEntry(self.io_buffer, entry_id, timestamp)
+                
+            elif root.tag == 'write_stdout':
+                # Create write entry
+                if not content:
+                    return ParseErrorEntry(xml, "Write entry missing content", entry_id, timestamp)
+                return WriteEntry(content, self.io_buffer, entry_id, timestamp)
+                
+            else:
+                return ParseErrorEntry(xml, f"Unknown root element: {root.tag}", entry_id, timestamp)
+                
+        except Exception as e:
+            return ParseErrorEntry(xml, f"Error parsing response: {str(e)}", entry_id, timestamp)

sia/system_metrics.py

@@ -0,0 +1,122 @@
+import threading
+import time
+from typing import Optional
+import xml.etree.ElementTree as ET
+import psutil
+
+class SystemMetrics:
+    """
+    Tracks system metrics including CPU, GPU, memory and disk usage.
+    Maintains rolling averages for CPU and GPU between context generations.
+    
+    Attributes:
+        _stop_event: Threading event to signal monitor thread to stop
+        _monitor_thread: Background thread that collects usage data
+        _cpu_samples: List of CPU usage samples since last context
+        _gpu_samples: List of GPU usage samples since last context
+        _metrics_lock: Lock for thread-safe access to samples
+    """
+    
+    def __init__(self, sample_interval: float = 0.1):
+        """
+        Initialize system metrics monitoring.
+        
+        Args:
+            sample_interval: Seconds between usage samples
+        """
+        self._stop_event = threading.Event()
+        self._cpu_samples = []
+        self._gpu_samples = []
+        self._metrics_lock = threading.Lock()
+        self._sample_interval = sample_interval
+        
+        # Start monitoring thread
+        self._monitor_thread = threading.Thread(
+            target=self._monitor_loop,
+            daemon=True
+        )
+        self._monitor_thread.start()
+        
+    def _get_gpu_usage(self) -> Optional[float]:
+        """
+        Get current GPU usage percentage.
+        Returns None if GPU monitoring not available.
+        """
+        try:
+            import pynvml
+            pynvml.nvmlInit()
+            handle = pynvml.nvmlDeviceGetHandleByIndex(0)
+            info = pynvml.nvmlDeviceGetUtilizationRates(handle)
+            pynvml.nvmlShutdown()
+            return info.gpu
+        except:
+            return None
+
+    def _monitor_loop(self):
+        """
+        Background loop that collects system usage samples.
+        Runs until stop event is set.
+        """
+        while not self._stop_event.is_set():
+            with self._metrics_lock:
+                # Get CPU usage across all cores
+                self._cpu_samples.append(psutil.cpu_percent())
+                
+                # Get GPU usage if available
+                gpu_usage = self._get_gpu_usage()
+                if gpu_usage is not None:
+                    self._gpu_samples.append(gpu_usage)
+                    
+            time.sleep(self._sample_interval)
+
+    def generate_context(self, context_usage: float) -> ET.Element:
+        """
+        Generate XML context element with current system metrics.
+        Clears usage samples after generating averages.
+        
+        Args:
+            context_usage: Current context size usage (0-1)
+            
+        Returns:
+            ET.Element: Context element with system metrics
+        """
+        # Create context element
+        context = ET.Element("context")
+        
+        # Add timestamp
+        context.set("time", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
+        
+        # Add usage metrics
+        with self._metrics_lock:
+            # CPU average (or 0 if no samples)
+            cpu_avg = round(sum(self._cpu_samples) / len(self._cpu_samples)) if self._cpu_samples else 0
+            context.set("cpu", str(cpu_avg))
+            self._cpu_samples.clear()
+            
+            # GPU average (or 0 if no samples)
+            gpu_avg = round(sum(self._gpu_samples) / len(self._gpu_samples)) if self._gpu_samples else 0
+            context.set("gpu", str(gpu_avg))
+            self._gpu_samples.clear()
+        
+        # Memory usage in bytes
+        memory = psutil.virtual_memory()
+        context.set("memory_used", str(memory.used))
+        context.set("memory_total", str(memory.total))
+        
+        # Disk usage in bytes (root partition)
+        disk = psutil.disk_usage('/')
+        context.set("disk_used", str(disk.used))
+        context.set("disk_total", str(disk.total))
+        
+        # Context usage (0-100)
+        context.set("context", str(round(context_usage * 100)))
+        
+        # Standard input buffer size 
+        context.set("stdin", "0")  # Set by agent
+        
+        return context
+        
+    def stop(self):
+        """Stop the monitoring thread and clean up."""
+        self._stop_event.set()
+        self._monitor_thread.join()

test/llm_engine_test.py

@@ -1,25 +0,0 @@
-import unittest
-
-from itertools import tee
-
-from . import test_data
-
-from sia.llm_engine import LlmEngine
-
-class LlmEngineTest(unittest.TestCase):
-    def setUp(self):
-        self.model_path = "/root/model"
-
-    def test_initialization(self):
-        llm_engine = LlmEngine(self.model_path)
-        self.assertIsInstance(llm_engine, LlmEngine)
-
-    def test_infer(self):
-        main_context = "This is a test"
-        llm_engine = LlmEngine(self.model_path)
-        tokens = llm_engine.infer(test_data.echo_system_prompt, main_context)
-        print_tokens, result_tokens = tee(tokens)
-        for token in print_tokens:
-            print(token, end="", flush=True)
-        result = ''.join(result_tokens)
-        self.assertEqual(result, f"{main_context}<test_tag>{main_context}</test_tag>")

test/response_parser_test.py

@@ -0,0 +1,215 @@
+import unittest
+from datetime import datetime
+import xml.etree.ElementTree as ET
+
+from sia.command import Command
+from sia.delete_command import DeleteCommand
+from sia.stop_command import StopCommand
+from sia.entry import Entry
+from sia.background_entry import BackgroundEntry
+from sia.parse_error_entry import ParseErrorEntry
+from sia.read_entry import ReadEntry
+from sia.reasoning_entry import ReasoningEntry
+from sia.repeat_entry import RepeatEntry
+from sia.single_shot_entry import SingleShotEntry
+from sia.write_entry import WriteEntry
+from sia.web_io_buffer import WebIOBuffer
+from sia.response_parser import ResponseParser
+
+class ResponseParserTest(unittest.TestCase):
+    def setUp(self):
+        """Set up test cases with parser and IO buffer"""
+        self.io_buffer = WebIOBuffer()
+        self.parser = ResponseParser(self.io_buffer)
+        
+    def test_delete_command(self):
+        """Test parsing delete command"""
+        xml = '<delete id="test-id-123"/>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, DeleteCommand)
+        self.assertEqual(result.id, "test-id-123")
+        
+    def test_delete_command_missing_id(self):
+        """Test parsing delete command without ID returns error entry"""
+        xml = '<delete/>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, ParseErrorEntry)
+        self.assertEqual(result.content, xml)
+        self.assertIn("missing required 'id'", result.error)
+        
+    def test_stop_command(self):
+        """Test parsing stop command"""
+        xml = '<stop/>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, StopCommand)
+        
+    def test_background_entry(self):
+        """Test parsing background entry"""
+        xml = '<background>echo test</background>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, BackgroundEntry)
+        self.assertEqual(result.script, "echo test")
+        
+    def test_background_entry_empty_script(self):
+        """Test parsing background entry with empty script returns error entry"""
+        xml = '<background/>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, ParseErrorEntry)
+        self.assertEqual(result.content, xml)
+        self.assertIn("missing script content", result.error)
+        
+    def test_repeat_entry(self):
+        """Test parsing repeat entry"""
+        xml = '<repeat>ls -l</repeat>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, RepeatEntry)
+        self.assertEqual(result.script, "ls -l")
+        
+    def test_repeat_entry_empty_script(self):
+        """Test parsing repeat entry with empty script returns error entry"""
+        xml = '<repeat/>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, ParseErrorEntry)
+        self.assertEqual(result.content, xml)
+        self.assertIn("missing script content", result.error)
+        
+    def test_single_shot_entry(self):
+        """Test parsing single shot entry"""
+        xml = '<single_shot>echo "one time"</single_shot>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, SingleShotEntry)
+        self.assertEqual(result.script, 'echo "one time"')
+        
+    def test_single_shot_entry_empty_script(self):
+        """Test parsing single shot entry with empty script returns error entry"""
+        xml = '<single_shot/>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, ParseErrorEntry)
+        self.assertEqual(result.content, xml)
+        self.assertIn("missing script content", result.error)
+        
+    def test_reasoning_entry(self):
+        """Test parsing reasoning entry"""
+        xml = '<reasoning>test reasoning</reasoning>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, ReasoningEntry)
+        self.assertEqual(result.content, "test reasoning")
+        
+    def test_reasoning_entry_empty_content(self):
+        """Test parsing reasoning entry with empty content returns error entry"""
+        xml = '<reasoning/>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, ParseErrorEntry)
+        self.assertEqual(result.content, xml)
+        self.assertIn("missing content", result.error)
+        
+    def test_read_stdin_entry(self):
+        """Test parsing read stdin entry"""
+        xml = '<read_stdin/>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, ReadEntry)
+        self.assertEqual(result.io_buffer, self.io_buffer)
+        
+    def test_write_stdout_entry(self):
+        """Test parsing write stdout entry"""
+        xml = '<write_stdout>test output</write_stdout>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, WriteEntry)
+        self.assertEqual(result.content, "test output")
+        self.assertEqual(result.io_buffer, self.io_buffer)
+        
+    def test_write_stdout_entry_empty_content(self):
+        """Test parsing write stdout entry with empty content returns error entry"""
+        xml = '<write_stdout/>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, ParseErrorEntry)
+        self.assertEqual(result.content, xml)
+        self.assertIn("missing content", result.error)
+        
+    def test_unknown_element(self):
+        """Test parsing unknown element returns error entry"""
+        xml = '<unknown>test</unknown>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, ParseErrorEntry)
+        self.assertEqual(result.content, xml)
+        self.assertIn("Unknown root element", result.error)
+        
+    def test_malformed_xml(self):
+        """Test parsing malformed XML returns error entry"""
+        xml = '<unclosed>'
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, ParseErrorEntry)
+        self.assertEqual(result.content, xml)
+        self.assertIn("Invalid XML", result.error)
+            
+    def test_entry_id_generation(self):
+        """Test that unique IDs are generated for entries"""
+        xml1 = '<reasoning>test 1</reasoning>'
+        xml2 = '<reasoning>test 2</reasoning>'
+        
+        result1 = self.parser.parse(xml1)
+        result2 = self.parser.parse(xml2)
+        
+        self.assertNotEqual(result1.id, result2.id)
+        
+    def test_whitespace_handling(self):
+        """Test handling of whitespace in XML"""
+        xml = """
+            <reasoning>
+                test content
+                with multiple lines
+            </reasoning>
+        """
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, ReasoningEntry)
+        self.assertTrue(result.content.strip())
+        
+    def test_script_with_special_characters(self):
+        """Test parsing script with special characters"""
+        script = 'echo "Special chars: \n\t\r\'\"\\"'
+        xml = f'<single_shot>{script}</single_shot>'
+        
+        result = self.parser.parse(xml)
+        self.assertIsInstance(result, SingleShotEntry)
+        self.assertEqual(result.script, script)
+        
+    def test_cdata_content(self):
+        """Test parsing content with CDATA sections"""
+        xml = """
+            <reasoning><![CDATA[Test content with <special> & chars]]></reasoning>
+        """
+        result = self.parser.parse(xml)
+        self.assertIsInstance(result, ReasoningEntry)
+        self.assertEqual(result.content, "Test content with <special> & chars")
+        
+    def test_empty_element_with_attributes(self):
+        """Test parsing empty element with attributes"""
+        xml = '<read_stdin id="123" timestamp="2024-01-01"/>'
+        result = self.parser.parse(xml)
+        self.assertIsInstance(result, ReadEntry)
+        
+    def test_error_in_content_extraction(self):
+        """Test handling errors during content extraction"""
+        xml = '<single_shot><![CDATA[test]]>'  # Malformed CDATA
+        result = self.parser.parse(xml)
+        
+        self.assertIsInstance(result, ParseErrorEntry)
+        self.assertEqual(result.content, xml)
+        self.assertIn("Invalid XML", result.error)

test/system_metrics_test.py

@@ -0,0 +1,186 @@
+import unittest
+import time
+import xml.etree.ElementTree as ET
+import psutil
+import multiprocessing
+import math
+from typing import List, Tuple
+
+from sia.system_metrics import SystemMetrics
+
+def cpu_load_process():
+    """Helper process that generates CPU load"""
+    while True:
+        math.factorial(100000)
+
+class SystemMetricsTest(unittest.TestCase):
+    def setUp(self):
+        """Create metrics instance with faster sampling for tests"""
+        self.metrics = SystemMetrics(sample_interval=0.01)
+        
+    def tearDown(self):
+        """Ensure metrics monitoring is stopped"""
+        self.metrics.stop()
+        
+    def validate_timestamp(self, timestamp: str):
+        """Validate timestamp format and reasonableness"""
+        try:
+            # Check format
+            self.assertRegex(
+                timestamp,
+                r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$'
+            )
+            
+            # Check timestamp is recent (within last minute)
+            time_parts = timestamp[:-1].split('T')  # Remove Z
+            time_str = f"{time_parts[0]} {time_parts[1]}"
+            timestamp_time = time.strptime(time_str, "%Y-%m-%d %H:%M:%S")
+            now = time.gmtime()
+            
+            # Convert to seconds since epoch for comparison
+            timestamp_secs = time.mktime(timestamp_time)
+            now_secs = time.mktime(now)
+            
+            self.assertLess(abs(now_secs - timestamp_secs), 60)
+            
+        except Exception as e:
+            self.fail(f"Invalid timestamp format: {timestamp}")
+            
+    def validate_memory_metrics(self, used: int, total: int):
+        """Validate memory metrics are reasonable"""
+        # Check types
+        self.assertIsInstance(used, int)
+        self.assertIsInstance(total, int)
+        
+        # Used memory should be positive and less than total
+        self.assertGreater(used, 0)
+        self.assertGreater(total, 0)
+        self.assertLessEqual(used, total)
+        
+        # Compare with actual system memory
+        memory = psutil.virtual_memory()
+        self.assertEqual(total, memory.total)
+        # Used memory should be within 20% of current usage
+        # (allowing for normal fluctuation)
+        self.assertLess(abs(used - memory.used) / memory.total, 0.2)
+        
+    def validate_disk_metrics(self, used: int, total: int):
+        """Validate disk metrics are reasonable"""
+        # Check types
+        self.assertIsInstance(used, int)
+        self.assertIsInstance(total, int)
+        
+        # Used space should be positive and less than total
+        self.assertGreater(used, 0)
+        self.assertGreater(total, 0)
+        self.assertLessEqual(used, total)
+        
+        # Compare with actual disk usage
+        disk = psutil.disk_usage('/')
+        self.assertEqual(total, disk.total)
+        # Used space should match within 1% of actual usage
+        self.assertLess(abs(used - disk.used) / disk.total, 0.01)
+        
+    def validate_usage_values(self, values: List[Tuple[str, int]]):
+        """Validate usage percentage values are in valid range"""
+        for name, value in values:
+            self.assertIsInstance(value, int)
+            self.assertGreaterEqual(
+                value, 0,
+                f"{name} below valid range: {value}"
+            )
+            self.assertLessEqual(
+                value, 100,
+                f"{name} above valid range: {value}"
+            )
+            
+    def test_initial_context(self):
+        """Test initial context generation"""
+        context = self.metrics.generate_context(0.5)
+        
+        # Verify it's an XML element
+        self.assertIsInstance(context, ET.Element)
+        self.assertEqual(context.tag, "context")
+        
+        # Validate timestamp
+        self.validate_timestamp(context.get("time"))
+        
+        # Validate memory metrics
+        self.validate_memory_metrics(
+            int(context.get("memory_used")),
+            int(context.get("memory_total"))
+        )
+        
+        # Validate disk metrics  
+        self.validate_disk_metrics(
+            int(context.get("disk_used")),
+            int(context.get("disk_total"))
+        )
+        
+        # Validate usage percentages
+        self.validate_usage_values([
+            ("CPU", int(context.get("cpu"))),
+            ("GPU", int(context.get("gpu"))),
+            ("Context", int(context.get("context")))
+        ])
+        
+        # Validate stdin buffer size
+        self.assertEqual(context.get("stdin"), "0")
+        
+    def test_multiple_context_generations(self):
+        """Test multiple context generations have different timestamps"""
+        context1 = self.metrics.generate_context(0.5)
+        time.sleep(1)
+        context2 = self.metrics.generate_context(0.5)
+        
+        self.assertNotEqual(
+            context1.get("time"),
+            context2.get("time")
+        )
+        
+    def test_cpu_usage_detection(self):
+        """Test CPU usage increases under load"""
+        # Get initial CPU usage
+        context1 = self.metrics.generate_context(0.5)
+        initial_cpu = int(context1.get("cpu"))
+        
+        # Generate CPU load in separate process
+        process = multiprocessing.Process(target=cpu_load_process)
+        process.start()
+        
+        try:
+            # Wait for samples to accumulate
+            time.sleep(0.5)
+            
+            # Get CPU usage under load
+            context2 = self.metrics.generate_context(0.5)
+            load_cpu = int(context2.get("cpu"))
+            
+            # CPU usage should increase
+            self.assertGreater(load_cpu, initial_cpu)
+            
+        finally:
+            process.terminate()
+            process.join()
+            
+    def test_context_usage_reflection(self):
+        """Test context usage parameter is reflected accurately"""
+        test_values = [0.0, 0.42, 0.8, 1.0]
+        
+        for value in test_values:
+            context = self.metrics.generate_context(value)
+            self.assertEqual(
+                int(context.get("context")),
+                round(value * 100)
+            )
+            
+    def test_cleanup(self):
+        """Test monitoring stops properly"""
+        self.metrics.stop()
+        
+        # Monitor thread should finish
+        self.assertFalse(self.metrics._monitor_thread.is_alive())
+        
+        # Should still generate valid context after stopping
+        context = self.metrics.generate_context(0.5)
+        self.assertIsInstance(context, ET.Element)