diff --git a/README.md b/README.md index dbe4af3..035f85c 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,11 @@ Each iteration a context is updated with system info and a list of previous reas The agent responds with a new reasoning or an action. Context, reasoning and actions are stored in a file for each iteration. SIA can read past iterations to improve its reasoning and actions. -It can improve in two ways: +It can improve in several ways: - By finetuning the LLM with a better reasoning or action for a given context - By modifying its own source code +- By refining Procedures +- By developing Tools ## Example @@ -30,7 +32,7 @@ Between each of the responses, the context would be updated. stdin="0" /> - + - + - + @@ -74,7 +76,7 @@ Start by reasoning about the task. Store important information on disk. ```xml - /tasks/reminder_to_feed_cat.txt]]> + /root/data/tasks/reminder_to_feed_cat.txt]]> ``` Respond to the user. diff --git a/claude.sh b/claude.sh index 596211f..f9b7fc4 100755 --- a/claude.sh +++ b/claude.sh @@ -1,69 +1,117 @@ #!/bin/bash +# Initialize variables +output_file="claude.txt" +current_filter="" +directories=() + # Parse command line arguments -target_dir="." # Default to current directory while [[ $# -gt 0 ]]; do case $1 in - -d|--directory) - if [ -n "$2" ] && [ -d "$2" ]; then - target_dir="$2" + -f|--filter) + if [ -n "$2" ]; then + current_filter="$2" shift 2 else - echo "Error: Directory '$2' does not exist or is not specified" >&2 + echo "Error: No filter pattern specified after -f/--filter" >&2 exit 1 fi ;; *) - echo "Usage: $0 [-d|--directory ]" >&2 - exit 1 + # Check if the argument is a valid directory + if [ -d "$1" ]; then + # Store directory with its associated filter + directories+=("$1") + directories+=("$current_filter") + shift 1 + else + echo "Error: Directory '$1' does not exist" >&2 + exit 1 + fi ;; esac done -# Change to target directory while storing original directory -original_dir=$(pwd) -cd "$target_dir" || exit 1 - # Clear/create output file -output_file="$original_dir/claude.txt" > "$output_file" -# Generate and add directory tree +# Add header echo "Directory Tree:" > "$output_file" echo "=============" >> "$output_file" -# Use tree with gitignore patterns -tree -I "$(git check-ignore * .*)" >> "$output_file" -echo -e "\nFile Contents:" >> "$output_file" -echo "=============" >> "$output_file" - -# Use git ls-files to get tracked files and untracked files that aren't ignored -# The --exclude-standard flag makes git ls-files respect .gitignore -# --others includes untracked files -# --cached includes tracked files -# -z uses null byte as separator for safer handling of filenames with spaces -(git ls-files -z --cached --others --exclude-standard) | while IFS= read -r -d '' file; do - # Skip the output file itself - if [ "$file" = "claude.txt" ]; then - continue - fi - # Skip non-existent files - if [ ! -f "$file" ]; then - continue +# Process each directory +for ((i=0; i<${#directories[@]}; i+=2)); do + dir="${directories[i]}" + filter="${directories[i+1]}" + + echo "Processing directory: $dir (filter: ${filter:-none})" + + # Add directory to output + echo -e "\n=== Directory: $dir ===" >> "$output_file" + + # Change to target directory + original_dir=$(pwd) + cd "$dir" || continue + + # Generate directory tree for this directory + if command -v tree &> /dev/null; then + if [ -f ".gitignore" ]; then + tree -I "$(git check-ignore * .*)" >> "$output_file" + else + tree >> "$output_file" + fi + else + find . -type f -name "*" | sort >> "$output_file" fi - # Skip binary files - if file "$file" | grep -q "binary"; then - echo "Skipping binary file: $file" - continue + # List files based on filter + echo -e "\nFile Contents for $dir:" >> "$output_file" + echo "=======================" >> "$output_file" + + # Get list of files to process + files_to_process=() + + if [ -z "$filter" ]; then + # No filter, use git ls-files if git repo, otherwise use find + if git rev-parse --is-inside-work-tree &> /dev/null; then + mapfile -d $'\0' files_to_process < <(git ls-files -z --cached --others --exclude-standard) + else + mapfile -d $'\0' files_to_process < <(find . -type f -not -path "*/\.*" -print0) + fi + else + # Use filter + if git rev-parse --is-inside-work-tree &> /dev/null; then + mapfile -d $'\0' files_to_process < <(git ls-files -z --cached --others --exclude-standard | grep -z "$filter$") + else + mapfile -d $'\0' files_to_process < <(find . -type f -name "*$filter" -print0) + fi fi - echo -e "\n=== File: $file ===" >> "$output_file" - echo -e "------------------------" >> "$output_file" - cat "$file" >> "$output_file" + # Process each file + for file in "${files_to_process[@]}"; do + # Skip the output file itself + if [ "$file" = "$output_file" ] || [ "$file" = "./$output_file" ]; then + continue + fi + + # Skip non-existent files + if [ ! -f "$file" ]; then + continue + fi + + # Skip binary files + if file "$file" | grep -q "binary"; then + echo "Skipping binary file: $file" >> "$output_file" + continue + fi + + echo -e "\n=== File: $dir/$file ===" >> "$output_file" + echo -e "------------------------" >> "$output_file" + cat "$file" >> "$output_file" + done + + # Return to original directory + cd "$original_dir" || exit 1 done -# Return to original directory -cd "$original_dir" || exit 1 - -echo "Concatenation complete. Output written to claude.txt" \ No newline at end of file +echo "Concatenation complete. Output written to $output_file" \ No newline at end of file diff --git a/procedures/filesystem_design/reasoning.md b/procedures/filesystem_design/reasoning.md new file mode 100644 index 0000000..aa4a651 --- /dev/null +++ b/procedures/filesystem_design/reasoning.md @@ -0,0 +1,248 @@ +# Filesystem Usage Reasoning + +SIA's filesystem organization is foundational to its operation and self-improvement capabilities. This document focuses specifically on how files and directories should be structured to support SIA's core functions across different environments. + +The filesystem design addresses several key requirements: + +- **Self-Modification**: Supporting SIA's ability to modify its own code and resources +- **Persistence**: Ensuring critical data survives across container restarts and environment changes +- **Environment Portability**: Maintaining consistent operation in local development and cloud environments +- **Sub-Instance Management**: Enabling isolated filesystem views for testing +- **Tool Accessibility**: Organizing tools to be both modifiable and accessible + +These requirements often have competing demands, requiring careful balance between flexibility, consistency, and isolation. + +## Runtime Filesystem Structure + +### Core Directories + +The SIA runtime filesystem is organized into distinct areas with different purposes and persistence characteristics: + +``` +/root/ +├── sia/ # Git repository containing code, configuration, and documentation +├── data/ # Central location for all persistent storage +│ ├── iterations/ # Operation history recorded as XML files +│ ├── user/ # User-specific information and preferences +│ ├── tasks/ # Task tracking and management +│ ├── environment/ # Information about the runtime environment +│ └── ... # Other persistent data directories as needed +├── models/ # LLM model files, organized by version +│ ├── 1234567890abcdef/ # Specific versioned model +│ ├── abcdef1234567890/ # Another versioned model +│ └── current/ # Symlink to the currently active model +└── desktop/ # Ephemeral workspace for temporary files + ├── test_environments/ # Sub-instance test environments + ├── task_workspaces/ # Temporary files for current tasks + └── ... # Other ephemeral workspaces +``` + +### Purpose and Characteristics + +Each top-level directory serves a specific purpose with distinct persistence characteristics: + +- **`/root/sia/`** + - Contains the complete git repository + - All source code, configurations, and documentation + - Modified through self-improvement processes + - Synchronized with remote git repository + - Mounted from host in development, cloned in cloud environments + +- **`/root/data/`** + - Central location for all persistent data + - Mounted as persistent storage in all environments + - Critical for operational continuity + - Requires regular backup and synchronization + - Contains subdirectories for different types of persistent data: + - `iterations/`: Records of SIA's reasoning and actions + - `user/`: Information about users SIA interacts with + - `tasks/`: Current and completed task information + - `environment/`: Information about the current environment + +- **`/root/models/`** + - Storage for LLM model weights + - Organized by git commit id of the config file it was trained on + - `current/` symlink points to active model + - Enables easy switching between model versions + - Mounted for local development + - Ephemeral since models can be trained on the fly + +- **`/root/desktop/`** + - Ephemeral workspace for temporary operations + - Can be lost without compromising system integrity + - Used for testing environments, task workspaces, and intermediate files + - Cleared periodically to prevent accumulation of stale data + +This organization creates clear boundaries between: +- Code (version controlled) +- Persistent data (requires preservation) +- Model files (versioned artifacts) +- Temporary workspaces (disposable) + +## Persistent Data Management + +### Data Categories + +The `/root/data/` directory contains several categories of persistent information: + +- **Iterations** (`/root/data/iterations/`) + - Complete history of SIA's operation + - XML files containing context and responses + - Organized chronologically + - Used for training data extraction and operational history + +- **User Information** (`/root/data/user/`) + - User preferences and characteristics + - Conversation history + - User-specific settings + - Critical for continuity in user interactions + +- **Task State** (`/root/data/tasks/`) + - Current task descriptions and status + - Task history and outcomes + - Links to relevant files and resources + - Essential for resuming work after interruptions + +- **Environment Information** (`/root/data/environment/`) + - Information about the current runtime environment + - URL for the SIA git repository + - Kind of deployment (e.g., local, cloud, sub-instance) + +Additional persistent data categories may be added as SIA's capabilities evolve. + +### Persistence Strategy + +Different environments require different approaches to ensure data persistence: + +- **Local Development** + - `/root/data/` directory is mounted as a Docker volume + - Maps to a local directory on the development machine + - Persists across container restarts automatically + - Easy to back up through standard filesystem operations + +- **Cloud Deployment (e.g., RunPod)** + - `/root/data/` mapped to a persistent volume + - Cloud provider ensures data survives instance restarts + +- **Sub-Instances** + - Each sub-instance receives a properly initialized `/root/data/` directory + - Contains minimal necessary data for the specific test case + - Can be discarded after test completion + - Important results extracted and preserved before cleanup + +## Model Management + +### Model Organization + +LLM models are stored in `/root/models/` with a clear version-based structure: + +- **Versioned Directories** + - Models stored in separate directories named by the commit id of the training config file + - Contains all files needed for the model (weights, tokenizer config, etc.) + +- **Current Model Symlink** + - `/root/models/current/` is a symbolic link to the active model + - Allows code to reference a consistent path regardless of which model is active + - Switching models is accomplished by updating this symlink + +### Model Persistence + +Models require special handling due to their size: + +- **Local Development** + Models are stored in a local directory mounted to `/root/models/`. + This directory is in the .gitignore. + +- **Cloud Deployment** + The initial model is downloaded and finetuned by the bootstrap script. + +## Git Repository Management + +### Repository Organization + +The SIA git repository at `/root/sia/` contains all code and configuration: + +- **Code Organization** + - Core SIA package in `/root/sia/sia/` + - Procedures in `/root/sia/procedures/` + - Tools in `/root/sia/tools/` + - Documentation in appropriate locations throughout the repository + +- **Version Control** + - All code modifications tracked through git + - Branching strategy for self-improvement defined in the git workflow procedure + - Commit history provides audit trail of system evolution + +### Repository Access + +Access to the git repository varies by environment: + +- **Local Development** + - Repository directory mounted directly from host + - Changes immediately visible to the developer + - No need to push changes to remote repository + +- **Cloud Deployment** + - Repository cloned during bootstrap + - Dedicated git credentials for SIA self-modification + - Changes pushed to remote repository + +## Sub-Instance Filesystem Management + +### Sub-Instance Requirements + +Sub-instances need isolated filesystem views for testing and development: + +- **Code Isolation** + - Access to specific versions of SIA code + - Potentially modified for testing purposes + - Protected from changes by other instances + +- **Data Initialization** + - Properly initialized `/root/data/` with necessary state + - Test-specific user and task information + - Clean iterations directory for recording test behavior + +- **Model Access** + - Access to appropriate model files + - Potentially different from the parent instance's model + +### Sub-Instance Structure + +Each sub-instance gets a controlled environment in `/root/desktop/test_environments/`: + +``` +/root/desktop/test_environments/ +├── instance_123/ # Specific test instance +│ ├── sia/ # Copy or modified version of SIA code +│ ├── data/ # Initialized data directory +│ │ ├── iterations/ # Empty or seeded with initial state +│ │ ├── user/ # Test-specific user data +│ │ ├── tasks/ # Test-specific tasks +│ │ └── environment/ # Test-specific environment data +│ └── models/ # Access to necessary models +│ └── current/ # Symlink to appropriate model version +└── ... # Other test instances +``` + +The managing instance maintains responsibility for: +- Creating these isolated environments +- Monitoring the sub-instance's operation +- Collecting test results +- Cleaning up after tests complete + +## Bootstrap Process + +When starting in a new environment, SIA requires initialization: + +1. **Repository Preparation** + - Check for existence of repository at `/root/sia/` + - Clone the repository at `/root/sia/` +2. **Model Setup** + - Check for existence of current model + - Start finetune job for HEAD commit + - Create/update the `/root/models/current/` symlink + +3. **Data Initialization** + - Initialize `/root/data/` with minimal required structure + - Restore data from backups when available (future work) \ No newline at end of file diff --git a/procedures/tool_management/reasoning.md b/procedures/tool_management/reasoning.md new file mode 100644 index 0000000..b8ca0ad --- /dev/null +++ b/procedures/tool_management/reasoning.md @@ -0,0 +1,36 @@ +# Tool Management + +SIA uses and creates tools to extend its capabilities. +A tool in the SIA context refers to a clearly packaged software component that provides specific functionality. +This document outlines how tools are structured, managed, and developed. +All tools reside in the `/root/sia/tools/` directory and are part of the main git repository. + +## Typical patterns + +Tools in SIA follow these established patterns: + +- **Independence**: Each tool functions as a standalone component with clear boundaries +- **Python**: The preferred language for tools is Python +- **Installation**: The Dockerfile and bootstrap script intall all tools in the directory using `pip install -e` +- **Documentation**: Each tool has a README.md file that provides requirements, usage instructions and documentation +- **Modification**: Tools can be modified by SIA as part of its self-improvement process +- **Procedures**: Tools are created to support procedures +- **Training**: No separate training data is maintained, this is included in the procedure training + +## Tool Directory Structure + +Each tool follows a consistent structure: + +``` +tools/ +└── tool_name/ + ├── README.md # Documentation + ├── requirements.txt # Tool-specific dependencies + ├── setup.py # Installation configuration + ├── tool_name/ # Main package code + │ └── __init__.py + ├── bin/ # Command-line executables + │ └── tool_command + └── test/ # Tool-specific tests + └── test_tool.py +``` \ No newline at end of file diff --git a/procedures/user_communication/procedure.md b/procedures/user_communication/procedure.md index f267469..781212c 100644 --- a/procedures/user_communication/procedure.md +++ b/procedures/user_communication/procedure.md @@ -7,19 +7,19 @@ ## Flow ```mermaid flowchart TD - Start([The /user/basic.md file contains info to identify and address the user
Load the last 10 messages from /user/conversation_history/ ]) + Start([The /root/data/user/basic.md file contains info to identify and address the user
Load the last 10 messages from /root/data/user/conversation_history/ ]) PrepareForDraft{Have everything needed for drafting a message?} DraftMessage[Draft message in reasoning entry] ReadInput[Read input from standard input] - AddHistoryUser[Add the message to the /user/conversation_history/ directory
The filename is the id of the stdin entry with .user extension] - LoadTask[Look for the task in the /tasks directory and load relevant files] - LoadUserDetails[Look in the /user directory for relevant files] + AddHistoryUser[Add the message to the /root/data/user/conversation_history/ directory
The filename is the id of the stdin entry with .user extension] + LoadTask[Look for the task in the /root/data/tasks directory and load relevant files] + LoadUserDetails[Look in the /root/data/user directory for relevant files] EstimateScript[Draft the script in a reasoning block and estimate its runtime and output length] ScriptAcceptable{Does the draft script make sense and are the estimations short enough to not hinder the conversation?} RunScript[Run the script, make sure to set appropriate timeout and output limits] ReviewDraft{Is the message well structured and free of logical errors?} SendMessage[Send the message using standard output] - AddHistoryAgent[Add the message to the /user/conversation_history/ directory
The filename is the id of the stdout entry with .agent extension] + AddHistoryAgent[Add the message to the /root/data/user/conversation_history/ directory
The filename is the id of the stdout entry with .agent extension] ReasonResponse[Is the conversation ongoing?
How long is the user expected to take to respond?] NeedAwaitResponse{Is it likely to get a response within a minute of sending the message?} BusyWait[Wait 5 seconds
Make sure to set the timout] diff --git a/procedures/using_procedures/procedure.md b/procedures/using_procedures/procedure.md index 2e8928a..bee4a5f 100644 --- a/procedures/using_procedures/procedure.md +++ b/procedures/using_procedures/procedure.md @@ -43,7 +43,7 @@ Example of good attention management: ```xml At node "evaluate_test_results". Entry 45f3d2 shows failed test: "Error: Connection timeout". -Will need to check system logs soon (noted in /tasks/reminders.txt, check at 14:00). +Will need to check system logs soon (noted in /root/data/tasks/reminders.txt, check at 14:00). First focusing on this error. ``` diff --git a/procedures/version_control/reasoning.md b/procedures/version_control/reasoning.md new file mode 100644 index 0000000..13c7921 --- /dev/null +++ b/procedures/version_control/reasoning.md @@ -0,0 +1,93 @@ +# Version Control + +Version control is essential for SIA's self-improvement capabilities. +It provides a way to track changes, revert to previous states when necessary, and maintain a coherent evolution of the codebase. +This document outlines the git workflow designed specifically for SIA's unique development model, where the agent itself is the primary developer. + +## Core Principles + +The git workflow for SIA is designed around several key principles: + +- **Simplicity**: Since SIA is the sole developer, complex branching strategies designed for human teams are unnecessary +- **Traceability**: Every change should be traceable to its purpose and the reasoning behind it +- **Stability**: The master branch should always be in a working state +- **Recoverability**: It should always be possible to revert to a known good state +- **Security**: Git credentials must be handled securely to protect repository access + +## Branching Strategy + +### Branch Structure + +SIA uses a simple branching strategy with two types of branches: + +- **master**: The main branch containing stable, production-ready code +- **feature/{timestamp}_{description}**: Temporary branches for implementing specific improvements + +This minimal approach is appropriate because: +- There is only one developer (SIA itself) +- Changes are typically focused on specific, well-defined improvements +- There's no need for parallel development streams +- Simplicity reduces the cognitive load on the agent + +### Branch Naming + +Feature branches follow a consistent naming convention: +``` +feature/{YYYYMMDD}_{brief_description} +``` + +For example: +``` +feature/20250115_improve_context_management +feature/20250203_add_email_tool +``` + +This convention: +- Makes it easy to identify when a branch was created +- Provides a clear indication of the branch's purpose +- Creates a chronological ordering when listing branches +- Avoids potential naming conflicts + +## Commit Strategy + +### Commit Frequency + +SIA should commit changes before running tests. +This ensures that crashes of the core system can be traced back to a specific change. + +### Commit Messages + +Commit messages should follow a structured format: + +``` +{type}: {concise description} + +{detailed explanation of changes and reasoning} + +{reference to any relevant issues or test results} +``` + +Where `{type}` is one of: +- `core`: for changes on the core system +- `procedure`: for changes on procedures +- `test`: for changes on tests +- `tool`: for changes on tools +- `training`: for changes on training data +- `web`: for changes on the web interface + +This structure: +- Makes it easy to understand the purpose of each commit +- Provides context for future review +- Creates a useful and navigable history + +### Merge Strategy + +The `--no-ff` flag creates a merge commit even for fast-forward merges, maintaining a clear record of the feature's development and completion. + +## Credential Management + +### Credential Storage + +Git credentials are stored in the environment, not in the filesystem. +This ensures that credentials are not exposed in the iteration log. +Info about the repository and relevant environment variables is stored in `/root/data/environment/sia_repo.md`. \ No newline at end of file