wip deepseek r1
This commit is contained in:
125
scripts/bootstrap.sh
Normal file
125
scripts/bootstrap.sh
Normal file
@@ -0,0 +1,125 @@
|
||||
#!/bin/bash
|
||||
# bootstrap.sh - Initialize SIA (Self-Improving Agent) environment for cloud deployment
|
||||
|
||||
set -eo pipefail # Exit on any error, pipe failures
|
||||
|
||||
# Hardcoded paths for cloud deployment
|
||||
SIA_REPO_URL="ssh://git@git.nielsgeens.be:222/llm/SIA.git"
|
||||
SIA_DIR="/root/sia"
|
||||
DATA_DIR="/root/data"
|
||||
MODELS_DIR="/root/models"
|
||||
DESKTOP_DIR="/root/desktop"
|
||||
STATIC_DIR="/root/static"
|
||||
VENVS_DIR="/root/venvs"
|
||||
|
||||
# Print header
|
||||
echo "==================================================="
|
||||
echo "SIA Bootstrap Script - Cloud Deployment"
|
||||
echo "==================================================="
|
||||
|
||||
# Create directory structure
|
||||
echo "Creating directory structure..."
|
||||
mkdir -p "$DATA_DIR/iterations"
|
||||
mkdir -p "$DESKTOP_DIR"
|
||||
mkdir -p "$VENVS_DIR"
|
||||
cd "$DESKTOP_DIR"
|
||||
|
||||
# Set up SSH keys
|
||||
echo "Setting up SSH keys for git access..."
|
||||
mkdir -p ~/.ssh
|
||||
chmod 700 ~/.ssh
|
||||
ssh-keygen -t sia_git -N "" -f ~/.ssh/sia_git -C "sia-agent"
|
||||
echo "New SSH key generated"
|
||||
|
||||
# Display public key for user to add to git server
|
||||
echo "==================================================="
|
||||
echo "Add this public key to your git server:"
|
||||
cat ~/.ssh/sia_git.pub
|
||||
echo "==================================================="
|
||||
|
||||
# Prompt user to confirm they've added the key
|
||||
read -p "Press Enter once you've added the SSH key to the git server..."
|
||||
|
||||
# Clone SIA repository
|
||||
echo "Cloning SIA repository..."
|
||||
git clone "$SIA_REPO_URL" "$SIA_DIR"
|
||||
|
||||
# Create and setup virtual environments
|
||||
echo "Setting up SIA virtual environments..."
|
||||
|
||||
# Setup ITB tool environment
|
||||
echo "Creating ITB tool environment..."
|
||||
python3 -m venv "$VENVS_DIR/itb"
|
||||
"$VENVS_DIR/itb/bin/pip" install -e "$SIA_DIR/tools/itb"
|
||||
|
||||
# Setup Train tool environment
|
||||
echo "Creating Train tool environment..."
|
||||
python3 -m venv "$VENVS_DIR/train"
|
||||
"$VENVS_DIR/train/bin/pip" install -e "$SIA_DIR/tools/train"
|
||||
|
||||
# Setup SIA core environment
|
||||
echo "Creating SIA core environment..."
|
||||
python3 -m venv "$VENVS_DIR/sia"
|
||||
"$VENVS_DIR/sia/bin/pip" install -e "$SIA_DIR"
|
||||
|
||||
# Build web interface
|
||||
echo "Building web interface"
|
||||
cd "$SIA_DIR/web"
|
||||
|
||||
# Install Node.js if needed
|
||||
if ! command -v node &> /dev/null; then
|
||||
echo "Installing Node.js..."
|
||||
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
|
||||
export NVM_DIR="$HOME/.nvm"
|
||||
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
|
||||
nvm install node
|
||||
fi
|
||||
|
||||
npm install
|
||||
npm run build
|
||||
|
||||
mkdir -p "$STATIC_DIR"
|
||||
cp -r "$SIA_DIR/web/dist/"* "$STATIC_DIR/"
|
||||
|
||||
echo "Web interface built successfully"
|
||||
|
||||
# Finetune model
|
||||
echo "Starting model finetuning..."
|
||||
|
||||
COMMIT_ID=$(cd "$SIA_DIR" && git rev-parse HEAD)
|
||||
echo "Current commit: $COMMIT_ID"
|
||||
|
||||
mkdir -p "$MODELS_DIR/$COMMIT_ID"
|
||||
mkdir -p "$MODELS_DIR/current"
|
||||
|
||||
# Run finetuning using the train environment
|
||||
"$VENVS_DIR/train/bin/train_deepseek" --output-dir "$MODELS_DIR/$COMMIT_ID"
|
||||
|
||||
ln -sf "$MODELS_DIR/$COMMIT_ID" "$MODELS_DIR/current"
|
||||
echo "Finetuning complete, model linked to current"
|
||||
|
||||
# Initialize environment information
|
||||
echo "Initializing environment information..."
|
||||
mkdir -p "$DATA_DIR/environment"
|
||||
cat > "$DATA_DIR/environment/sia_repo.md" << EOF
|
||||
# SIA Repository Information
|
||||
|
||||
- Repository URL: $SIA_REPO_URL
|
||||
- ssh key: ~/.ssh/sia_git.pub
|
||||
EOF
|
||||
|
||||
# Create .env file for local model only
|
||||
cat > "$SIA_DIR/.env" << EOF
|
||||
SIA_DEEPSEEK_ENABLED=true
|
||||
SIA_DEEPSEEK_MODEL=$MODELS_DIR/current
|
||||
SIA_DEEPSEEK_TEMPERATURE=0.6
|
||||
EOF
|
||||
|
||||
# Print header
|
||||
echo "==================================================="
|
||||
echo "SIA environment initialization complete!"
|
||||
echo "==================================================="
|
||||
|
||||
# Start SIA using restart script
|
||||
echo "Starting SIA..."
|
||||
"$SIA_DIR/scripts/restart.sh"
|
||||
@@ -6,7 +6,7 @@ declare -A FILTER_SETS=(
|
||||
["py"]="-f .*(\\.py|requirements.txt)$"
|
||||
["web"]="-f .*\\.(js|jsx|json|css|html)$"
|
||||
["doc"]="-f .*\\.md$"
|
||||
["deploy"]="-f .*(Dockerfile|\\.sh|\\.xsd)$"
|
||||
["deploy"]="-f .*(Dockerfile|\\.sh|\\.xsd|\\.yaml)$"
|
||||
|
||||
["core"]="-s py ./sia ./tools -s deploy . -f ^(?!procedures/).*\\.md$ ."
|
||||
["webui"]="-s web ./web"
|
||||
@@ -291,4 +291,4 @@ main() {
|
||||
echo "Concatenation complete. Output written to $OUTPUT" >&2
|
||||
}
|
||||
|
||||
main "$@"
|
||||
main "$@"
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
cd /
|
||||
git clone https://git.nielsgeens.be/llm/SIA.git
|
||||
cd /SIA
|
||||
pip3 install -r requirements.txt
|
||||
|
||||
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
|
||||
nvm install node
|
||||
cd /SIA/web
|
||||
npm install
|
||||
npm install -D tailwindcss
|
||||
npm run build
|
||||
mv /root/SIA/web/dist/ /root/SIA/static
|
||||
|
||||
apt update
|
||||
apt install -y vim tmux
|
||||
vim .env
|
||||
|
||||
cd /root/SIA
|
||||
python3 -m sia
|
||||
|
||||
#The SIA source is located in /root/sia. Not all features are implemented yet. Look at the readme and code to find what is missing. Make sure to unit test your work.
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
while true; do
|
||||
PYTHONPATH="/root/sia:$PYTHONPATH" python3 -m sia
|
||||
sia
|
||||
if [ $? -eq 42 ]; then
|
||||
echo "SIA exited with code 42. Restarting."
|
||||
else
|
||||
|
||||
64
scripts/setup_binaries.py
Normal file
64
scripts/setup_binaries.py
Normal file
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Script to extract binary names from setup.py files and create placeholder files.
|
||||
Usage: setup_binaries.py /path/to/setup.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
|
||||
def extract_setup_binaries(setup_path):
|
||||
"""Extract binary names from a setup.py file and create placeholder files."""
|
||||
try:
|
||||
# Read the setup.py file
|
||||
with open(setup_path, 'r') as f:
|
||||
setup_content = f.read()
|
||||
|
||||
# Find all references to scripts in bin/ directory
|
||||
scripts = re.findall(r"'bin/[^']+?'|\"bin/[^\"]+?\"", setup_content)
|
||||
if not scripts:
|
||||
print(f"No bin scripts found in {setup_path}")
|
||||
return True # Not an error, just no scripts
|
||||
|
||||
# Clean up the extracted script names
|
||||
scripts = [script.strip('\'"') for script in scripts]
|
||||
|
||||
# Create placeholder files
|
||||
base_dir = os.path.dirname(setup_path)
|
||||
created_count = 0
|
||||
|
||||
for script in scripts:
|
||||
script_path = os.path.join(base_dir, script)
|
||||
script_dir = os.path.dirname(script_path)
|
||||
|
||||
# Create directory if it doesn't exist
|
||||
os.makedirs(script_dir, exist_ok=True)
|
||||
|
||||
# Create an empty file
|
||||
with open(script_path, 'w') as f:
|
||||
pass
|
||||
|
||||
# Make the file executable
|
||||
os.chmod(script_path, 0o755)
|
||||
created_count += 1
|
||||
|
||||
print(f"Created {created_count} placeholder binary files from {setup_path}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error processing {setup_path}: {e}")
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
print(f"Usage: {sys.argv[0]} /path/to/setup.py")
|
||||
sys.exit(1)
|
||||
|
||||
setup_path = sys.argv[1]
|
||||
if not os.path.exists(setup_path):
|
||||
print(f"Error: {setup_path} not found")
|
||||
sys.exit(1)
|
||||
|
||||
success = extract_setup_binaries(setup_path)
|
||||
sys.exit(0 if success else 1)
|
||||
@@ -1,12 +1,20 @@
|
||||
#!/bin/bash
|
||||
|
||||
docker build \
|
||||
--target sia-test \
|
||||
--tag sia-test \
|
||||
--tag sia \
|
||||
.
|
||||
|
||||
# Run tests within the SIA virtual environment
|
||||
docker run \
|
||||
--rm \
|
||||
-ti \
|
||||
--gpus=all \
|
||||
-v /$(pwd)/model/:/root/model/ \
|
||||
sia-test
|
||||
-p 8080:8080 \
|
||||
--env-file .env \
|
||||
-v /$(pwd)/model/:/root/models/current/ \
|
||||
-v /$(pwd)/iterations/:/root/data/iterations/ \
|
||||
-v /$(pwd)/tasks/:/root/data/tasks/ \
|
||||
-v /$(pwd)/user/:/root/data/user/ \
|
||||
-v /$(pwd)/environment/:/root/data/environment/ \
|
||||
-v /$(pwd)/:/root/sia/ \
|
||||
sia /root/venvs/sia/bin/python -m unittest discover -v -p "*test.py"
|
||||
Reference in New Issue
Block a user