llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Retry deploy if not available

Browse Source
This commit is contained in:
Niels Geens
2025-04-23 15:16:41 +02:00
parent bf94a0838a
commit c834141ead
3 changed files with 159 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
scripts/bootstrap.sh
View File

@@ -21,19 +21,35 @@ fi
# Install required packages
apt-get update
apt-get install -y \
build-essential \
git \
gnupg \
python3-dev \
python3-venv \
vim \
wget \
;
# Install Chrome
wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list
apt-get update && \
apt-get install -y \
google-chrome-stable
# Install Rust
curl https://sh.rustup.rs -sSf | bash -s -- -y
export PATH="/root/.cargo/bin:${PATH}"
# Install Node.js
if ! command -v node &> /dev/null; then
echo "Installing Node.js..."
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
export NVM_DIR="/root/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
nvm install node
fi
# Create directory structure
echo "Creating directory structure..."
mkdir -p "/root/data/iterations"
@@ -52,15 +68,6 @@ if [ ! -d "/root/sia" ]; then
git config --global core.editor vim
fi
# Install Node.js if needed
if ! command -v node &> /dev/null; then
echo "Installing Node.js..."
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
export NVM_DIR="/root/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
nvm install node
fi
# Build web interface
echo "Building web interface"
cd "/root/sia/web"

197
scripts/deploy.sh
View File

@@ -1,25 +1,23 @@
#!/bin/bash
# Script for setting up and running SIA on RunPod
# Script for setting up and running SIA on RunPod with retry logic
# Requires the RunPod CLI (runpodctl) to be installed and configured
# Export MSYS_NO_PATHCONV=1 for Git Bash compatibility
export MSYS_NO_PATHCONV=1
# =================== Configuration ===================
# These can be overridden by environment variables or .env file
# Pod configuration
GPU_TYPE=${GPU_TYPE:-"NVIDIA RTX A6000"}
GPU_COUNT=${GPU_COUNT:-1}
CONTAINER_DISK_SIZE=${CONTAINER_DISK_SIZE:-200} # GB
CPU_COUNT=${CPU_COUNT:-1} # vCPUs
POD_NAME=${POD_NAME:-"sia-agent"}
VOLUME_SIZE=${VOLUME_SIZE:-50} # GB
VOLUME_PATH=${VOLUME_PATH:-"/root/data"} # Mount path within container
POD_NAME="sia-agent"
CONTAINER_DISK_SIZE=200 # GB
VOLUME_SIZE=50 # GB
VOLUME_PATH="/root/data" # Mount path within container
DOCKER_IMAGE="nvidia/cuda:12.2.0-runtime-ubuntu22.04"
# Docker configuration
DOCKER_IMAGE=${DOCKER_IMAGE:-"runpod/vscode-server:0.0.0"}
GPU_CONFIGS=(
"NVIDIA RTX 6000 Ada Generation"
"NVIDIA RTX A6000"
)
# Load environment variables from .env file if it exists
if [ -f .env ]; then
@@ -27,53 +25,93 @@ if [ -f .env ]; then
source .env
fi
# Create pod
echo "Creating RunPod instance with GPU type: $GPU_TYPE"
output=$(runpodctl create pod \
--name "$POD_NAME" \
--imageName "$DOCKER_IMAGE" \
--gpuType "$GPU_TYPE" \
--gpuCount $GPU_COUNT \
# Function to create pod with given parameters
create_pod() {
local gpu_type=$1
local cloud_arg=$2
echo "Attempting to create pod with:"
echo "- GPU Type: $gpu_type"
echo "- Cloud Option: $cloud_arg"
create_cmd="runpodctl create pod \
--name \"$POD_NAME\" \
--imageName \"$DOCKER_IMAGE\" \
--gpuType \"$gpu_type\" \
--gpuCount 1 \
--containerDiskSize $CONTAINER_DISK_SIZE \
--volumeSize $VOLUME_SIZE \
--volumePath "$VOLUME_PATH" \
--vcpu $CPU_COUNT \
--ports "8080/http,8888/http" \
--env "SIA_REPO_URL=$SIA_REPO_URL" \
--env "SIA_REPO_USER=$SIA_REPO_USER" \
--env "SIA_REPO_PAT=$SIA_REPO_PAT" \
--env "SIA_HF_API_KEY=$SIA_HF_API_KEY" \
--env "SIA_QWQ_ENABLED=1" \
--env "JUPYTER_PASSWORD=1" \
#--communityCloud
)
--volumePath \"$VOLUME_PATH\" \
--vcpu 1 \
--ports \"8080/http,8888/http\" \
--env \"SIA_REPO_URL=$SIA_REPO_URL\" \
--env \"SIA_REPO_USER=$SIA_REPO_USER\" \
--env \"SIA_REPO_PAT=$SIA_REPO_PAT\" \
--env \"SIA_HF_API_KEY=$SIA_HF_API_KEY\" \
--env \"SIA_QWQ_ENABLED=1\" \
--env \"JUPYTER_PASSWORD=1\" \
--args \"/bin/bash -c \\\"apt-get update && \
apt-get install -y curl && \
curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz && \
tar -xf vscode_cli.tar.gz && \
mkdir -p /usr/local/bin && \
mv code /usr/local/bin/ && \
rm vscode_cli.tar.gz && \
yes | code tunnel \
--name runpod \
--install-extension ms-toolsai.jupyter \
--install-extension ms-python.python\\\"\""
# Add cloud argument if specified
if [ -n "$cloud_arg" ]; then
create_cmd="$create_cmd $cloud_arg"
fi
# Execute the command and capture output
output=$(eval $create_cmd 2>&1)
# Return the output
echo "$output"
}
echo "$output"
# Extract the last line of output (should contain pod ID)
last_line=$(echo "$output" | tail -n 1)
# Extract pod ID - look for the word "pod" followed by ID in quotes
if [[ $last_line =~ pod\ \"([a-zA-Z0-9]+)\" ]]; then
# Function to extract pod ID from output
extract_pod_id() {
local output=$1
local pod_id=""
# Extract the last line of output (should contain pod ID)
local last_line=$(echo "$output" | tail -n 1)
# Extract pod ID - look for the word "pod" followed by ID in quotes
if [[ $last_line =~ pod\ \"([a-zA-Z0-9]+)\" ]]; then
pod_id="${BASH_REMATCH[1]}"
else
else
# Try to get the last word which is often the pod ID
pod_id=$(echo "$last_line" | awk '{print $NF}')
fi
fi
# If the ID has a double quote at the end, remove it
pod_id=${pod_id%\"}
# Validate we have a proper pod ID (alphanumeric)
if [[ $pod_id =~ ^[a-zA-Z0-9]+$ ]]; then
echo "$pod_id"
return 0
else
return 1
fi
}
# If the ID has a double quote at the end, remove it
pod_id=${pod_id%\"}
# Validate we have a proper pod ID (alphanumeric)
if ! [[ $pod_id =~ ^[a-zA-Z0-9]+$ ]]; then
echo "Invalid pod ID detected: $pod_id" >&2
exit 1
fi
while [ true ]; do
echo "Waiting for pod $pod_id to be ready..."
# Function to wait for pod to be ready
wait_for_pod() {
local pod_id=$1
local max_attempts=30
local attempt=1
while [ $attempt -le $max_attempts ]; do
echo "Waiting for pod $pod_id to be ready... (Attempt $attempt/$max_attempts)"
sleep 5
# Get the output directly and check if it contains RUNNING
output=$(runpodctl get pod "$pod_id" 2>&1)
@@ -82,8 +120,59 @@ while [ true ]; do
# Check both STATUS RUNNING (with two spaces) and STATUS RUNNING (with one space)
if echo "$output" | grep -q "STATUS *RUNNING" || echo "$output" | grep -q "RUNNING"; then
echo "Pod is now running!"
echo "https://www.runpod.io/console/pods"
break
echo "Pod is now running!"
echo "https://www.runpod.io/console/pods"
return 0
fi
attempt=$((attempt + 1))
done
echo "Pod did not reach RUNNING state within the timeout period."
return 1
}
# Main execution logic with retry mechanism
echo "Starting pod creation with retry logic..."
# Try different combinations of GPU types and cloud options
for gpu_type in "${GPU_CONFIGS[@]}"; do
# Try standard cloud (no specific arg)
echo "Trying with GPU type: $gpu_type (standard cloud)"
output=$(create_pod "$gpu_type" "")
echo "$output"
pod_id=$(extract_pod_id "$output")
if [ -n "$pod_id" ]; then
echo "Successfully created pod with ID: $pod_id"
wait_for_pod "$pod_id"
exit 0
fi
# Try community cloud
echo "Trying with GPU type: $gpu_type (community cloud)"
output=$(create_pod "$gpu_type" "--communityCloud")
echo "$output"
pod_id=$(extract_pod_id "$output")
if [ -n "$pod_id" ]; then
echo "Successfully created pod with ID: $pod_id"
wait_for_pod "$pod_id"
exit 0
fi
# Try secure cloud
echo "Trying with GPU type: $gpu_type (secure cloud)"
output=$(create_pod "$gpu_type" "--secureCloud")
echo "$output"
pod_id=$(extract_pod_id "$output")
if [ -n "$pod_id" ]; then
echo "Successfully created pod with ID: $pod_id"
wait_for_pod "$pod_id"
exit 0
fi
done
echo "Failed to create pod after trying all GPU types and cloud options."
exit 1

2
scripts/install.sh
View File

@@ -10,12 +10,10 @@ if [ -z "${SIA_INSTALL_NO_TRAIN}" ]; then
echo "Installing Train tool..."
python3 -m venv /root/venvs/train
/root/venvs/train/bin/pip install -e /root/sia/tools/train
/root/venvs/train/bin/ipython kernel install --name=train
fi
if [ -z "${SIA_INSTALL_NO_CORE}" ]; then
echo "Installing SIA core..."
python3 -m venv /root/venvs/sia
/root/venvs/sia/bin/pip install -e /root/sia
/root/venvs/sia/bin/ipython kernel install --name=sia
fi