llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Retry deploy if not available

Browse Source
This commit is contained in:
Niels Geens
2025-04-23 15:16:41 +02:00
parent bf94a0838a
commit c834141ead
3 changed files with 159 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
scripts/bootstrap.sh
View File

@@ -21,19 +21,35 @@ fi
# Install required packages # Install required packages
apt-get update apt-get update
apt-get install -y \ apt-get install -y \
build-essential \
git \
gnupg \ gnupg \
python3-dev \
python3-venv \
vim \ vim \
wget \ wget \
; ;
# Install Chrome
wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list
apt-get update && \ apt-get update && \
apt-get install -y \ apt-get install -y \
google-chrome-stable google-chrome-stable
# Install Rust
curl https://sh.rustup.rs -sSf | bash -s -- -y curl https://sh.rustup.rs -sSf | bash -s -- -y
export PATH="/root/.cargo/bin:${PATH}" export PATH="/root/.cargo/bin:${PATH}"
# Install Node.js
if ! command -v node &> /dev/null; then
echo "Installing Node.js..."
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
export NVM_DIR="/root/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
nvm install node
fi
# Create directory structure # Create directory structure
echo "Creating directory structure..." echo "Creating directory structure..."
mkdir -p "/root/data/iterations" mkdir -p "/root/data/iterations"
@@ -52,15 +68,6 @@ if [ ! -d "/root/sia" ]; then
git config --global core.editor vim git config --global core.editor vim
fi fi
# Install Node.js if needed
if ! command -v node &> /dev/null; then
echo "Installing Node.js..."
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
export NVM_DIR="/root/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
nvm install node
fi
# Build web interface # Build web interface
echo "Building web interface" echo "Building web interface"
cd "/root/sia/web" cd "/root/sia/web"

183
scripts/deploy.sh
View File

@@ -1,25 +1,23 @@
#!/bin/bash #!/bin/bash
# Script for setting up and running SIA on RunPod # Script for setting up and running SIA on RunPod with retry logic
# Requires the RunPod CLI (runpodctl) to be installed and configured # Requires the RunPod CLI (runpodctl) to be installed and configured
# Export MSYS_NO_PATHCONV=1 for Git Bash compatibility # Export MSYS_NO_PATHCONV=1 for Git Bash compatibility
export MSYS_NO_PATHCONV=1 export MSYS_NO_PATHCONV=1
# =================== Configuration =================== # =================== Configuration ===================
# These can be overridden by environment variables or .env file
# Pod configuration POD_NAME="sia-agent"
GPU_TYPE=${GPU_TYPE:-"NVIDIA RTX A6000"} CONTAINER_DISK_SIZE=200 # GB
GPU_COUNT=${GPU_COUNT:-1} VOLUME_SIZE=50 # GB
CONTAINER_DISK_SIZE=${CONTAINER_DISK_SIZE:-200} # GB VOLUME_PATH="/root/data" # Mount path within container
CPU_COUNT=${CPU_COUNT:-1} # vCPUs DOCKER_IMAGE="nvidia/cuda:12.2.0-runtime-ubuntu22.04"
POD_NAME=${POD_NAME:-"sia-agent"}
VOLUME_SIZE=${VOLUME_SIZE:-50} # GB
VOLUME_PATH=${VOLUME_PATH:-"/root/data"} # Mount path within container
# Docker configuration GPU_CONFIGS=(
DOCKER_IMAGE=${DOCKER_IMAGE:-"runpod/vscode-server:0.0.0"} "NVIDIA RTX 6000 Ada Generation"
"NVIDIA RTX A6000"
)
# Load environment variables from .env file if it exists # Load environment variables from .env file if it exists
if [ -f .env ]; then if [ -f .env ]; then
@@ -27,51 +25,91 @@ if [ -f .env ]; then
source .env source .env
fi fi
# Create pod # Function to create pod with given parameters
echo "Creating RunPod instance with GPU type: $GPU_TYPE" create_pod() {
output=$(runpodctl create pod \ local gpu_type=$1
--name "$POD_NAME" \ local cloud_arg=$2
--imageName "$DOCKER_IMAGE" \
--gpuType "$GPU_TYPE" \ echo "Attempting to create pod with:"
--gpuCount $GPU_COUNT \ echo "- GPU Type: $gpu_type"
echo "- Cloud Option: $cloud_arg"
create_cmd="runpodctl create pod \
--name \"$POD_NAME\" \
--imageName \"$DOCKER_IMAGE\" \
--gpuType \"$gpu_type\" \
--gpuCount 1 \
--containerDiskSize $CONTAINER_DISK_SIZE \ --containerDiskSize $CONTAINER_DISK_SIZE \
--volumeSize $VOLUME_SIZE \ --volumeSize $VOLUME_SIZE \
--volumePath "$VOLUME_PATH" \ --volumePath \"$VOLUME_PATH\" \
--vcpu $CPU_COUNT \ --vcpu 1 \
--ports "8080/http,8888/http" \ --ports \"8080/http,8888/http\" \
--env "SIA_REPO_URL=$SIA_REPO_URL" \ --env \"SIA_REPO_URL=$SIA_REPO_URL\" \
--env "SIA_REPO_USER=$SIA_REPO_USER" \ --env \"SIA_REPO_USER=$SIA_REPO_USER\" \
--env "SIA_REPO_PAT=$SIA_REPO_PAT" \ --env \"SIA_REPO_PAT=$SIA_REPO_PAT\" \
--env "SIA_HF_API_KEY=$SIA_HF_API_KEY" \ --env \"SIA_HF_API_KEY=$SIA_HF_API_KEY\" \
--env "SIA_QWQ_ENABLED=1" \ --env \"SIA_QWQ_ENABLED=1\" \
--env "JUPYTER_PASSWORD=1" \ --env \"JUPYTER_PASSWORD=1\" \
#--communityCloud --args \"/bin/bash -c \\\"apt-get update && \
) apt-get install -y curl && \
curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz && \
tar -xf vscode_cli.tar.gz && \
mkdir -p /usr/local/bin && \
mv code /usr/local/bin/ && \
rm vscode_cli.tar.gz && \
yes | code tunnel \
--name runpod \
--install-extension ms-toolsai.jupyter \
--install-extension ms-python.python\\\"\""
echo "$output" # Add cloud argument if specified
if [ -n "$cloud_arg" ]; then
create_cmd="$create_cmd $cloud_arg"
fi
# Extract the last line of output (should contain pod ID) # Execute the command and capture output
last_line=$(echo "$output" | tail -n 1) output=$(eval $create_cmd 2>&1)
# Extract pod ID - look for the word "pod" followed by ID in quotes
if [[ $last_line =~ pod\ \"([a-zA-Z0-9]+)\" ]]; then # Return the output
echo "$output"
}
# Function to extract pod ID from output
extract_pod_id() {
local output=$1
local pod_id=""
# Extract the last line of output (should contain pod ID)
local last_line=$(echo "$output" | tail -n 1)
# Extract pod ID - look for the word "pod" followed by ID in quotes
if [[ $last_line =~ pod\ \"([a-zA-Z0-9]+)\" ]]; then
pod_id="${BASH_REMATCH[1]}" pod_id="${BASH_REMATCH[1]}"
else else
# Try to get the last word which is often the pod ID # Try to get the last word which is often the pod ID
pod_id=$(echo "$last_line" | awk '{print $NF}') pod_id=$(echo "$last_line" | awk '{print $NF}')
fi fi
# If the ID has a double quote at the end, remove it # If the ID has a double quote at the end, remove it
pod_id=${pod_id%\"} pod_id=${pod_id%\"}
# Validate we have a proper pod ID (alphanumeric) # Validate we have a proper pod ID (alphanumeric)
if ! [[ $pod_id =~ ^[a-zA-Z0-9]+$ ]]; then if [[ $pod_id =~ ^[a-zA-Z0-9]+$ ]]; then
echo "Invalid pod ID detected: $pod_id" >&2 echo "$pod_id"
exit 1 return 0
fi else
return 1
fi
}
# Function to wait for pod to be ready
wait_for_pod() {
local pod_id=$1
local max_attempts=30
local attempt=1
while [ true ]; do while [ $attempt -le $max_attempts ]; do
echo "Waiting for pod $pod_id to be ready..." echo "Waiting for pod $pod_id to be ready... (Attempt $attempt/$max_attempts)"
sleep 5 sleep 5
# Get the output directly and check if it contains RUNNING # Get the output directly and check if it contains RUNNING
@@ -84,6 +122,57 @@ while [ true ]; do
if echo "$output" | grep -q "STATUS *RUNNING" || echo "$output" | grep -q "RUNNING"; then if echo "$output" | grep -q "STATUS *RUNNING" || echo "$output" | grep -q "RUNNING"; then
echo "Pod is now running!" echo "Pod is now running!"
echo "https://www.runpod.io/console/pods" echo "https://www.runpod.io/console/pods"
break return 0
fi
attempt=$((attempt + 1))
done
echo "Pod did not reach RUNNING state within the timeout period."
return 1
}
# Main execution logic with retry mechanism
echo "Starting pod creation with retry logic..."
# Try different combinations of GPU types and cloud options
for gpu_type in "${GPU_CONFIGS[@]}"; do
# Try standard cloud (no specific arg)
echo "Trying with GPU type: $gpu_type (standard cloud)"
output=$(create_pod "$gpu_type" "")
echo "$output"
pod_id=$(extract_pod_id "$output")
if [ -n "$pod_id" ]; then
echo "Successfully created pod with ID: $pod_id"
wait_for_pod "$pod_id"
exit 0
fi
# Try community cloud
echo "Trying with GPU type: $gpu_type (community cloud)"
output=$(create_pod "$gpu_type" "--communityCloud")
echo "$output"
pod_id=$(extract_pod_id "$output")
if [ -n "$pod_id" ]; then
echo "Successfully created pod with ID: $pod_id"
wait_for_pod "$pod_id"
exit 0
fi
# Try secure cloud
echo "Trying with GPU type: $gpu_type (secure cloud)"
output=$(create_pod "$gpu_type" "--secureCloud")
echo "$output"
pod_id=$(extract_pod_id "$output")
if [ -n "$pod_id" ]; then
echo "Successfully created pod with ID: $pod_id"
wait_for_pod "$pod_id"
exit 0
fi fi
done done
echo "Failed to create pod after trying all GPU types and cloud options."
exit 1

2
scripts/install.sh
View File

@@ -10,12 +10,10 @@ if [ -z "${SIA_INSTALL_NO_TRAIN}" ]; then
echo "Installing Train tool..." echo "Installing Train tool..."
python3 -m venv /root/venvs/train python3 -m venv /root/venvs/train
/root/venvs/train/bin/pip install -e /root/sia/tools/train /root/venvs/train/bin/pip install -e /root/sia/tools/train
/root/venvs/train/bin/ipython kernel install --name=train
fi fi
if [ -z "${SIA_INSTALL_NO_CORE}" ]; then if [ -z "${SIA_INSTALL_NO_CORE}" ]; then
echo "Installing SIA core..." echo "Installing SIA core..."
python3 -m venv /root/venvs/sia python3 -m venv /root/venvs/sia
/root/venvs/sia/bin/pip install -e /root/sia /root/venvs/sia/bin/pip install -e /root/sia
/root/venvs/sia/bin/ipython kernel install --name=sia
fi fi