Retry deploy if not available
This commit is contained in:
Niels Geens
@@ -1,25 +1,23 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Script for setting up and running SIA on RunPod
|
||||
# Script for setting up and running SIA on RunPod with retry logic
|
||||
# Requires the RunPod CLI (runpodctl) to be installed and configured
|
||||
|
||||
# Export MSYS_NO_PATHCONV=1 for Git Bash compatibility
|
||||
export MSYS_NO_PATHCONV=1
|
||||
|
||||
# =================== Configuration ===================
|
||||
# These can be overridden by environment variables or .env file
|
||||
|
||||
# Pod configuration
|
||||
GPU_TYPE=${GPU_TYPE:-"NVIDIA RTX A6000"}
|
||||
GPU_COUNT=${GPU_COUNT:-1}
|
||||
CONTAINER_DISK_SIZE=${CONTAINER_DISK_SIZE:-200} # GB
|
||||
CPU_COUNT=${CPU_COUNT:-1} # vCPUs
|
||||
POD_NAME=${POD_NAME:-"sia-agent"}
|
||||
VOLUME_SIZE=${VOLUME_SIZE:-50} # GB
|
||||
VOLUME_PATH=${VOLUME_PATH:-"/root/data"} # Mount path within container
|
||||
POD_NAME="sia-agent"
|
||||
CONTAINER_DISK_SIZE=200 # GB
|
||||
VOLUME_SIZE=50 # GB
|
||||
VOLUME_PATH="/root/data" # Mount path within container
|
||||
DOCKER_IMAGE="nvidia/cuda:12.2.0-runtime-ubuntu22.04"
|
||||
|
||||
# Docker configuration
|
||||
DOCKER_IMAGE=${DOCKER_IMAGE:-"runpod/vscode-server:0.0.0"}
|
||||
GPU_CONFIGS=(
|
||||
"NVIDIA RTX 6000 Ada Generation"
|
||||
"NVIDIA RTX A6000"
|
||||
)
|
||||
|
||||
# Load environment variables from .env file if it exists
|
||||
if [ -f .env ]; then
|
||||
@@ -27,53 +25,93 @@ if [ -f .env ]; then
|
||||
source .env
|
||||
fi
|
||||
|
||||
# Create pod
|
||||
echo "Creating RunPod instance with GPU type: $GPU_TYPE"
|
||||
output=$(runpodctl create pod \
|
||||
--name "$POD_NAME" \
|
||||
--imageName "$DOCKER_IMAGE" \
|
||||
--gpuType "$GPU_TYPE" \
|
||||
--gpuCount $GPU_COUNT \
|
||||
# Function to create pod with given parameters
|
||||
create_pod() {
|
||||
local gpu_type=$1
|
||||
local cloud_arg=$2
|
||||
|
||||
echo "Attempting to create pod with:"
|
||||
echo "- GPU Type: $gpu_type"
|
||||
echo "- Cloud Option: $cloud_arg"
|
||||
|
||||
create_cmd="runpodctl create pod \
|
||||
--name \"$POD_NAME\" \
|
||||
--imageName \"$DOCKER_IMAGE\" \
|
||||
--gpuType \"$gpu_type\" \
|
||||
--gpuCount 1 \
|
||||
--containerDiskSize $CONTAINER_DISK_SIZE \
|
||||
--volumeSize $VOLUME_SIZE \
|
||||
--volumePath "$VOLUME_PATH" \
|
||||
--vcpu $CPU_COUNT \
|
||||
--ports "8080/http,8888/http" \
|
||||
--env "SIA_REPO_URL=$SIA_REPO_URL" \
|
||||
--env "SIA_REPO_USER=$SIA_REPO_USER" \
|
||||
--env "SIA_REPO_PAT=$SIA_REPO_PAT" \
|
||||
--env "SIA_HF_API_KEY=$SIA_HF_API_KEY" \
|
||||
--env "SIA_QWQ_ENABLED=1" \
|
||||
--env "JUPYTER_PASSWORD=1" \
|
||||
#--communityCloud
|
||||
)
|
||||
--volumePath \"$VOLUME_PATH\" \
|
||||
--vcpu 1 \
|
||||
--ports \"8080/http,8888/http\" \
|
||||
--env \"SIA_REPO_URL=$SIA_REPO_URL\" \
|
||||
--env \"SIA_REPO_USER=$SIA_REPO_USER\" \
|
||||
--env \"SIA_REPO_PAT=$SIA_REPO_PAT\" \
|
||||
--env \"SIA_HF_API_KEY=$SIA_HF_API_KEY\" \
|
||||
--env \"SIA_QWQ_ENABLED=1\" \
|
||||
--env \"JUPYTER_PASSWORD=1\" \
|
||||
--args \"/bin/bash -c \\\"apt-get update && \
|
||||
apt-get install -y curl && \
|
||||
curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz && \
|
||||
tar -xf vscode_cli.tar.gz && \
|
||||
mkdir -p /usr/local/bin && \
|
||||
mv code /usr/local/bin/ && \
|
||||
rm vscode_cli.tar.gz && \
|
||||
yes | code tunnel \
|
||||
--name runpod \
|
||||
--install-extension ms-toolsai.jupyter \
|
||||
--install-extension ms-python.python\\\"\""
|
||||
|
||||
# Add cloud argument if specified
|
||||
if [ -n "$cloud_arg" ]; then
|
||||
create_cmd="$create_cmd $cloud_arg"
|
||||
fi
|
||||
|
||||
# Execute the command and capture output
|
||||
output=$(eval $create_cmd 2>&1)
|
||||
|
||||
# Return the output
|
||||
echo "$output"
|
||||
}
|
||||
|
||||
echo "$output"
|
||||
|
||||
# Extract the last line of output (should contain pod ID)
|
||||
last_line=$(echo "$output" | tail -n 1)
|
||||
# Extract pod ID - look for the word "pod" followed by ID in quotes
|
||||
if [[ $last_line =~ pod\ \"([a-zA-Z0-9]+)\" ]]; then
|
||||
# Function to extract pod ID from output
|
||||
extract_pod_id() {
|
||||
local output=$1
|
||||
local pod_id=""
|
||||
|
||||
# Extract the last line of output (should contain pod ID)
|
||||
local last_line=$(echo "$output" | tail -n 1)
|
||||
|
||||
# Extract pod ID - look for the word "pod" followed by ID in quotes
|
||||
if [[ $last_line =~ pod\ \"([a-zA-Z0-9]+)\" ]]; then
|
||||
pod_id="${BASH_REMATCH[1]}"
|
||||
else
|
||||
else
|
||||
# Try to get the last word which is often the pod ID
|
||||
pod_id=$(echo "$last_line" | awk '{print $NF}')
|
||||
fi
|
||||
fi
|
||||
|
||||
# If the ID has a double quote at the end, remove it
|
||||
pod_id=${pod_id%\"}
|
||||
|
||||
# Validate we have a proper pod ID (alphanumeric)
|
||||
if [[ $pod_id =~ ^[a-zA-Z0-9]+$ ]]; then
|
||||
echo "$pod_id"
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# If the ID has a double quote at the end, remove it
|
||||
pod_id=${pod_id%\"}
|
||||
|
||||
# Validate we have a proper pod ID (alphanumeric)
|
||||
if ! [[ $pod_id =~ ^[a-zA-Z0-9]+$ ]]; then
|
||||
echo "Invalid pod ID detected: $pod_id" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
while [ true ]; do
|
||||
echo "Waiting for pod $pod_id to be ready..."
|
||||
# Function to wait for pod to be ready
|
||||
wait_for_pod() {
|
||||
local pod_id=$1
|
||||
local max_attempts=30
|
||||
local attempt=1
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
echo "Waiting for pod $pod_id to be ready... (Attempt $attempt/$max_attempts)"
|
||||
sleep 5
|
||||
|
||||
|
||||
# Get the output directly and check if it contains RUNNING
|
||||
output=$(runpodctl get pod "$pod_id" 2>&1)
|
||||
|
||||
@@ -82,8 +120,59 @@ while [ true ]; do
|
||||
|
||||
# Check both STATUS RUNNING (with two spaces) and STATUS RUNNING (with one space)
|
||||
if echo "$output" | grep -q "STATUS *RUNNING" || echo "$output" | grep -q "RUNNING"; then
|
||||
echo "Pod is now running!"
|
||||
echo "https://www.runpod.io/console/pods"
|
||||
break
|
||||
echo "Pod is now running!"
|
||||
echo "https://www.runpod.io/console/pods"
|
||||
return 0
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "Pod did not reach RUNNING state within the timeout period."
|
||||
return 1
|
||||
}
|
||||
|
||||
# Main execution logic with retry mechanism
|
||||
echo "Starting pod creation with retry logic..."
|
||||
|
||||
# Try different combinations of GPU types and cloud options
|
||||
for gpu_type in "${GPU_CONFIGS[@]}"; do
|
||||
# Try standard cloud (no specific arg)
|
||||
echo "Trying with GPU type: $gpu_type (standard cloud)"
|
||||
output=$(create_pod "$gpu_type" "")
|
||||
echo "$output"
|
||||
|
||||
pod_id=$(extract_pod_id "$output")
|
||||
if [ -n "$pod_id" ]; then
|
||||
echo "Successfully created pod with ID: $pod_id"
|
||||
wait_for_pod "$pod_id"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Try community cloud
|
||||
echo "Trying with GPU type: $gpu_type (community cloud)"
|
||||
output=$(create_pod "$gpu_type" "--communityCloud")
|
||||
echo "$output"
|
||||
|
||||
pod_id=$(extract_pod_id "$output")
|
||||
if [ -n "$pod_id" ]; then
|
||||
echo "Successfully created pod with ID: $pod_id"
|
||||
wait_for_pod "$pod_id"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Try secure cloud
|
||||
echo "Trying with GPU type: $gpu_type (secure cloud)"
|
||||
output=$(create_pod "$gpu_type" "--secureCloud")
|
||||
echo "$output"
|
||||
|
||||
pod_id=$(extract_pod_id "$output")
|
||||
if [ -n "$pod_id" ]; then
|
||||
echo "Successfully created pod with ID: $pod_id"
|
||||
wait_for_pod "$pod_id"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Failed to create pod after trying all GPU types and cloud options."
|
||||
exit 1
|
||||
Reference in New Issue
Block a user