From c834141eadc12f0c71c931809615f7000ef537fc Mon Sep 17 00:00:00 2001 From: Niels Geens Date: Wed, 23 Apr 2025 15:16:41 +0200 Subject: [PATCH] Retry deploy if not available --- scripts/bootstrap.sh | 25 ++++-- scripts/deploy.sh | 197 +++++++++++++++++++++++++++++++------------ scripts/install.sh | 2 - 3 files changed, 159 insertions(+), 65 deletions(-) diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index e65c321..68f960d 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -21,19 +21,35 @@ fi # Install required packages apt-get update apt-get install -y \ + build-essential \ + git \ gnupg \ + python3-dev \ + python3-venv \ vim \ wget \ ; + +# Install Chrome wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list apt-get update && \ apt-get install -y \ google-chrome-stable +# Install Rust curl https://sh.rustup.rs -sSf | bash -s -- -y export PATH="/root/.cargo/bin:${PATH}" +# Install Node.js +if ! command -v node &> /dev/null; then + echo "Installing Node.js..." + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash + export NVM_DIR="/root/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" + nvm install node +fi + # Create directory structure echo "Creating directory structure..." mkdir -p "/root/data/iterations" @@ -52,15 +68,6 @@ if [ ! -d "/root/sia" ]; then git config --global core.editor vim fi -# Install Node.js if needed -if ! command -v node &> /dev/null; then - echo "Installing Node.js..." - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash - export NVM_DIR="/root/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" - nvm install node -fi - # Build web interface echo "Building web interface" cd "/root/sia/web" diff --git a/scripts/deploy.sh b/scripts/deploy.sh index b3d0666..8969d29 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -1,25 +1,23 @@ #!/bin/bash -# Script for setting up and running SIA on RunPod +# Script for setting up and running SIA on RunPod with retry logic # Requires the RunPod CLI (runpodctl) to be installed and configured # Export MSYS_NO_PATHCONV=1 for Git Bash compatibility export MSYS_NO_PATHCONV=1 # =================== Configuration =================== -# These can be overridden by environment variables or .env file -# Pod configuration -GPU_TYPE=${GPU_TYPE:-"NVIDIA RTX A6000"} -GPU_COUNT=${GPU_COUNT:-1} -CONTAINER_DISK_SIZE=${CONTAINER_DISK_SIZE:-200} # GB -CPU_COUNT=${CPU_COUNT:-1} # vCPUs -POD_NAME=${POD_NAME:-"sia-agent"} -VOLUME_SIZE=${VOLUME_SIZE:-50} # GB -VOLUME_PATH=${VOLUME_PATH:-"/root/data"} # Mount path within container +POD_NAME="sia-agent" +CONTAINER_DISK_SIZE=200 # GB +VOLUME_SIZE=50 # GB +VOLUME_PATH="/root/data" # Mount path within container +DOCKER_IMAGE="nvidia/cuda:12.2.0-runtime-ubuntu22.04" -# Docker configuration -DOCKER_IMAGE=${DOCKER_IMAGE:-"runpod/vscode-server:0.0.0"} +GPU_CONFIGS=( + "NVIDIA RTX 6000 Ada Generation" + "NVIDIA RTX A6000" +) # Load environment variables from .env file if it exists if [ -f .env ]; then @@ -27,53 +25,93 @@ if [ -f .env ]; then source .env fi -# Create pod -echo "Creating RunPod instance with GPU type: $GPU_TYPE" -output=$(runpodctl create pod \ - --name "$POD_NAME" \ - --imageName "$DOCKER_IMAGE" \ - --gpuType "$GPU_TYPE" \ - --gpuCount $GPU_COUNT \ +# Function to create pod with given parameters +create_pod() { + local gpu_type=$1 + local cloud_arg=$2 + + echo "Attempting to create pod with:" + echo "- GPU Type: $gpu_type" + echo "- Cloud Option: $cloud_arg" + + create_cmd="runpodctl create pod \ + --name \"$POD_NAME\" \ + --imageName \"$DOCKER_IMAGE\" \ + --gpuType \"$gpu_type\" \ + --gpuCount 1 \ --containerDiskSize $CONTAINER_DISK_SIZE \ --volumeSize $VOLUME_SIZE \ - --volumePath "$VOLUME_PATH" \ - --vcpu $CPU_COUNT \ - --ports "8080/http,8888/http" \ - --env "SIA_REPO_URL=$SIA_REPO_URL" \ - --env "SIA_REPO_USER=$SIA_REPO_USER" \ - --env "SIA_REPO_PAT=$SIA_REPO_PAT" \ - --env "SIA_HF_API_KEY=$SIA_HF_API_KEY" \ - --env "SIA_QWQ_ENABLED=1" \ - --env "JUPYTER_PASSWORD=1" \ - #--communityCloud -) + --volumePath \"$VOLUME_PATH\" \ + --vcpu 1 \ + --ports \"8080/http,8888/http\" \ + --env \"SIA_REPO_URL=$SIA_REPO_URL\" \ + --env \"SIA_REPO_USER=$SIA_REPO_USER\" \ + --env \"SIA_REPO_PAT=$SIA_REPO_PAT\" \ + --env \"SIA_HF_API_KEY=$SIA_HF_API_KEY\" \ + --env \"SIA_QWQ_ENABLED=1\" \ + --env \"JUPYTER_PASSWORD=1\" \ + --args \"/bin/bash -c \\\"apt-get update && \ + apt-get install -y curl && \ + curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz && \ + tar -xf vscode_cli.tar.gz && \ + mkdir -p /usr/local/bin && \ + mv code /usr/local/bin/ && \ + rm vscode_cli.tar.gz && \ + yes | code tunnel \ + --name runpod \ + --install-extension ms-toolsai.jupyter \ + --install-extension ms-python.python\\\"\"" + + # Add cloud argument if specified + if [ -n "$cloud_arg" ]; then + create_cmd="$create_cmd $cloud_arg" + fi + + # Execute the command and capture output + output=$(eval $create_cmd 2>&1) + + # Return the output + echo "$output" +} -echo "$output" - -# Extract the last line of output (should contain pod ID) -last_line=$(echo "$output" | tail -n 1) -# Extract pod ID - look for the word "pod" followed by ID in quotes -if [[ $last_line =~ pod\ \"([a-zA-Z0-9]+)\" ]]; then +# Function to extract pod ID from output +extract_pod_id() { + local output=$1 + local pod_id="" + + # Extract the last line of output (should contain pod ID) + local last_line=$(echo "$output" | tail -n 1) + + # Extract pod ID - look for the word "pod" followed by ID in quotes + if [[ $last_line =~ pod\ \"([a-zA-Z0-9]+)\" ]]; then pod_id="${BASH_REMATCH[1]}" -else + else # Try to get the last word which is often the pod ID pod_id=$(echo "$last_line" | awk '{print $NF}') -fi + fi + + # If the ID has a double quote at the end, remove it + pod_id=${pod_id%\"} + + # Validate we have a proper pod ID (alphanumeric) + if [[ $pod_id =~ ^[a-zA-Z0-9]+$ ]]; then + echo "$pod_id" + return 0 + else + return 1 + fi +} -# If the ID has a double quote at the end, remove it -pod_id=${pod_id%\"} - -# Validate we have a proper pod ID (alphanumeric) -if ! [[ $pod_id =~ ^[a-zA-Z0-9]+$ ]]; then - echo "Invalid pod ID detected: $pod_id" >&2 - exit 1 -fi - - -while [ true ]; do - echo "Waiting for pod $pod_id to be ready..." +# Function to wait for pod to be ready +wait_for_pod() { + local pod_id=$1 + local max_attempts=30 + local attempt=1 + + while [ $attempt -le $max_attempts ]; do + echo "Waiting for pod $pod_id to be ready... (Attempt $attempt/$max_attempts)" sleep 5 - + # Get the output directly and check if it contains RUNNING output=$(runpodctl get pod "$pod_id" 2>&1) @@ -82,8 +120,59 @@ while [ true ]; do # Check both STATUS RUNNING (with two spaces) and STATUS RUNNING (with one space) if echo "$output" | grep -q "STATUS *RUNNING" || echo "$output" | grep -q "RUNNING"; then - echo "Pod is now running!" - echo "https://www.runpod.io/console/pods" - break + echo "Pod is now running!" + echo "https://www.runpod.io/console/pods" + return 0 fi + + attempt=$((attempt + 1)) + done + + echo "Pod did not reach RUNNING state within the timeout period." + return 1 +} + +# Main execution logic with retry mechanism +echo "Starting pod creation with retry logic..." + +# Try different combinations of GPU types and cloud options +for gpu_type in "${GPU_CONFIGS[@]}"; do + # Try standard cloud (no specific arg) + echo "Trying with GPU type: $gpu_type (standard cloud)" + output=$(create_pod "$gpu_type" "") + echo "$output" + + pod_id=$(extract_pod_id "$output") + if [ -n "$pod_id" ]; then + echo "Successfully created pod with ID: $pod_id" + wait_for_pod "$pod_id" + exit 0 + fi + + # Try community cloud + echo "Trying with GPU type: $gpu_type (community cloud)" + output=$(create_pod "$gpu_type" "--communityCloud") + echo "$output" + + pod_id=$(extract_pod_id "$output") + if [ -n "$pod_id" ]; then + echo "Successfully created pod with ID: $pod_id" + wait_for_pod "$pod_id" + exit 0 + fi + + # Try secure cloud + echo "Trying with GPU type: $gpu_type (secure cloud)" + output=$(create_pod "$gpu_type" "--secureCloud") + echo "$output" + + pod_id=$(extract_pod_id "$output") + if [ -n "$pod_id" ]; then + echo "Successfully created pod with ID: $pod_id" + wait_for_pod "$pod_id" + exit 0 + fi done + +echo "Failed to create pod after trying all GPU types and cloud options." +exit 1 \ No newline at end of file diff --git a/scripts/install.sh b/scripts/install.sh index 3bb8059..2f1519a 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -10,12 +10,10 @@ if [ -z "${SIA_INSTALL_NO_TRAIN}" ]; then echo "Installing Train tool..." python3 -m venv /root/venvs/train /root/venvs/train/bin/pip install -e /root/sia/tools/train - /root/venvs/train/bin/ipython kernel install --name=train fi if [ -z "${SIA_INSTALL_NO_CORE}" ]; then echo "Installing SIA core..." python3 -m venv /root/venvs/sia /root/venvs/sia/bin/pip install -e /root/sia - /root/venvs/sia/bin/ipython kernel install --name=sia fi \ No newline at end of file