Retry deploy if not available

scripts/bootstrap.sh

@@ -21,19 +21,35 @@ fi
 # Install required packages
 apt-get update
 apt-get install -y \
+    build-essential \
+    git \
     gnupg \
+    python3-dev \
+    python3-venv \
     vim \
     wget \
     ;
+
+# Install Chrome
 wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
 echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list
 apt-get update && \
     apt-get install -y \
     google-chrome-stable
 
+# Install Rust
 curl https://sh.rustup.rs -sSf | bash -s -- -y
 export PATH="/root/.cargo/bin:${PATH}"
 
+# Install Node.js
+if ! command -v node &> /dev/null; then
+    echo "Installing Node.js..."
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
+    export NVM_DIR="/root/.nvm"
+    [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
+    nvm install node
+fi
+
 # Create directory structure
 echo "Creating directory structure..."
 mkdir -p "/root/data/iterations"
@@ -52,15 +68,6 @@ if [ ! -d "/root/sia" ]; then
     git config --global core.editor vim
 fi
 
-# Install Node.js if needed
-if ! command -v node &> /dev/null; then
-    echo "Installing Node.js..."
-    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
-    export NVM_DIR="/root/.nvm"
-    [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
-    nvm install node
-fi
-
 # Build web interface
 echo "Building web interface"
 cd "/root/sia/web"

scripts/deploy.sh

@@ -1,25 +1,23 @@
 #!/bin/bash
 
-# Script for setting up and running SIA on RunPod
+# Script for setting up and running SIA on RunPod with retry logic
 # Requires the RunPod CLI (runpodctl) to be installed and configured
 
 # Export MSYS_NO_PATHCONV=1 for Git Bash compatibility
 export MSYS_NO_PATHCONV=1
 
 # =================== Configuration ===================
-# These can be overridden by environment variables or .env file
 
-# Pod configuration
+POD_NAME="sia-agent"
-GPU_TYPE=${GPU_TYPE:-"NVIDIA RTX A6000"}
+CONTAINER_DISK_SIZE=200     # GB
-GPU_COUNT=${GPU_COUNT:-1}
+VOLUME_SIZE=50              # GB
-CONTAINER_DISK_SIZE=${CONTAINER_DISK_SIZE:-200}     # GB
+VOLUME_PATH="/root/data"    # Mount path within container
-CPU_COUNT=${CPU_COUNT:-1}                           # vCPUs
+DOCKER_IMAGE="nvidia/cuda:12.2.0-runtime-ubuntu22.04"
-POD_NAME=${POD_NAME:-"sia-agent"}
-VOLUME_SIZE=${VOLUME_SIZE:-50}                     # GB
-VOLUME_PATH=${VOLUME_PATH:-"/root/data"}            # Mount path within container
 
-# Docker configuration
+GPU_CONFIGS=(
-DOCKER_IMAGE=${DOCKER_IMAGE:-"runpod/vscode-server:0.0.0"}
+  "NVIDIA RTX 6000 Ada Generation"
+  "NVIDIA RTX A6000"
+)
 
 # Load environment variables from .env file if it exists
 if [ -f .env ]; then
@@ -27,31 +25,63 @@ if [ -f .env ]; then
     source .env
 fi
 
-# Create pod
+# Function to create pod with given parameters
-echo "Creating RunPod instance with GPU type: $GPU_TYPE"
+create_pod() {
-output=$(runpodctl create pod \
+  local gpu_type=$1
-    --name "$POD_NAME" \
+  local cloud_arg=$2
-    --imageName "$DOCKER_IMAGE" \
+  
-    --gpuType "$GPU_TYPE" \
+  echo "Attempting to create pod with:"
-    --gpuCount $GPU_COUNT \
+  echo "- GPU Type: $gpu_type"
+  echo "- Cloud Option: $cloud_arg"
+  
+  create_cmd="runpodctl create pod \
+    --name \"$POD_NAME\" \
+    --imageName \"$DOCKER_IMAGE\" \
+    --gpuType \"$gpu_type\" \
+    --gpuCount 1 \
     --containerDiskSize $CONTAINER_DISK_SIZE \
     --volumeSize $VOLUME_SIZE \
-    --volumePath "$VOLUME_PATH" \
+    --volumePath \"$VOLUME_PATH\" \
-    --vcpu $CPU_COUNT \
+    --vcpu 1 \
-    --ports "8080/http,8888/http" \
+    --ports \"8080/http,8888/http\" \
-    --env "SIA_REPO_URL=$SIA_REPO_URL" \
+    --env \"SIA_REPO_URL=$SIA_REPO_URL\" \
-    --env "SIA_REPO_USER=$SIA_REPO_USER" \
+    --env \"SIA_REPO_USER=$SIA_REPO_USER\" \
-    --env "SIA_REPO_PAT=$SIA_REPO_PAT" \
+    --env \"SIA_REPO_PAT=$SIA_REPO_PAT\" \
-    --env "SIA_HF_API_KEY=$SIA_HF_API_KEY" \
+    --env \"SIA_HF_API_KEY=$SIA_HF_API_KEY\" \
-    --env "SIA_QWQ_ENABLED=1" \
+    --env \"SIA_QWQ_ENABLED=1\" \
-    --env "JUPYTER_PASSWORD=1" \
+    --env \"JUPYTER_PASSWORD=1\" \
-    #--communityCloud
+    --args \"/bin/bash -c \\\"apt-get update && \
-)
+      apt-get install -y curl && \
+      curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz && \
+      tar -xf vscode_cli.tar.gz && \
+      mkdir -p /usr/local/bin && \
+      mv code /usr/local/bin/ && \
+      rm vscode_cli.tar.gz && \
+      yes | code tunnel \
+        --name runpod \
+        --install-extension ms-toolsai.jupyter \
+        --install-extension ms-python.python\\\"\""
   
+  # Add cloud argument if specified
+  if [ -n "$cloud_arg" ]; then
+    create_cmd="$create_cmd $cloud_arg"
+  fi
+  
+  # Execute the command and capture output
+  output=$(eval $create_cmd 2>&1)
+  
+  # Return the output
   echo "$output"
+}
+
+# Function to extract pod ID from output
+extract_pod_id() {
+  local output=$1
+  local pod_id=""
   
   # Extract the last line of output (should contain pod ID)
-last_line=$(echo "$output" | tail -n 1)
+  local last_line=$(echo "$output" | tail -n 1)
+  
   # Extract pod ID - look for the word "pod" followed by ID in quotes
   if [[ $last_line =~ pod\ \"([a-zA-Z0-9]+)\" ]]; then
     pod_id="${BASH_REMATCH[1]}"
@@ -64,14 +94,22 @@ fi
   pod_id=${pod_id%\"}
   
   # Validate we have a proper pod ID (alphanumeric)
-if ! [[ $pod_id =~ ^[a-zA-Z0-9]+$ ]]; then
+  if [[ $pod_id =~ ^[a-zA-Z0-9]+$ ]]; then
-    echo "Invalid pod ID detected: $pod_id" >&2
+    echo "$pod_id"
-    exit 1
+    return 0
+  else
+    return 1
   fi
+}
 
+# Function to wait for pod to be ready
+wait_for_pod() {
+  local pod_id=$1
+  local max_attempts=30
+  local attempt=1
   
-while [ true ]; do
+  while [ $attempt -le $max_attempts ]; do
-    echo "Waiting for pod $pod_id to be ready..."
+    echo "Waiting for pod $pod_id to be ready... (Attempt $attempt/$max_attempts)"
     sleep 5
     
     # Get the output directly and check if it contains RUNNING
@@ -84,6 +122,57 @@ while [ true ]; do
     if echo "$output" | grep -q "STATUS  *RUNNING" || echo "$output" | grep -q "RUNNING"; then
       echo "Pod is now running!"
       echo "https://www.runpod.io/console/pods"
-        break
+      return 0
+    fi
+    
+    attempt=$((attempt + 1))
+  done
+  
+  echo "Pod did not reach RUNNING state within the timeout period."
+  return 1
+}
+
+# Main execution logic with retry mechanism
+echo "Starting pod creation with retry logic..."
+
+# Try different combinations of GPU types and cloud options
+for gpu_type in "${GPU_CONFIGS[@]}"; do
+  # Try standard cloud (no specific arg)
+  echo "Trying with GPU type: $gpu_type (standard cloud)"
+  output=$(create_pod "$gpu_type" "")
+  echo "$output"
+  
+  pod_id=$(extract_pod_id "$output")
+  if [ -n "$pod_id" ]; then
+    echo "Successfully created pod with ID: $pod_id"
+    wait_for_pod "$pod_id"
+    exit 0
+  fi
+  
+  # Try community cloud
+  echo "Trying with GPU type: $gpu_type (community cloud)"
+  output=$(create_pod "$gpu_type" "--communityCloud")
+  echo "$output"
+  
+  pod_id=$(extract_pod_id "$output")
+  if [ -n "$pod_id" ]; then
+    echo "Successfully created pod with ID: $pod_id"
+    wait_for_pod "$pod_id"
+    exit 0
+  fi
+  
+  # Try secure cloud
+  echo "Trying with GPU type: $gpu_type (secure cloud)"
+  output=$(create_pod "$gpu_type" "--secureCloud")
+  echo "$output"
+  
+  pod_id=$(extract_pod_id "$output")
+  if [ -n "$pod_id" ]; then
+    echo "Successfully created pod with ID: $pod_id"
+    wait_for_pod "$pod_id"
+    exit 0
   fi
 done
+
+echo "Failed to create pod after trying all GPU types and cloud options."
+exit 1

scripts/install.sh

@@ -10,12 +10,10 @@ if [ -z "${SIA_INSTALL_NO_TRAIN}" ]; then
     echo "Installing Train tool..."
     python3 -m venv /root/venvs/train
     /root/venvs/train/bin/pip install -e /root/sia/tools/train
-    /root/venvs/train/bin/ipython kernel install --name=train
 fi
 
 if [ -z "${SIA_INSTALL_NO_CORE}" ]; then
     echo "Installing SIA core..."
     python3 -m venv /root/venvs/sia
     /root/venvs/sia/bin/pip install -e /root/sia
-    /root/venvs/sia/bin/ipython kernel install --name=sia
 fi