Use PAT to authenticate

2025-03-20 14:42:42 +01:00
parent 9a171ab622
commit 8d9d3ca576
4 changed files with 162 additions and 34 deletions
--- a/scripts/bootstrap.sh
+++ b/scripts/bootstrap.sh
@@ -8,6 +8,16 @@ if [ -z "$SIA_REPO_URL" ]; then
    exit 1
 fi

+if [ -z "$SIA_REPO_USER" ]; then
+    echo "Error: SIA_REPO_USER environment variable is not set"
+    exit 1
+fi
+
+if [ -z "$SIA_REPO_PAT" ]; then
+    echo "Error: SIA_REPO_PAT environment variable is not set"
+    exit 1
+fi
+
 # Create directory structure
 echo "Creating directory structure..."
 mkdir -p "/root/data/iterations"
@@ -16,7 +26,7 @@ mkdir -p "/root/venvs"

 # Clone SIA repository
 echo "Cloning SIA repository..."
-[ ! -d "/root/sia" ] && git clone "$SIA_REPO_URL" "/root/sia"
+[ ! -d "/root/sia" ] && git clone "https://$SIA_REPO_USER:$SIA_REPO_PAT@${SIA_REPO_URL#https://}" "/root/sia"

 # Fixing permissions
 echo "Fixing permissions..."
@@ -39,7 +49,7 @@ npm run build
 ln -s "/root/sia/web/dist" "/root/static"

 # Install SIA dependencies
-"/root/sia/scripts/install.sh"
+source "/root/sia/scripts/install.sh"

 # Finetune model
 echo "Finetuning model..."
--- a/scripts/deploy.sh
+++ b/scripts/deploy.sh
@@ -8,7 +8,6 @@ export MSYS_NO_PATHCONV=1

 # =================== Configuration ===================
 # These can be overridden by environment variables or .env file
-SIA_REPO_URL=${SIA_REPO_URL:-"git@git.nielsgeens.be:2222/llm/SIA.git"}

 # Pod configuration
 GPU_TYPE=${GPU_TYPE:-"NVIDIA GeForce RTX 3090"}
@@ -43,8 +42,8 @@ output=$(runpodctl create pod \
    --vcpu $CPU_COUNT \
    --ports "8080/http,8888/http" \
    --env "SIA_REPO_URL=$SIA_REPO_URL" \
-    --env "SIA_GITHUB_API_KEY=$SIA_GITHUB_API_KEY" \
-    --env "SIA_GITEA_API_KEY=$SIA_GITEA_API_KEY" \
+    --env "SIA_REPO_USER=$SIA_REPO_USER" \
+    --env "SIA_REPO_PAT=$SIA_REPO_PAT" \
    --env "SIA_HF_API_KEY=$SIA_HF_API_KEY" \
    --env "SIA_QWQ_ENABLED=1" \
    --env "JUPYTER_PASSWORD=1" \
--- a/scripts/gitea_keys.sh
+++ b/scripts/gitea_keys.sh
@@ -12,25 +12,58 @@ if [ -z "$SIA_REPO_URL" ]; then
    exit 1
 fi

-# Extract domain and port from repo URL
-if [[ "$SIA_REPO_URL" == *":"*"/"* ]]; then
-    # Format: git@domain:port/path or domain:port/path
-    DOMAIN_PORT=$(echo "$SIA_REPO_URL" | cut -d'/' -f1)
-    DOMAIN=$(echo "$DOMAIN_PORT" | cut -d':' -f1)
-    PORT=$(echo "$DOMAIN_PORT" | cut -d':' -f2)
+# Install dependencies
+apt-get update
+apt-get install -y jq openssh-client git
+
+# Extract host, user and port from SIA_REPO_URL
+if [[ "$SIA_REPO_URL" == ssh://* ]]; then
+    # Handle ssh:// format
+    URL_PATH=${SIA_REPO_URL#ssh://}
+    
+    # Get user if specified
+    if [[ "$URL_PATH" == *"@"* ]]; then
+        SSH_USER=${URL_PATH%%@*}
+        URL_PATH=${URL_PATH#*@}
+    else
+        SSH_USER="git"
+    fi
+    
+    # Extract domain and port
+    if [[ "$URL_PATH" == *":"* ]]; then
+        # Format with port: domain:port/path
+        SSH_HOST=${URL_PATH%%:*}
+        PORT_PATH=${URL_PATH#*:}
+        SSH_PORT=${PORT_PATH%%/*}
+    else
+        # Format without port: domain/path
+        SSH_HOST=${URL_PATH%%/*}
+        SSH_PORT=22
+    fi
 else
-    # Format: git@domain/path or domain/path (no port specified)
-    DOMAIN=$(echo "$SIA_REPO_URL" | cut -d'/' -f1)
-    PORT=22
+    # Handle git@ format (git@github.com:user/repo.git)
+    if [[ "$SIA_REPO_URL" == *"@"* ]]; then
+        SSH_USER=${SIA_REPO_URL%%@*}
+        URL_PATH=${SIA_REPO_URL#*@}
+        
+        if [[ "$URL_PATH" == *":"* ]]; then
+            SSH_HOST=${URL_PATH%%:*}
+            SSH_PORT=22  # Standard format doesn't specify port
+        else
+            SSH_HOST=${URL_PATH%%/*}
+            SSH_PORT=22
+        fi
+    else
+        SSH_USER="git"
+        SSH_HOST=${SIA_REPO_URL%%/*}
+        SSH_PORT=22
+    fi
 fi

-# If domain includes username (like git@domain), extract just the domain
-if [[ "$DOMAIN" == *"@"* ]]; then
-    DOMAIN=$(echo "$DOMAIN" | cut -d'@' -f2)
-fi
-
-echo "Gitea domain: $DOMAIN"
-echo "Gitea port: $PORT"
+echo "Using SSH configuration:"
+echo "  Host: $SSH_HOST"
+echo "  User: $SSH_USER"
+echo "  Port: $SSH_PORT"

 # Create SSH directory if it doesn't exist
 mkdir -p ~/.ssh
@@ -42,31 +75,117 @@ if [ ! -f ~/.ssh/sia_repo ]; then
    ssh-keygen -t ed25519 -f ~/.ssh/sia_repo -N ""
 fi

+# Ensure proper permissions on SSH keys
+chmod 600 ~/.ssh/sia_repo
+chmod 644 ~/.ssh/sia_repo.pub
+
 # Get the public key
 SSH_PUBLIC_KEY=$(cat ~/.ssh/sia_repo.pub)

 # Add host key to known_hosts to avoid verification prompt
 echo "Adding host key to known_hosts..."
-ssh-keyscan -t rsa -p $PORT $DOMAIN >> ~/.ssh/known_hosts
+ssh-keyscan -t rsa -p $SSH_PORT $SSH_HOST >> ~/.ssh/known_hosts
 chmod 600 ~/.ssh/known_hosts

+# Clear any existing connections in SSH config
+if [ -f ~/.ssh/config ]; then
+    rm -f ~/.ssh/config
+fi
+
 # Check if a key with title "sia" already exists and delete it
 echo "Checking for existing SSH key in Gitea..."
-EXISTING_KEYS=$(curl -s -X GET "https://$DOMAIN/api/v1/user/keys" \
+RESPONSE=$(curl -s -X GET "https://$SSH_HOST/api/v1/user/keys" \
    -H "accept: application/json" \
    -H "Authorization: token $SIA_GITEA_API_KEY")
-KEY_ID=$(echo "$EXISTING_KEYS" | jq -r '.[] | select(.title=="sia") | .id')
-if [ ! -z "$KEY_ID" ]; then
-    echo "Found existing key with ID $KEY_ID, deleting..."
-    curl -X DELETE "https://$DOMAIN/api/v1/user/keys/$KEY_ID" \
-        -H "accept: application/json" \
-        -H "Authorization: token $SIA_GITEA_API_KEY"
+
+# Debug: Print the response to help diagnose issues
+echo "API Response: $RESPONSE"
+
+# Check if response is valid JSON
+if echo "$RESPONSE" | jq empty 2>/dev/null; then
+    echo "Response is valid JSON"
+    
+    # Check if response is an array
+    if [ "$(echo "$RESPONSE" | jq 'if type=="array" then "array" else "not_array" end')" = '"array"' ]; then
+        echo "Response is an array, proceeding with key check"
+        KEY_ID=$(echo "$RESPONSE" | jq -r '.[] | select(.title=="sia") | .id')
+        
+        if [ ! -z "$KEY_ID" ] && [ "$KEY_ID" != "null" ]; then
+            echo "Found existing key with ID $KEY_ID, deleting..."
+            curl -X DELETE "https://$SSH_HOST/api/v1/user/keys/$KEY_ID" \
+                -H "accept: application/json" \
+                -H "Authorization: token $SIA_GITEA_API_KEY"
+        else
+            echo "No existing key with title 'sia' found"
+        fi
+    else
+        echo "Response is not an array, skipping key check"
+    fi
+else
+    echo "Invalid JSON response from API, skipping key check"
 fi

 # Add the SSH key to Gitea using the API
 echo "Adding SSH key to Gitea using API..."
-curl -X POST "https://$DOMAIN/api/v1/user/keys" \
+ADD_RESPONSE=$(curl -s -X POST "https://$SSH_HOST/api/v1/user/keys" \
    -H "accept: application/json" \
    -H "Content-Type: application/json" \
    -H "Authorization: token $SIA_GITEA_API_KEY" \
-    -d "{\"title\":\"sia\", \"key\":\"$SSH_PUBLIC_KEY\"}"
+    -d "{\"title\":\"sia\", \"key\":\"$SSH_PUBLIC_KEY\"}")
+
+# Debug: Print the response
+echo "Add key response: $ADD_RESPONSE"
+
+# Check if the response indicates success
+if echo "$ADD_RESPONSE" | jq -e '.id' > /dev/null 2>&1; then
+    echo "SSH key added successfully!"
+else
+    echo "Failed to add SSH key. Please check your API token and try again."
+    exit 1
+fi
+
+# Create SSH config file to specify which key to use
+echo "Configuring SSH to use the correct key for repository host..."
+cat > ~/.ssh/config << EOL
+Host $SSH_HOST
+    User $SSH_USER
+    Port $SSH_PORT
+    IdentityFile ~/.ssh/sia_repo
+    PreferredAuthentications publickey
+    StrictHostKeyChecking no
+EOL
+
+chmod 600 ~/.ssh/config
+
+# Stop any running SSH agents to start fresh
+pkill ssh-agent || true
+
+# Start SSH agent and add the key
+eval $(ssh-agent -s)
+ssh-add ~/.ssh/sia_repo
+
+# Set up the Git SSH command to force use of the config file
+export GIT_SSH_COMMAND="ssh -F ~/.ssh/config"
+
+# Test the SSH connection
+echo "Testing SSH connection... (this might fail if the server rejects the connection)"
+ssh -F ~/.ssh/config -T -p $SSH_PORT $SSH_USER@$SSH_HOST || true
+
+# Ensure git is configured with a name and email
+if ! git config --global user.email > /dev/null; then
+    echo "Configuring default git user email..."
+    git config --global user.email "niels.geens@gmail.com"
+fi
+
+if ! git config --global user.name > /dev/null; then
+    echo "Configuring default git user name..."
+    git config --global user.name "Niels Geens"
+fi
+
+# Set the GIT_SSH_COMMAND environment variable persistently
+echo 'export GIT_SSH_COMMAND="ssh -F ~/.ssh/config"' >> ~/.bashrc
+
+# Add a helper function to modify bootstrap.sh behavior without changing it
+echo 'alias git="GIT_SSH_COMMAND=\"ssh -F ~/.ssh/config\" git"' >> ~/.bashrc
+
+echo "SSH key setup completed. The environment has been configured for git operations."
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -3,17 +3,17 @@
 echo "Installing ITB tool..."
 python3 -m venv "/root/venvs/itb"
 /root/venvs/itb/bin/pip install -e /root/sia/tools/itb/
-echo "export PATH=\"/root/venvs/itb/bin/:\$PATH\"" > "/etc/profile.d/venv_itb.sh" \
+echo "PATH=\"/root/venvs/itb/bin/:\$PATH\"" > "/etc/profile.d/venv_itb.sh"
 source "/etc/profile.d/venv_itb.sh"

 echo "Installing Train tool..."
 python3 -m venv "/root/venvs/train"
 /root/venvs/train/bin/pip install -e /root/sia/tools/train/
-echo "export PATH=\"/root/venvs/train/bin/:\$PATH\"" > "/etc/profile.d/venv_train.sh" \
+echo "PATH=\"/root/venvs/train/bin/:\$PATH\"" > "/etc/profile.d/venv_train.sh"
 source "/etc/profile.d/venv_train.sh"

 echo "Installing SIA core..."
 python3 -m venv "/root/venvs/sia"
 /root/venvs/sia/bin/pip install -e /root/sia/
-echo "export PATH=\"/root/venvs/sia/bin/:\$PATH\"" > "/etc/profile.d/venv_sia.sh" \
+echo "PATH=\"/root/venvs/sia/bin/:\$PATH\"" > "/etc/profile.d/venv_sia.sh"
 source "/etc/profile.d/venv_sia.sh"