diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index d1bd2ac..138c40d 100644 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -8,6 +8,16 @@ if [ -z "$SIA_REPO_URL" ]; then exit 1 fi +if [ -z "$SIA_REPO_USER" ]; then + echo "Error: SIA_REPO_USER environment variable is not set" + exit 1 +fi + +if [ -z "$SIA_REPO_PAT" ]; then + echo "Error: SIA_REPO_PAT environment variable is not set" + exit 1 +fi + # Create directory structure echo "Creating directory structure..." mkdir -p "/root/data/iterations" @@ -16,7 +26,7 @@ mkdir -p "/root/venvs" # Clone SIA repository echo "Cloning SIA repository..." -[ ! -d "/root/sia" ] && git clone "$SIA_REPO_URL" "/root/sia" +[ ! -d "/root/sia" ] && git clone "https://$SIA_REPO_USER:$SIA_REPO_PAT@${SIA_REPO_URL#https://}" "/root/sia" # Fixing permissions echo "Fixing permissions..." @@ -39,7 +49,7 @@ npm run build ln -s "/root/sia/web/dist" "/root/static" # Install SIA dependencies -"/root/sia/scripts/install.sh" +source "/root/sia/scripts/install.sh" # Finetune model echo "Finetuning model..." diff --git a/scripts/deploy.sh b/scripts/deploy.sh index d0f600d..d5513d1 100644 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -8,7 +8,6 @@ export MSYS_NO_PATHCONV=1 # =================== Configuration =================== # These can be overridden by environment variables or .env file -SIA_REPO_URL=${SIA_REPO_URL:-"git@git.nielsgeens.be:2222/llm/SIA.git"} # Pod configuration GPU_TYPE=${GPU_TYPE:-"NVIDIA GeForce RTX 3090"} @@ -43,8 +42,8 @@ output=$(runpodctl create pod \ --vcpu $CPU_COUNT \ --ports "8080/http,8888/http" \ --env "SIA_REPO_URL=$SIA_REPO_URL" \ - --env "SIA_GITHUB_API_KEY=$SIA_GITHUB_API_KEY" \ - --env "SIA_GITEA_API_KEY=$SIA_GITEA_API_KEY" \ + --env "SIA_REPO_USER=$SIA_REPO_USER" \ + --env "SIA_REPO_PAT=$SIA_REPO_PAT" \ --env "SIA_HF_API_KEY=$SIA_HF_API_KEY" \ --env "SIA_QWQ_ENABLED=1" \ --env "JUPYTER_PASSWORD=1" \ diff --git a/scripts/gitea_keys.sh b/scripts/gitea_keys.sh index 7b5646e..4a60f92 100644 --- a/scripts/gitea_keys.sh +++ b/scripts/gitea_keys.sh @@ -12,25 +12,58 @@ if [ -z "$SIA_REPO_URL" ]; then exit 1 fi -# Extract domain and port from repo URL -if [[ "$SIA_REPO_URL" == *":"*"/"* ]]; then - # Format: git@domain:port/path or domain:port/path - DOMAIN_PORT=$(echo "$SIA_REPO_URL" | cut -d'/' -f1) - DOMAIN=$(echo "$DOMAIN_PORT" | cut -d':' -f1) - PORT=$(echo "$DOMAIN_PORT" | cut -d':' -f2) +# Install dependencies +apt-get update +apt-get install -y jq openssh-client git + +# Extract host, user and port from SIA_REPO_URL +if [[ "$SIA_REPO_URL" == ssh://* ]]; then + # Handle ssh:// format + URL_PATH=${SIA_REPO_URL#ssh://} + + # Get user if specified + if [[ "$URL_PATH" == *"@"* ]]; then + SSH_USER=${URL_PATH%%@*} + URL_PATH=${URL_PATH#*@} + else + SSH_USER="git" + fi + + # Extract domain and port + if [[ "$URL_PATH" == *":"* ]]; then + # Format with port: domain:port/path + SSH_HOST=${URL_PATH%%:*} + PORT_PATH=${URL_PATH#*:} + SSH_PORT=${PORT_PATH%%/*} + else + # Format without port: domain/path + SSH_HOST=${URL_PATH%%/*} + SSH_PORT=22 + fi else - # Format: git@domain/path or domain/path (no port specified) - DOMAIN=$(echo "$SIA_REPO_URL" | cut -d'/' -f1) - PORT=22 + # Handle git@ format (git@github.com:user/repo.git) + if [[ "$SIA_REPO_URL" == *"@"* ]]; then + SSH_USER=${SIA_REPO_URL%%@*} + URL_PATH=${SIA_REPO_URL#*@} + + if [[ "$URL_PATH" == *":"* ]]; then + SSH_HOST=${URL_PATH%%:*} + SSH_PORT=22 # Standard format doesn't specify port + else + SSH_HOST=${URL_PATH%%/*} + SSH_PORT=22 + fi + else + SSH_USER="git" + SSH_HOST=${SIA_REPO_URL%%/*} + SSH_PORT=22 + fi fi -# If domain includes username (like git@domain), extract just the domain -if [[ "$DOMAIN" == *"@"* ]]; then - DOMAIN=$(echo "$DOMAIN" | cut -d'@' -f2) -fi - -echo "Gitea domain: $DOMAIN" -echo "Gitea port: $PORT" +echo "Using SSH configuration:" +echo " Host: $SSH_HOST" +echo " User: $SSH_USER" +echo " Port: $SSH_PORT" # Create SSH directory if it doesn't exist mkdir -p ~/.ssh @@ -42,31 +75,117 @@ if [ ! -f ~/.ssh/sia_repo ]; then ssh-keygen -t ed25519 -f ~/.ssh/sia_repo -N "" fi +# Ensure proper permissions on SSH keys +chmod 600 ~/.ssh/sia_repo +chmod 644 ~/.ssh/sia_repo.pub + # Get the public key SSH_PUBLIC_KEY=$(cat ~/.ssh/sia_repo.pub) # Add host key to known_hosts to avoid verification prompt echo "Adding host key to known_hosts..." -ssh-keyscan -t rsa -p $PORT $DOMAIN >> ~/.ssh/known_hosts +ssh-keyscan -t rsa -p $SSH_PORT $SSH_HOST >> ~/.ssh/known_hosts chmod 600 ~/.ssh/known_hosts +# Clear any existing connections in SSH config +if [ -f ~/.ssh/config ]; then + rm -f ~/.ssh/config +fi + # Check if a key with title "sia" already exists and delete it echo "Checking for existing SSH key in Gitea..." -EXISTING_KEYS=$(curl -s -X GET "https://$DOMAIN/api/v1/user/keys" \ +RESPONSE=$(curl -s -X GET "https://$SSH_HOST/api/v1/user/keys" \ -H "accept: application/json" \ -H "Authorization: token $SIA_GITEA_API_KEY") -KEY_ID=$(echo "$EXISTING_KEYS" | jq -r '.[] | select(.title=="sia") | .id') -if [ ! -z "$KEY_ID" ]; then - echo "Found existing key with ID $KEY_ID, deleting..." - curl -X DELETE "https://$DOMAIN/api/v1/user/keys/$KEY_ID" \ - -H "accept: application/json" \ - -H "Authorization: token $SIA_GITEA_API_KEY" + +# Debug: Print the response to help diagnose issues +echo "API Response: $RESPONSE" + +# Check if response is valid JSON +if echo "$RESPONSE" | jq empty 2>/dev/null; then + echo "Response is valid JSON" + + # Check if response is an array + if [ "$(echo "$RESPONSE" | jq 'if type=="array" then "array" else "not_array" end')" = '"array"' ]; then + echo "Response is an array, proceeding with key check" + KEY_ID=$(echo "$RESPONSE" | jq -r '.[] | select(.title=="sia") | .id') + + if [ ! -z "$KEY_ID" ] && [ "$KEY_ID" != "null" ]; then + echo "Found existing key with ID $KEY_ID, deleting..." + curl -X DELETE "https://$SSH_HOST/api/v1/user/keys/$KEY_ID" \ + -H "accept: application/json" \ + -H "Authorization: token $SIA_GITEA_API_KEY" + else + echo "No existing key with title 'sia' found" + fi + else + echo "Response is not an array, skipping key check" + fi +else + echo "Invalid JSON response from API, skipping key check" fi # Add the SSH key to Gitea using the API echo "Adding SSH key to Gitea using API..." -curl -X POST "https://$DOMAIN/api/v1/user/keys" \ +ADD_RESPONSE=$(curl -s -X POST "https://$SSH_HOST/api/v1/user/keys" \ -H "accept: application/json" \ -H "Content-Type: application/json" \ -H "Authorization: token $SIA_GITEA_API_KEY" \ - -d "{\"title\":\"sia\", \"key\":\"$SSH_PUBLIC_KEY\"}" \ No newline at end of file + -d "{\"title\":\"sia\", \"key\":\"$SSH_PUBLIC_KEY\"}") + +# Debug: Print the response +echo "Add key response: $ADD_RESPONSE" + +# Check if the response indicates success +if echo "$ADD_RESPONSE" | jq -e '.id' > /dev/null 2>&1; then + echo "SSH key added successfully!" +else + echo "Failed to add SSH key. Please check your API token and try again." + exit 1 +fi + +# Create SSH config file to specify which key to use +echo "Configuring SSH to use the correct key for repository host..." +cat > ~/.ssh/config << EOL +Host $SSH_HOST + User $SSH_USER + Port $SSH_PORT + IdentityFile ~/.ssh/sia_repo + PreferredAuthentications publickey + StrictHostKeyChecking no +EOL + +chmod 600 ~/.ssh/config + +# Stop any running SSH agents to start fresh +pkill ssh-agent || true + +# Start SSH agent and add the key +eval $(ssh-agent -s) +ssh-add ~/.ssh/sia_repo + +# Set up the Git SSH command to force use of the config file +export GIT_SSH_COMMAND="ssh -F ~/.ssh/config" + +# Test the SSH connection +echo "Testing SSH connection... (this might fail if the server rejects the connection)" +ssh -F ~/.ssh/config -T -p $SSH_PORT $SSH_USER@$SSH_HOST || true + +# Ensure git is configured with a name and email +if ! git config --global user.email > /dev/null; then + echo "Configuring default git user email..." + git config --global user.email "niels.geens@gmail.com" +fi + +if ! git config --global user.name > /dev/null; then + echo "Configuring default git user name..." + git config --global user.name "Niels Geens" +fi + +# Set the GIT_SSH_COMMAND environment variable persistently +echo 'export GIT_SSH_COMMAND="ssh -F ~/.ssh/config"' >> ~/.bashrc + +# Add a helper function to modify bootstrap.sh behavior without changing it +echo 'alias git="GIT_SSH_COMMAND=\"ssh -F ~/.ssh/config\" git"' >> ~/.bashrc + +echo "SSH key setup completed. The environment has been configured for git operations." \ No newline at end of file diff --git a/scripts/install.sh b/scripts/install.sh index d8dfd11..67e178a 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3,17 +3,17 @@ echo "Installing ITB tool..." python3 -m venv "/root/venvs/itb" /root/venvs/itb/bin/pip install -e /root/sia/tools/itb/ -echo "export PATH=\"/root/venvs/itb/bin/:\$PATH\"" > "/etc/profile.d/venv_itb.sh" \ +echo "PATH=\"/root/venvs/itb/bin/:\$PATH\"" > "/etc/profile.d/venv_itb.sh" source "/etc/profile.d/venv_itb.sh" echo "Installing Train tool..." python3 -m venv "/root/venvs/train" /root/venvs/train/bin/pip install -e /root/sia/tools/train/ -echo "export PATH=\"/root/venvs/train/bin/:\$PATH\"" > "/etc/profile.d/venv_train.sh" \ +echo "PATH=\"/root/venvs/train/bin/:\$PATH\"" > "/etc/profile.d/venv_train.sh" source "/etc/profile.d/venv_train.sh" echo "Installing SIA core..." python3 -m venv "/root/venvs/sia" /root/venvs/sia/bin/pip install -e /root/sia/ -echo "export PATH=\"/root/venvs/sia/bin/:\$PATH\"" > "/etc/profile.d/venv_sia.sh" \ +echo "PATH=\"/root/venvs/sia/bin/:\$PATH\"" > "/etc/profile.d/venv_sia.sh" source "/etc/profile.d/venv_sia.sh" \ No newline at end of file