Gemma training script

scripts/bootstrap.sh

@@ -1,9 +1,12 @@
 #!/bin/bash
 
-#SIA_INSTALL_NO_ITB=1
-#SIA_INSTALL_NO_TRAIN=1
-#SIA_INSTALL_NO_NOTEBOOK=1
+#SIA_INSTALL_NO_WEB=1
+SIA_INSTALL_NO_NOTEBOOK=1
 #SIA_INSTALL_NO_CORE=1
+SIA_INSTALL_NO_ITB=1
+SIA_INSTALL_NO_MISTRAL_INFER=1
+#SIA_INSTALL_NO_GEMMA_INFER=1
+#SIA_INSTALL_NO_GEMMA_TRAIN=1
 
 trap 'return 1' ERR
 
@@ -29,11 +32,14 @@ apt-get install -y \
     build-essential \
     cmake \
     cuda-toolkit \
+    curl \
     git \
     gnupg \
+    jq \
     libcurl4-nss-dev \
     python3-dev \
     python3-venv \
+    tmux \
     vim \
     wget \
     ;
@@ -48,18 +54,15 @@ if [ -z "${SIA_INSTALL_NO_ITB}" ]; then
 fi
 
 # Install Rust
-if [ -z "${SIA_INSTALL_NO_TRAIN}" ]; then
-    curl https://sh.rustup.rs -sSf | bash -s -- -y
-    export PATH="/root/.cargo/bin:${PATH}"
-fi
+curl https://sh.rustup.rs -sSf | bash -s -- -y
+export PATH="/root/.cargo/bin:${PATH}"
 
 # Install Node.js
 if [ -z "${SIA_INSTALL_NO_CORE}" ]; then
     echo "Installing Node.js..."
     curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
-    export NVM_DIR="/root/.nvm"
-    [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
-    nvm install node
+    export NVM_DIR=/root/.nvm
+    . "$NVM_DIR/nvm.sh" && nvm install node
 fi
 
 # Install llama.cpp
@@ -87,36 +90,14 @@ if [ ! -d "/root/sia" ]; then
     git config --global core.editor vim
 fi
 
-# Build web interface
-if [ -z "${SIA_INSTALL_NO_CORE}" ]; then
-    echo "Building web interface"
-    cd "/root/sia/web"
-    npm install
-    npm run build
-    ln -s "/root/sia/web/dist" "/root/static"
-fi
-
 # Install SIA dependencies
 cd /root/desktop
 source "/root/sia/scripts/install.sh"
 
-# Add venvs to path in .bashrc
-if ! grep -q "source /root/sia/scripts/add_venvs_to_path.sh" /root/.bashrc; then
-    echo 'source /root/sia/scripts/add_venvs_to_path.sh' >> /root/.bashrc
-fi
-
-# Add venvs to path in profile
-if ! grep -q "source /root/sia/scripts/add_venvs_to_path.sh" /etc/profile; then
-    echo 'source /root/sia/scripts/add_venvs_to_path.sh' >> /etc/profile
-fi
-
-# Add venvs to path in current shell
-source /root/sia/scripts/add_venvs_to_path.sh
-
 # Finetune model
 if [ -z "${SIA_INSTALL_NO_TRAIN}" ]; then
     echo "Finetuning model..."
-    train /root/models/bootstrap
+    /root/sia/scripts/train.sh /root/models/bootstrap
     ln -s /root/models/bootstrap /root/models/current
 fi
 

scripts/install.sh

@@ -1,46 +1,68 @@
 #!/bin/bash
 
+#SIA_INSTALL_NO_WEB=1
 SIA_INSTALL_NO_NOTEBOOK=1
 #SIA_INSTALL_NO_CORE=1
 SIA_INSTALL_NO_ITB=1
 SIA_INSTALL_NO_MISTRAL_INFER=1
 #SIA_INSTALL_NO_GEMMA_INFER=1
-SIA_INSTALL_NO_TRAIN=1
+#SIA_INSTALL_NO_GEMMA_TRAIN=1
+
+cd "/root/desktop"
+
+if [ -z "${SIA_INSTALL_NO_WEB}" ]; then
+    echo "Building web interface"
+    (
+        cd "/root/sia/web"
+        npm install
+        npm run build
+        ln -s "/root/sia/web/dist" "/root/static"
+    )
+fi
 
 if [ -z "${SIA_INSTALL_NO_NOTEBOOK}" ]; then
-    echo "Installing venv for running notebooks..."
+    echo "Installing venv for running notebooks"
     python3 -m venv /root/venvs/notebook
     /root/venvs/notebook/bin/pip install jupyter ipykernel ipywidgets
     /root/venvs/notebook/bin/ipython kernel install --name=notebook
 fi
 
 if [ -z "${SIA_INSTALL_NO_CORE}" ]; then
-    echo "Installing SIA core..."
+    echo "Installing SIA core"
     python3 -m venv /root/venvs/sia
     /root/venvs/sia/bin/pip install -e /root/sia
 fi
 
 if [ -z "${SIA_INSTALL_NO_ITB}" ]; then
-    echo "Installing ITB tool..."
+    echo "Installing ITB tool"
     python3 -m venv /root/venvs/itb
     /root/venvs/itb/bin/pip install -e /root/sia/tools/itb
 fi
 
 if [ -z "${SIA_INSTALL_NO_MISTRAL_INFER}" ]; then
-    echo "Installing venv for mistral inference..."
+    echo "Installing venv for mistral inference"
     python3 -m venv /root/venvs/mistral_infer
     /root/venvs/mistral_infer/bin/pip install -e /root/sia/tools/mistral_infer
 fi
 
 if [ -z "${SIA_INSTALL_NO_GEMMA_INFER}" ]; then
-    echo "Installing venv for gemma inference..."
+    echo "Installing venv for gemma inference"
     python3 -m venv /root/venvs/gemma_infer
     /root/venvs/gemma_infer/bin/pip install -e /root/sia/tools/gemma_infer
 fi
 
-if [ -z "${SIA_INSTALL_NO_TRAIN}" ]; then
-    echo "Installing Train tool..."
-    python3 -m venv /root/venvs/train
-    /root/venvs/train/bin/pip install -e /root/sia/tools/train
-    /root/venvs/train/bin/ipython kernel install --name=train
-fi
+if [ -z "${SIA_INSTALL_NO_GEMMA_TRAIN}" ]; then
+    echo "Installing venv for gemma training"
+    python3 -m venv /root/venvs/gemma_train
+    /root/venvs/gemma_train/bin/pip install -e /root/sia/tools/gemma_train
+fi
+
+if ! grep -q "source /root/sia/scripts/add_venvs_to_path.sh" /root/.bashrc; then
+    echo 'source /root/sia/scripts/add_venvs_to_path.sh' >> /root/.bashrc
+fi
+
+if ! grep -q "source /root/sia/scripts/add_venvs_to_path.sh" /etc/profile; then
+    echo 'source /root/sia/scripts/add_venvs_to_path.sh' >> /etc/profile
+fi
+
+source /root/sia/scripts/add_venvs_to_path.sh

scripts/mock_llm.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+# Mock LLM engine for testing
+# This script simulates an LLM engine subprocess that responds to the three commands:
+# <token_limit/>, <token_count>...</token_count>, and <infer_xml>...</infer_xml>
+
+# Function to read XML input until a complete closing tag is found
+read_xml_input() {
+    local input=""
+    local line
+    local char_count=0
+    
+    # Read until we get a complete XML command
+    while IFS= read -r line; do
+        input="${input}${line}"
+        char_count=$((char_count + ${#line}))
+        
+        # Debug the actual content (first 30 chars)
+        
+        if [[ "$input" == *"<token_limit/>"* ]]; then
+            printf "1024" 
+            printf "\\004"  # EOT character (hex 04)
+            return
+        fi
+        
+        if [[ "$input" == *"<token_count>"*"</token_count>"* ]]; then
+            printf "405"
+            printf "\\004"  # EOT character (hex 04)
+            return
+        fi
+        
+        if [[ "$input" == *"<infer_xml>"*"</infer_xml>"* ]]; then
+            generate_response
+            return
+        fi
+        
+    done
+}
+
+# Function to generate a response token by token
+generate_response() {
+    printf "<"
+    sleep 0.3
+    
+    printf "reason"
+    sleep 0.3
+    
+    printf "ing"
+    sleep 0.3
+    
+    printf ">"
+    sleep 0.3
+    
+    printf "This"
+    sleep 0.3
+    
+    printf " is"
+    sleep 0.3
+    
+    printf " a"
+    sleep 0.3
+    
+    printf " test"
+    sleep 0.3
+    
+    printf " response."
+    sleep 0.3
+    
+    printf "</"
+    sleep 0.3
+    
+    printf "reason"
+    sleep 0.3
+    
+    printf "ing"
+    sleep 0.3
+    
+    printf ">"
+    sleep 0.3
+    
+    printf "\\004"
+}
+
+# Main loop - keep reading input and responding
+iteration=0
+while true; do
+    iteration=$((iteration + 1))
+    read_xml_input
+done

scripts/train.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+SIA_DIR="/root/sia"
+OUTPUT_DIR="${1:-/root/models/$(cd "$SIA_DIR" && git rev-parse HEAD)}"
+
+echo "Output dir: $OUTPUT_DIR"
+
+if [ -n "$(cd "$SIA_DIR" && git status --porcelain)" ]; then
+    echo "Uncommitted changes in SIA directory"
+    #exit 1
+fi
+
+mkdir -p "$OUTPUT_DIR"
+
+train_gemma --output-dir "$OUTPUT_DIR"