Switch to Vulkan

Dockerfile

@@ -12,10 +12,15 @@ RUN apt-get update && \
     gnupg \
     jq \
     libcurl4-nss-dev \
+    libvulkan1 \
+    libvulkan-dev \
+    mesa-vulkan-drivers \
     python3-dev \
     python3-venv \
     tmux \
+    unzip \
     vim \
+    vulkan-tools \
     wget
 
 # Install chrome
@@ -36,10 +41,13 @@ RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
 ENV PATH="/root/.cargo/bin:${PATH}"
 
 # Install llama.cpp
-RUN curl -O https://git.nielsgeens.be/api/packages/llm/generic/llama.cpp/b5269/llama.cpp.tar
+RUN curl -o /tmp/vulkan-libs.tar https://git.nielsgeens.be/api/packages/llm/generic/llama-cpp-python/v0.3.16-vulkan/libs.tar
-RUN tar -xf /llama.cpp.tar -C /usr/local/bin --wildcards --no-anchored "llama-*"
+RUN mkdir -p /usr/local/lib/llama-cpp-python
-RUN tar -xf /llama.cpp.tar -C /usr/local/lib --wildcards --no-anchored "*.so"
+RUN tar -xf /tmp/vulkan-libs.tar -C /usr/local/lib/llama-cpp-python
-RUN rm llama.cpp.tar
+RUN cp -f /usr/local/lib/llama-cpp-python/*.so* /usr/local/lib/
+RUN rm /tmp/vulkan-libs.tar
+# Set Vulkan environment to use CPU fallback (LavaPipe)
+ENV VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/lvp_icd.x86_64.json
 
 # Create directory structure
 RUN mkdir -p \

scripts/build_and_upload_vulkan_libs.sh

@@ -0,0 +1,202 @@
+#!/bin/bash
+# Complete script to build llama.cpp with Vulkan support and upload to Gitea
+# This script installs dependencies, builds libraries, packages them, and uploads
+#
+# Usage:
+#   export GITEA_TOKEN='your-token'  # Or will prompt for credentials
+#   ./scripts/build_and_upload_vulkan_libs.sh
+
+set -e
+
+# Configuration
+LLAMA_CPP_COMMIT="d3248d9b6557c75d59954c594bb53cf517591e91"  # b6173 release
+BUILD_DIR="/tmp/llama-cpp-build"
+OUTPUT_DIR="/root/models"
+PACKAGE_NAME="llama-cpp-python-v0.3.16-vulkan-libs.tar"
+GITEA_URL="https://git.nielsgeens.be/api/packages/llm/generic/llama-cpp-python/v0.3.16-vulkan/libs.tar"
+
+echo "========================================="
+echo "Build & Upload Vulkan Libraries"
+echo "========================================="
+echo ""
+echo "This will:"
+echo "  1. Install build dependencies (Vulkan SDK, glslc)"
+echo "  2. Build llama.cpp with Vulkan support"
+echo "  3. Package the libraries"
+echo "  4. Upload to Gitea"
+echo ""
+
+# =============================================================================
+# STEP 1: Install build dependencies
+# =============================================================================
+echo "Step 1/5: Installing build dependencies..."
+echo ""
+
+# Check if we have the newer Vulkan SDK
+if ! dpkg -l | grep -q vulkan-sdk; then
+    echo "Installing LunarG Vulkan SDK..."
+    wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add -
+    wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list \
+        https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+    apt-get update
+    apt-get install -y vulkan-sdk
+    echo "✓ Vulkan SDK installed"
+else
+    echo "✓ Vulkan SDK already installed"
+fi
+
+# Check if glslc is available
+if ! command -v glslc &> /dev/null; then
+    echo "Building shaderc for glslc..."
+    cd /tmp
+    git clone --depth 1 --branch v2024.3 https://github.com/google/shaderc.git
+    cd shaderc
+    python3 ./utils/git-sync-deps
+    mkdir build && cd build
+    cmake -DCMAKE_BUILD_TYPE=Release -DSHADERC_SKIP_TESTS=ON ..
+    make -j$(nproc) glslc_exe
+    cp ./glslc/glslc /usr/local/bin/
+    rm -rf /tmp/shaderc
+    echo "✓ glslc installed"
+else
+    echo "✓ glslc already installed at $(which glslc)"
+fi
+
+# =============================================================================
+# STEP 2: Build llama.cpp with Vulkan
+# =============================================================================
+mkdir -p "$BUILD_DIR"
+mkdir -p "$OUTPUT_DIR"
+cd "$BUILD_DIR"
+
+echo ""
+echo "Step 2/5: Building llama.cpp with Vulkan..."
+echo ""
+
+if [ ! -d "llama.cpp" ]; then
+    git clone https://github.com/ggml-org/llama.cpp.git
+fi
+
+cd llama.cpp
+git fetch origin "$LLAMA_CPP_COMMIT"
+git checkout "$LLAMA_CPP_COMMIT"
+
+mkdir -p build
+cd build
+
+# Set Vulkan environment to use CPU fallback (LavaPipe)
+export VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/lvp_icd.x86_64.json
+
+cmake .. \
+    -DGGML_VULKAN=ON \
+    -DGGML_VULKAN_VALIDATE=OFF \
+    -DGGML_VULKAN_RUN_TESTS=OFF \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DBUILD_SHARED_LIBS=ON
+
+echo ""
+echo "Building (this takes ~30-60 minutes)..."
+echo "Started at: $(date)"
+make -j$(nproc)
+echo "Completed at: $(date)"
+
+# =============================================================================
+# STEP 3: Package the libraries
+# =============================================================================
+echo ""
+echo "Step 3/5: Packaging libraries..."
+echo ""
+
+STAGING_DIR="/tmp/llama-cpp-libs-staging"
+mkdir -p "$STAGING_DIR"
+
+# Copy all shared libraries
+find "$BUILD_DIR/llama.cpp/build" -name "*.so*" -type f -exec cp -v {} "$STAGING_DIR/" \;
+
+echo ""
+echo "Libraries to package:"
+ls -lh "$STAGING_DIR/"
+
+# Verify Vulkan library is present
+if [ -f "$STAGING_DIR/libggml-vulkan.so" ]; then
+    VULKAN_SIZE=$(ls -lh "$STAGING_DIR/libggml-vulkan.so" | awk '{print $5}')
+    echo ""
+    echo "✓ Vulkan backend library found! (size: $VULKAN_SIZE)"
+else
+    echo ""
+    echo "✗ ERROR: Vulkan backend library NOT found!"
+    exit 1
+fi
+
+# Create tar package
+cd "$STAGING_DIR"
+tar -cvf "$OUTPUT_DIR/$PACKAGE_NAME" *.so*
+
+echo ""
+echo "Package created:"
+ls -lh "$OUTPUT_DIR/$PACKAGE_NAME"
+
+# Cleanup build directory
+rm -rf "$BUILD_DIR"
+rm -rf "$STAGING_DIR"
+
+# =============================================================================
+# STEP 4: Verify package
+# =============================================================================
+echo ""
+echo "Step 4/5: Verifying package..."
+echo ""
+tar -tf "$OUTPUT_DIR/$PACKAGE_NAME"
+
+# =============================================================================
+# STEP 5: Upload to Gitea
+# =============================================================================
+echo ""
+echo "Step 5/5: Uploading to Gitea..."
+echo ""
+
+PACKAGE_PATH="$OUTPUT_DIR/$PACKAGE_NAME"
+PACKAGE_SIZE=$(ls -lh "$PACKAGE_PATH" | awk '{print $5}')
+
+echo "Package: $PACKAGE_PATH"
+echo "Size: $PACKAGE_SIZE"
+echo "URL: $GITEA_URL"
+echo ""
+
+# Check for authentication
+if [ -z "$GITEA_TOKEN" ]; then
+    echo "GITEA_TOKEN not set. Please provide credentials:"
+    read -p "Username: " GITEA_USER
+    read -s -p "Password: " GITEA_PASS
+    echo ""
+
+    # Upload with username/password
+    curl -X PUT \
+        --upload-file "$PACKAGE_PATH" \
+        --user "$GITEA_USER:$GITEA_PASS" \
+        "$GITEA_URL" \
+        --fail \
+        --show-error \
+        --progress-bar
+else
+    # Upload with token
+    curl -X PUT \
+        --upload-file "$PACKAGE_PATH" \
+        -H "Authorization: token $GITEA_TOKEN" \
+        "$GITEA_URL" \
+        --fail \
+        --show-error \
+        --progress-bar
+fi
+
+echo ""
+echo "========================================="
+echo "SUCCESS!"
+echo "========================================="
+echo ""
+echo "Package uploaded to:"
+echo "  $GITEA_URL"
+echo ""
+echo "The Dockerfile will automatically download this file"
+echo "during container build."
+echo "========================================="

scripts/local.sh

@@ -25,7 +25,6 @@ docker run \
     --init \
     --rm \
     -ti \
-    --gpus=all \
     -p 8080:8080 \
     -p 8000:8000 \
     --env-file .env \

tools/gemma_infer/pyproject.toml

@@ -9,7 +9,7 @@ requires-python = ">=3.8"
 
 dependencies = [
     "blobfile>=3.0.0",
-    "llama-cpp-python @ git+https://github.com/abetlen/llama-cpp-python.git#egg=llama-cpp-python&env=CMAKE_ARGS=-DLLAMA_BUILD=OFF",
+    "llama-cpp-python @ git+https://github.com/abetlen/llama-cpp-python.git@v0.3.16#egg=llama-cpp-python&env=CMAKE_ARGS=-DLLAMA_BUILD=OFF",
     "llm_engine_utils @ file:///root/sia/lib/llm_engine_utils",
     "protobuf>=6.0.0",
     "python-dotenv>=1.0.0",

tools/gemma_infer/src/gemma_infer/gemma_llm_engine.py

@@ -3,13 +3,15 @@ os.environ["LLAMA_CPP_LIB_PATH"] = "/usr/local/lib"
 os.environ["LD_LIBRARY_PATH"] += ":/usr/local/lib"
 os.chdir("/usr/local/lib")
 
-from llama_cpp import Llama, LogitsProcessorList
+from llama_cpp import Llama, LogitsProcessorList, llama_cpp
 from llm_engine_utils import LlmEngine
 from pathlib import Path
 from transformers import AutoTokenizer
 from typing import Iterator
 from xml_schema_validator import LlamaCppLogitsProcessor
 
+llama_cpp._lib.ggml_backend_load_all()
+
 class GemmaLlmEngine(LlmEngine):
     def __init__(
             self,

tools/gemma_train/src/gemma_train/__main__.py

@@ -1,7 +1,7 @@
 from llm_engine_utils.dataset import Dataset
 from pathlib import Path
 from peft import LoraConfig, AutoPeftModelForCausalLM
-from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, TrainingArguments
+from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments
 from trl import SFTTrainer
 import os
 import sys
@@ -23,19 +23,13 @@ def train(config: Config):
     )
     tokenizer.save_pretrained(config.output_dir/"tokenizer")
 
-    bnb_config = BitsAndBytesConfig(
-        load_in_4bit=True,
-        bnb_4bit_use_double_quant=True,
-        bnb_4bit_quant_type="nf4",
-        bnb_4bit_compute_dtype=torch.bfloat16
-    )
-
     model = AutoModelForCausalLM.from_pretrained(
         config.model,
-        quantization_config=bnb_config,
+        torch_dtype=torch.float32,
-        device_map="auto",
+        device_map="cpu",
         token=config.api_key,
         attn_implementation='eager',
+        low_cpu_mem_usage=True,
     )
 
     dataset = Dataset(config.config_path)
@@ -57,14 +51,15 @@ def train(config: Config):
         warmup_steps=1,
         max_steps=1,
         learning_rate=1e-3,
-        fp16=True,
+        fp16=False,
         logging_steps=1,
         save_strategy="steps",
         save_steps=1,
         output_dir=config.output_dir/"lora",
-        optim="paged_adamw_8bit",
+        optim="adamw_torch",
         seed=42,
         group_by_length=True,
+        use_cpu=True,
     )
 
     trainer = SFTTrainer(