From b89cb9cfe1c1dccfdca7ab686025740ab3bd3372 Mon Sep 17 00:00:00 2001 From: Niels Geens Date: Wed, 7 Jan 2026 19:32:50 +0100 Subject: [PATCH] Switch to Vulkan --- Dockerfile | 16 +- scripts/build_and_upload_vulkan_libs.sh | 202 ++++++++++++++++++ scripts/local.sh | 1 - tools/gemma_infer/pyproject.toml | 2 +- .../src/gemma_infer/gemma_llm_engine.py | 4 +- tools/gemma_train/src/gemma_train/__main__.py | 19 +- 6 files changed, 225 insertions(+), 19 deletions(-) create mode 100644 scripts/build_and_upload_vulkan_libs.sh diff --git a/Dockerfile b/Dockerfile index b7895ec..c321422 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,10 +12,15 @@ RUN apt-get update && \ gnupg \ jq \ libcurl4-nss-dev \ + libvulkan1 \ + libvulkan-dev \ + mesa-vulkan-drivers \ python3-dev \ python3-venv \ tmux \ + unzip \ vim \ + vulkan-tools \ wget # Install chrome @@ -36,10 +41,13 @@ RUN curl https://sh.rustup.rs -sSf | bash -s -- -y ENV PATH="/root/.cargo/bin:${PATH}" # Install llama.cpp -RUN curl -O https://git.nielsgeens.be/api/packages/llm/generic/llama.cpp/b5269/llama.cpp.tar -RUN tar -xf /llama.cpp.tar -C /usr/local/bin --wildcards --no-anchored "llama-*" -RUN tar -xf /llama.cpp.tar -C /usr/local/lib --wildcards --no-anchored "*.so" -RUN rm llama.cpp.tar +RUN curl -o /tmp/vulkan-libs.tar https://git.nielsgeens.be/api/packages/llm/generic/llama-cpp-python/v0.3.16-vulkan/libs.tar +RUN mkdir -p /usr/local/lib/llama-cpp-python +RUN tar -xf /tmp/vulkan-libs.tar -C /usr/local/lib/llama-cpp-python +RUN cp -f /usr/local/lib/llama-cpp-python/*.so* /usr/local/lib/ +RUN rm /tmp/vulkan-libs.tar +# Set Vulkan environment to use CPU fallback (LavaPipe) +ENV VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/lvp_icd.x86_64.json # Create directory structure RUN mkdir -p \ diff --git a/scripts/build_and_upload_vulkan_libs.sh b/scripts/build_and_upload_vulkan_libs.sh new file mode 100644 index 0000000..438de5f --- /dev/null +++ b/scripts/build_and_upload_vulkan_libs.sh @@ -0,0 +1,202 @@ +#!/bin/bash +# Complete script to build llama.cpp with Vulkan support and upload to Gitea +# This script installs dependencies, builds libraries, packages them, and uploads +# +# Usage: +# export GITEA_TOKEN='your-token' # Or will prompt for credentials +# ./scripts/build_and_upload_vulkan_libs.sh + +set -e + +# Configuration +LLAMA_CPP_COMMIT="d3248d9b6557c75d59954c594bb53cf517591e91" # b6173 release +BUILD_DIR="/tmp/llama-cpp-build" +OUTPUT_DIR="/root/models" +PACKAGE_NAME="llama-cpp-python-v0.3.16-vulkan-libs.tar" +GITEA_URL="https://git.nielsgeens.be/api/packages/llm/generic/llama-cpp-python/v0.3.16-vulkan/libs.tar" + +echo "=========================================" +echo "Build & Upload Vulkan Libraries" +echo "=========================================" +echo "" +echo "This will:" +echo " 1. Install build dependencies (Vulkan SDK, glslc)" +echo " 2. Build llama.cpp with Vulkan support" +echo " 3. Package the libraries" +echo " 4. Upload to Gitea" +echo "" + +# ============================================================================= +# STEP 1: Install build dependencies +# ============================================================================= +echo "Step 1/5: Installing build dependencies..." +echo "" + +# Check if we have the newer Vulkan SDK +if ! dpkg -l | grep -q vulkan-sdk; then + echo "Installing LunarG Vulkan SDK..." + wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - + wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list \ + https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list + apt-get update + apt-get install -y vulkan-sdk + echo "✓ Vulkan SDK installed" +else + echo "✓ Vulkan SDK already installed" +fi + +# Check if glslc is available +if ! command -v glslc &> /dev/null; then + echo "Building shaderc for glslc..." + cd /tmp + git clone --depth 1 --branch v2024.3 https://github.com/google/shaderc.git + cd shaderc + python3 ./utils/git-sync-deps + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release -DSHADERC_SKIP_TESTS=ON .. + make -j$(nproc) glslc_exe + cp ./glslc/glslc /usr/local/bin/ + rm -rf /tmp/shaderc + echo "✓ glslc installed" +else + echo "✓ glslc already installed at $(which glslc)" +fi + +# ============================================================================= +# STEP 2: Build llama.cpp with Vulkan +# ============================================================================= +mkdir -p "$BUILD_DIR" +mkdir -p "$OUTPUT_DIR" +cd "$BUILD_DIR" + +echo "" +echo "Step 2/5: Building llama.cpp with Vulkan..." +echo "" + +if [ ! -d "llama.cpp" ]; then + git clone https://github.com/ggml-org/llama.cpp.git +fi + +cd llama.cpp +git fetch origin "$LLAMA_CPP_COMMIT" +git checkout "$LLAMA_CPP_COMMIT" + +mkdir -p build +cd build + +# Set Vulkan environment to use CPU fallback (LavaPipe) +export VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/lvp_icd.x86_64.json + +cmake .. \ + -DGGML_VULKAN=ON \ + -DGGML_VULKAN_VALIDATE=OFF \ + -DGGML_VULKAN_RUN_TESTS=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON + +echo "" +echo "Building (this takes ~30-60 minutes)..." +echo "Started at: $(date)" +make -j$(nproc) +echo "Completed at: $(date)" + +# ============================================================================= +# STEP 3: Package the libraries +# ============================================================================= +echo "" +echo "Step 3/5: Packaging libraries..." +echo "" + +STAGING_DIR="/tmp/llama-cpp-libs-staging" +mkdir -p "$STAGING_DIR" + +# Copy all shared libraries +find "$BUILD_DIR/llama.cpp/build" -name "*.so*" -type f -exec cp -v {} "$STAGING_DIR/" \; + +echo "" +echo "Libraries to package:" +ls -lh "$STAGING_DIR/" + +# Verify Vulkan library is present +if [ -f "$STAGING_DIR/libggml-vulkan.so" ]; then + VULKAN_SIZE=$(ls -lh "$STAGING_DIR/libggml-vulkan.so" | awk '{print $5}') + echo "" + echo "✓ Vulkan backend library found! (size: $VULKAN_SIZE)" +else + echo "" + echo "✗ ERROR: Vulkan backend library NOT found!" + exit 1 +fi + +# Create tar package +cd "$STAGING_DIR" +tar -cvf "$OUTPUT_DIR/$PACKAGE_NAME" *.so* + +echo "" +echo "Package created:" +ls -lh "$OUTPUT_DIR/$PACKAGE_NAME" + +# Cleanup build directory +rm -rf "$BUILD_DIR" +rm -rf "$STAGING_DIR" + +# ============================================================================= +# STEP 4: Verify package +# ============================================================================= +echo "" +echo "Step 4/5: Verifying package..." +echo "" +tar -tf "$OUTPUT_DIR/$PACKAGE_NAME" + +# ============================================================================= +# STEP 5: Upload to Gitea +# ============================================================================= +echo "" +echo "Step 5/5: Uploading to Gitea..." +echo "" + +PACKAGE_PATH="$OUTPUT_DIR/$PACKAGE_NAME" +PACKAGE_SIZE=$(ls -lh "$PACKAGE_PATH" | awk '{print $5}') + +echo "Package: $PACKAGE_PATH" +echo "Size: $PACKAGE_SIZE" +echo "URL: $GITEA_URL" +echo "" + +# Check for authentication +if [ -z "$GITEA_TOKEN" ]; then + echo "GITEA_TOKEN not set. Please provide credentials:" + read -p "Username: " GITEA_USER + read -s -p "Password: " GITEA_PASS + echo "" + + # Upload with username/password + curl -X PUT \ + --upload-file "$PACKAGE_PATH" \ + --user "$GITEA_USER:$GITEA_PASS" \ + "$GITEA_URL" \ + --fail \ + --show-error \ + --progress-bar +else + # Upload with token + curl -X PUT \ + --upload-file "$PACKAGE_PATH" \ + -H "Authorization: token $GITEA_TOKEN" \ + "$GITEA_URL" \ + --fail \ + --show-error \ + --progress-bar +fi + +echo "" +echo "=========================================" +echo "SUCCESS!" +echo "=========================================" +echo "" +echo "Package uploaded to:" +echo " $GITEA_URL" +echo "" +echo "The Dockerfile will automatically download this file" +echo "during container build." +echo "=========================================" diff --git a/scripts/local.sh b/scripts/local.sh index 63279bf..0407e54 100755 --- a/scripts/local.sh +++ b/scripts/local.sh @@ -25,7 +25,6 @@ docker run \ --init \ --rm \ -ti \ - --gpus=all \ -p 8080:8080 \ -p 8000:8000 \ --env-file .env \ diff --git a/tools/gemma_infer/pyproject.toml b/tools/gemma_infer/pyproject.toml index 54ea708..3a1cea3 100644 --- a/tools/gemma_infer/pyproject.toml +++ b/tools/gemma_infer/pyproject.toml @@ -9,7 +9,7 @@ requires-python = ">=3.8" dependencies = [ "blobfile>=3.0.0", - "llama-cpp-python @ git+https://github.com/abetlen/llama-cpp-python.git#egg=llama-cpp-python&env=CMAKE_ARGS=-DLLAMA_BUILD=OFF", + "llama-cpp-python @ git+https://github.com/abetlen/llama-cpp-python.git@v0.3.16#egg=llama-cpp-python&env=CMAKE_ARGS=-DLLAMA_BUILD=OFF", "llm_engine_utils @ file:///root/sia/lib/llm_engine_utils", "protobuf>=6.0.0", "python-dotenv>=1.0.0", diff --git a/tools/gemma_infer/src/gemma_infer/gemma_llm_engine.py b/tools/gemma_infer/src/gemma_infer/gemma_llm_engine.py index 5e85458..d06fd11 100644 --- a/tools/gemma_infer/src/gemma_infer/gemma_llm_engine.py +++ b/tools/gemma_infer/src/gemma_infer/gemma_llm_engine.py @@ -3,13 +3,15 @@ os.environ["LLAMA_CPP_LIB_PATH"] = "/usr/local/lib" os.environ["LD_LIBRARY_PATH"] += ":/usr/local/lib" os.chdir("/usr/local/lib") -from llama_cpp import Llama, LogitsProcessorList +from llama_cpp import Llama, LogitsProcessorList, llama_cpp from llm_engine_utils import LlmEngine from pathlib import Path from transformers import AutoTokenizer from typing import Iterator from xml_schema_validator import LlamaCppLogitsProcessor +llama_cpp._lib.ggml_backend_load_all() + class GemmaLlmEngine(LlmEngine): def __init__( self, diff --git a/tools/gemma_train/src/gemma_train/__main__.py b/tools/gemma_train/src/gemma_train/__main__.py index 170d9af..cc92719 100644 --- a/tools/gemma_train/src/gemma_train/__main__.py +++ b/tools/gemma_train/src/gemma_train/__main__.py @@ -1,7 +1,7 @@ from llm_engine_utils.dataset import Dataset from pathlib import Path from peft import LoraConfig, AutoPeftModelForCausalLM -from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, TrainingArguments +from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments from trl import SFTTrainer import os import sys @@ -23,19 +23,13 @@ def train(config: Config): ) tokenizer.save_pretrained(config.output_dir/"tokenizer") - bnb_config = BitsAndBytesConfig( - load_in_4bit=True, - bnb_4bit_use_double_quant=True, - bnb_4bit_quant_type="nf4", - bnb_4bit_compute_dtype=torch.bfloat16 - ) - model = AutoModelForCausalLM.from_pretrained( config.model, - quantization_config=bnb_config, - device_map="auto", + torch_dtype=torch.float32, + device_map="cpu", token=config.api_key, attn_implementation='eager', + low_cpu_mem_usage=True, ) dataset = Dataset(config.config_path) @@ -57,14 +51,15 @@ def train(config: Config): warmup_steps=1, max_steps=1, learning_rate=1e-3, - fp16=True, + fp16=False, logging_steps=1, save_strategy="steps", save_steps=1, output_dir=config.output_dir/"lora", - optim="paged_adamw_8bit", + optim="adamw_torch", seed=42, group_by_length=True, + use_cpu=True, ) trainer = SFTTrainer(