Gemma inference with llama.cpp and logits processor

scripts/bootstrap.sh

@@ -27,8 +27,11 @@ fi
 apt-get update
 apt-get install -y \
     build-essential \
+    cmake \
+    cuda-toolkit \
     git \
     gnupg \
+    libcurl4-nss-dev \
     python3-dev \
     python3-venv \
     vim \
@@ -59,6 +62,13 @@ if [ -z "${SIA_INSTALL_NO_CORE}" ]; then
     nvm install node
 fi
 
+# Install llama.cpp
+cd /
+curl -O https://git.nielsgeens.be/api/packages/llm/generic/llama.cpp/b5269/llama.cpp.tar
+tar -xf /llama.cpp.tar -C /usr/local/bin --wildcards --no-anchored "llama-*"
+tar -xf /llama.cpp.tar -C /usr/local/lib --wildcards --no-anchored "*.so"
+rm llama.cpp.tar
+
 # Create directory structure
 echo "Creating directory structure..."
 mkdir -p "/root/data/iterations"

scripts/local.sh

@@ -27,6 +27,7 @@ docker run \
     -ti \
     --gpus=all \
     -p 8080:8080 \
+    -p 8000:8000 \
     --env-file .env \
     -v /$(pwd)/models/:/root/models/ \
     -v /$(pwd)/iterations/:/root/data/iterations/ \