Gemma inference with llama.cpp and logits processor

Dockerfile

@@ -1,18 +1,24 @@
 FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base
+
+# Install base packages
 RUN apt-get update && \
     apt-get upgrade -y && \
     apt install -y \
     build-essential \
+    cmake \
+    cuda-toolkit \
     curl \
     git \
     gnupg \
     jq \
+    libcurl4-nss-dev \
     python3-dev \
     python3-venv \
     tmux \
     vim \
     wget
 
+# Install chrome
 RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor -o /etc/apt/trusted.gpg.d/google.gpg
 RUN echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list
 RUN apt-get update && \
@@ -21,9 +27,16 @@ RUN apt-get update && \
 
 RUN rm -rf /var/lib/apt/lists/*
 
+# Install Rust
 RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
 ENV PATH="/root/.cargo/bin:${PATH}"
 
+# Install llama.cpp
+RUN curl -O https://git.nielsgeens.be/api/packages/llm/generic/llama.cpp/b5269/llama.cpp.tar
+RUN tar -xf /llama.cpp.tar -C /usr/local/bin --wildcards --no-anchored "llama-*"
+RUN tar -xf /llama.cpp.tar -C /usr/local/lib --wildcards --no-anchored "*.so"
+RUN rm llama.cpp.tar
+
 # Create directory structure
 RUN mkdir -p \
     /root/sia \