Switch to Vulkan

tools/gemma_infer/pyproject.toml

@@ -9,7 +9,7 @@ requires-python = ">=3.8"
 
 dependencies = [
     "blobfile>=3.0.0",
-    "llama-cpp-python @ git+https://github.com/abetlen/llama-cpp-python.git#egg=llama-cpp-python&env=CMAKE_ARGS=-DLLAMA_BUILD=OFF",
+    "llama-cpp-python @ git+https://github.com/abetlen/llama-cpp-python.git@v0.3.16#egg=llama-cpp-python&env=CMAKE_ARGS=-DLLAMA_BUILD=OFF",
     "llm_engine_utils @ file:///root/sia/lib/llm_engine_utils",
     "protobuf>=6.0.0",
     "python-dotenv>=1.0.0",

tools/gemma_infer/src/gemma_infer/gemma_llm_engine.py

@@ -3,13 +3,15 @@ os.environ["LLAMA_CPP_LIB_PATH"] = "/usr/local/lib"
 os.environ["LD_LIBRARY_PATH"] += ":/usr/local/lib"
 os.chdir("/usr/local/lib")
 
-from llama_cpp import Llama, LogitsProcessorList
+from llama_cpp import Llama, LogitsProcessorList, llama_cpp
 from llm_engine_utils import LlmEngine
 from pathlib import Path
 from transformers import AutoTokenizer
 from typing import Iterator
 from xml_schema_validator import LlamaCppLogitsProcessor
 
+llama_cpp._lib.ggml_backend_load_all()
+
 class GemmaLlmEngine(LlmEngine):
     def __init__(
             self,