Fixed context usage calculation

2024-11-14 18:23:33 +01:00
parent ede0a642d3
commit 4ce421bbce
13 changed files with 249 additions and 119 deletions
--- a/sia/llm_engine.py
+++ b/sia/llm_engine.py
@@ -2,7 +2,14 @@ from typing import Iterator
 from abc import ABC, abstractmethod

 class LlmEngine(ABC):
-
    @abstractmethod
    def infer(self, system_prompt: str, main_context: str) -> Iterator[str]:
+        pass
+
+    @abstractmethod
+    def token_count(self, system_prompt: str, main_context: str) -> int:
+        pass
+
+    @abstractmethod
+    def token_limit(self) -> int:
        pass