llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed context usage calculation

Browse Source
This commit is contained in:
Niels Geens
2024-11-14 18:23:33 +01:00
parent ede0a642d3
commit 4ce421bbce
13 changed files with 249 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
sia/__main__.py
View File

@@ -25,14 +25,6 @@ mimetypes.add_type("application/javascript", ".jsx")
mimetypes.add_type("text/javascript", ".js")
mimetypes.add_type("text/javascript", ".jsx")
class TestLLM:
def infer(self, prompt: str, context: str):
yield "<reasoning>"
time.sleep(2)
yield "test reasoning"
time.sleep(2)
yield "</reasoning>"
class Main:
@classmethod
async def create(cls, config: Config):
@@ -44,20 +36,24 @@ class Main:
match self._config.llm_engine:
case "local":
self._llm = LocalLlmEngine(self._config.model)
self._llm = LocalLlmEngine(
self._config.model,
self._config.temperature,
self._config.token_limit
)
case "hf":
self._llm = HfLlmEngine(
model_id=self._config.model,
api_token=self._config.api_token,
temperature=self._config.temperature
self._config.model,
self._config.temperature,
self._config.api_token,
)
case "openai":
self._llm = OpenAILlmEngine(
model=self._config.model,
api_key=self._config.api_token
self._config.model,
self._config.temperature,
self._config.api_token,
self._config.token_limit
)
case "test":
self._llm = TestLLM()
case _:
raise ValueError(f"Invalid LLM engine: {self._config.llm_engine}")
self._io_buffer = WebIOBuffer()
@@ -65,7 +61,7 @@ class Main:
system_prompt=self._system_prompt,
action_schema=self._action_schema,
working_memory=WorkingMemory(),
system_metrics=SystemMetrics(),
metrics=SystemMetrics(),
llm=self._llm,
validator=XMLValidator(self._action_schema),
parser=ResponseParser(self._io_buffer)