Allow stopping of inference

2024-11-30 12:44:13 +01:00
parent a0353d0d49
commit e71bd7e9eb
15 changed files with 139 additions and 69 deletions
--- a/sia/config.py
+++ b/sia/config.py
@@ -82,6 +82,12 @@ class Config:
            default=int(os.getenv('SIA_LOCAL_TOKEN_LIMIT', '2048')),
            help='Local LLM token limit (env: SIA_LOCAL_TOKEN_LIMIT)'
        )
+        parser.add_argument(
+            '--local-api-key',
+            type=str,
+            default=os.getenv('SIA_LOCAL_API_KEY'),
+            help='API key for local models (env: SIA_LOCAL_API_KEY)'
+        )

        # OpenAI configuration
        parser.add_argument(
@@ -233,6 +239,10 @@ class Config:
    @property
    def local_token_limit(self) -> int:
        return self.args.local_token_limit
+    
+    @property
+    def local_api_key(self) -> Optional[str]:
+        return self.args.local_api_key

    # OpenAI properties
    @property