added typical_p

Author: yk

inference/text-client/text_client_utils.py

@@ -36,11 +36,8 @@ def send_message(self, message, model_id):
                 "content": message,
                 "work_parameters": {
                     "model_name": model_id,
-                    "top_k": 1024,
                     "top_p": 0.9,
-                    "temperature": 1.0,
                     "repetition_penalty": 1.2,
-                    "max_new_tokens": 1024,
                 },
             },
             headers=self.auth_headers,

inference/worker/interface.py

@@ -9,6 +9,7 @@ class GenerateStreamParameters(pydantic.BaseModel):
     do_sample: bool | None
     top_k: int | None
     top_p: float | None
+    typical_p: float | None
     temperature: float | None
     repetition_penalty: float | None
     seed: int | None
@@ -26,6 +27,7 @@ def from_work_parameters(params: inference.WorkParameters) -> "GenerateStreamPar
             do_sample=params.do_sample,
             top_k=params.top_k,
             top_p=params.top_p,
+            typical_p=params.typical_p,
             temperature=params.temperature,
             repetition_penalty=params.repetition_penalty,
             seed=params.seed,

oasst-shared/oasst_shared/schemas/inference.py

@@ -110,6 +110,7 @@ class WorkParametersInput(pydantic.BaseModel):
     model_name: str = DEFAULT_MODEL_NAME
     top_k: int | None = None
     top_p: float | None = None
+    typical_p: float | None = None
     temperature: float | None = None
     repetition_penalty: float | None = None
     max_new_tokens: int | None = None