Add timesout for each chat

Author: ZHallen122

llm-server/src/llm-provider.ts

@@ -104,16 +104,29 @@ export class LLMProvider {
     return model;
   }
 
-  async chat(input: MessageInput): Promise<string> {
+  async chat(input: MessageInput, timeoutMs: number): Promise<string> {
     try {
       const model = this.getModelInstance(input.model);
-      const completion = await model.chat(input.messages);
-      return completion.choices[0].message.content || '';
+  
+      // Set a timeout dynamically based on the provided value
+      const timeoutPromise = new Promise<string>((_, reject) =>
+        setTimeout(() => reject(new Error('Chat request timed out')), timeoutMs),
+      );
+  
+      // Race between the actual model call and the timeout
+      const completion = await Promise.race([
+        model.chat(input.messages),
+        timeoutPromise,
+      ]);
+  
+      return (completion as any).choices[0].message.content || '';
     } catch (error) {
-      this.logger.error('Error in chat:', error);
+      this.logger.error(`Error in chat (Timeout: ${timeoutMs}ms):`, error);
       throw error;
     }
   }
+  
+  
 
   async *chatStream(
     input: MessageInput,

llm-server/src/main.ts

@@ -137,7 +137,8 @@ export class App {
         }
       } else {
         // Handle regular response
-        const response = await this.llmProvider.chat(input);
+        // TODO make it to dynamic Now is 200 second by defult.
+        const response = await this.llmProvider.chat(input, 200000);
         res.json({
           model: input.model,
           choices: [

llm-server/src/model/remote-model-instance.ts

@@ -73,7 +73,6 @@ export class RemoteOpenAIModelEngine implements ModelInstance {
         return await this.client.chat.completions.create({
           model: this.config.model,
           messages,
-          temperature: 1, // Default to 0.7 if not specified
         });
       });