[feat] The asr backend supports model switching.

Author: LittleMouse

api_server.py

@@ -61,6 +61,7 @@ class ModelDispatcher:
     def __init__(self):
         self.backends = {}
         self.llm_models = set()
+        self.asr_models = set()
         self.lock = asyncio.Lock()
 
     async def get_backend(self, model_name):
@@ -85,7 +86,14 @@ async def get_backend(self, model_name):
                 elif model_config["type"] == "tts":
                     self.backends[model_name] = TtsClientBackend(model_config)
                 elif model_config["type"] == "asr":
+                    if model_name not in self.asr_models:
+                        for old_model_name in list(self.asr_models):
+                            old_instance = self.backends.pop(old_model_name, None)
+                            if old_instance:
+                                await old_instance.close()
+                        self.asr_models.clear()
                     self.backends[model_name] = ASRClientBackend(model_config)
+                    self.asr_models.add(model_name)
                 else:
                     return None
             return self.backends.get(model_name)

backend/asr_client_backend.py

@@ -81,6 +81,17 @@ async def _release_client(self, client):
             self._client_pool.append(client)
             self.logger.debug(f"Returned client to pool | ID:{id(client)}")
 
+    async def close(self):
+        for task in self._active_tasks:
+            task.cancel()
+        if self._active_tasks:
+            await asyncio.wait(self._active_tasks, timeout=2)
+        for client in self._client_pool:
+            client.exit()
+        self._client_pool.clear()
+        self._active_clients.clear()
+        self._inference_executor.shutdown(wait=False)
+
     async def _inference(self, client, audio_b64: str):
         loop = asyncio.get_event_loop()
         for chunk in await loop.run_in_executor(