Verified Commit e2e6efaf authored by Richard Glosner's avatar Richard Glosner
Browse files

add timeout with default value for LLM requests

parent 4f527d27

llm/apps.py

+1 −0
Original line number Diff line number Diff line
@@ -16,4 +16,5 @@ class LlmConfig(AppConfig): 
        client_module.LLM_CLIENT = client_module.OllamaLLM(

            model=settings.LLM_MODEL,

            llm_url=settings.LLM_URL,

            timeout=settings.LLM_TIMEOUT,

        )

llm/llm_classes.py

+3 −2
Original line number Diff line number Diff line
@@ -17,14 +17,15 @@ LLM_CLIENT: Optional[LLMInterface] = None 




class OllamaLLM(LLMInterface):

    def __init__(self, model: str, llm_url: str):

    def __init__(self, model: str, llm_url: str, timeout: float):

        self.model = model

        self.llm_url = llm_url

        self.timeout = timeout



    def query_llm(self, messages: List[LLMMessage]) -> str:

        payload = {"model": self.model, "stream": False}

        payload["messages"] = messages

        response = post(self.llm_url, json=payload)

        response = post(self.llm_url, json=payload, timeout=self.timeout)

        response.raise_for_status()

        return response.json()["message"]["content"]

ttxbackend/settings.py

+1 −0
Original line number Diff line number Diff line
@@ -354,3 +354,4 @@ OPENSEARCH_PASSWORD = os.environ.get("INJECT_OPENSEARCH_PASSWORD", "") 
LLM_URL = os.environ.get("INJECT_LLM_URL", "")

LLM_MODEL = os.environ.get("INJECT_LLM_MODEL", "")

LLM_API_KEY = os.environ.get("INJECT_LLM_API_KEY", "")

LLM_TIMEOUT = float(os.environ.get("INJECT_LLM_TIMEOUT", 25.0))