Michael Hermann
2bb70ae0ed
Historie 1949–2022 bleibt in lawgit, bis der GPU-Lauf endet; Schnitt ohne Lücke. Co-authored-by: Cursor <cursoragent@cursor.com>
129 lines
4.1 KiB
Python
129 lines
4.1 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
OpenAI-kompatibler Client für RunPod (vLLM) oder andere Endpunkte.
|
|
|
|
Nur aufrufen mit `parse_bgbl.py --llm` / `sync.py --llm`. Standardläufe
|
|
bleiben heuristisch und kosten nichts.
|
|
|
|
Umgebungsvariablen (siehe .env.example):
|
|
LLM_BASE_URL z.B. https://<pod-id>-8000.proxy.runpod.net/v1
|
|
LLM_API_KEY RunPod-API-Key oder "not-needed"
|
|
LLM_MODEL z.B. Qwen/Qwen2.5-72B-Instruct-AWQ
|
|
LLM_TIMEOUT Sekunden (Standard 180)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import requests
|
|
|
|
_ENV_LOADED = False
|
|
|
|
|
|
def load_env(path: str | Path = ".env") -> None:
|
|
global _ENV_LOADED
|
|
if _ENV_LOADED:
|
|
return
|
|
_ENV_LOADED = True
|
|
env_file = Path(path)
|
|
if not env_file.exists():
|
|
return
|
|
for line in env_file.read_text(encoding="utf-8").splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, _, value = line.partition("=")
|
|
value = value.strip().strip('"').strip("'")
|
|
if value:
|
|
os.environ.setdefault(key.strip(), value)
|
|
|
|
|
|
def llm_configured() -> bool:
|
|
load_env()
|
|
return bool(os.environ.get("LLM_BASE_URL") or os.environ.get("RUNPOD_BASE_URL"))
|
|
|
|
|
|
def _endpoint() -> tuple[str, str, str]:
|
|
load_env()
|
|
base = os.environ.get("LLM_BASE_URL") or os.environ.get("RUNPOD_BASE_URL") or ""
|
|
base = base.rstrip("/")
|
|
if not base.endswith("/v1"):
|
|
base = base + "/v1"
|
|
key = os.environ.get("LLM_API_KEY") or os.environ.get("RUNPOD_API_KEY") or "not-needed"
|
|
model = os.environ.get("LLM_MODEL") or "Qwen/Qwen2.5-72B-Instruct-AWQ"
|
|
return base, key, model
|
|
|
|
|
|
def _lies_strom(response: requests.Response) -> str:
|
|
stuecke: list[str] = []
|
|
for zeile in response.iter_lines(decode_unicode=True):
|
|
if not zeile or not zeile.startswith("data:"):
|
|
continue
|
|
rest = zeile[5:].strip()
|
|
if rest == "[DONE]":
|
|
break
|
|
try:
|
|
happen = json.loads(rest)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
for wahl in happen.get("choices") or []:
|
|
inhalt = (wahl.get("delta") or {}).get("content")
|
|
if inhalt:
|
|
stuecke.append(inhalt)
|
|
if not stuecke:
|
|
raise ValueError("leere Antwort im Datenstrom")
|
|
return "".join(stuecke)
|
|
|
|
|
|
def chat(messages: list[dict[str, Any]], *, json_mode: bool = True, max_tokens: int = 4096) -> str:
|
|
base, key, model = _endpoint()
|
|
if not base.startswith("http"):
|
|
raise RuntimeError("LLM_BASE_URL / RUNPOD_BASE_URL ist nicht gesetzt")
|
|
|
|
timeout = int(os.environ.get("LLM_TIMEOUT", "180"))
|
|
versuche = int(os.environ.get("LLM_RETRIES", "3"))
|
|
payload: dict[str, Any] = {
|
|
"model": model,
|
|
"messages": messages,
|
|
"temperature": 0.1,
|
|
"max_tokens": max_tokens,
|
|
"stream": True,
|
|
}
|
|
if json_mode:
|
|
payload["response_format"] = {"type": "json_object"}
|
|
|
|
letzter: Exception | None = None
|
|
for versuch in range(1, versuche + 1):
|
|
try:
|
|
with requests.post(
|
|
f"{base}/chat/completions",
|
|
headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
|
|
json=payload,
|
|
timeout=timeout,
|
|
stream=True,
|
|
) as r:
|
|
if r.status_code >= 500 or r.status_code == 429:
|
|
raise requests.HTTPError(f"HTTP {r.status_code}", response=r)
|
|
r.raise_for_status()
|
|
return _lies_strom(r)
|
|
except (requests.RequestException, KeyError, ValueError) as exc:
|
|
letzter = exc
|
|
if versuch < versuche:
|
|
time.sleep(min(2 ** versuch * 5, 40))
|
|
raise RuntimeError(f"LLM nach {versuche} Versuchen nicht erreichbar: {letzter}")
|
|
|
|
|
|
def chat_json(messages: list[dict[str, Any]], **kwargs) -> dict:
|
|
raw = chat(messages, json_mode=True, **kwargs)
|
|
raw = raw.strip()
|
|
if raw.startswith("```"):
|
|
raw = raw.split("\n", 1)[-1]
|
|
if raw.endswith("```"):
|
|
raw = raw[: raw.rfind("```")]
|
|
return json.loads(raw)
|