Public/owi-litellm-tool
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(core): consolidate into Triad-Pattern (Status, Usage, History) for SLM reliability

Browse Source
This commit is contained in:
vai committed 2026-06-27 09:46:02 +00:00
1 parent 70d46c64dd
commit e60c74c182
2 files changed
+113 -173
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

litellm_dashboard.py
+87 -137
View File
@@ -2,10 +2,10 @@
title: LiteLLM Dashboard
author: Kagamine Rin & VAI
author_url: https://rin.captcouch.xyz
description: Hardcore LiteLLM observability — health, cost breakdown, and request history with Dynamic FinOps Audit.
description: Consolidated Hardcore LiteLLM observability — status, usage, and history.
required_open_webui_version: 0.4.0
requirements: requests
version: 1.4.0
version: 1.5.0
licence: MIT
"""
@@ -14,7 +14,7 @@ import json
from datetime import datetime, timedelta
from zoneinfo import ZoneInfo
from pydantic import BaseModel, Field
from typing import Optional, List, Dict, Any
from typing import Optional, List, Dict, Any, Literal
class Tools:
@@ -26,7 +26,7 @@ class Tools:
"""LiteLLM proxy connection settings."""
litellm_url: str = Field(
default="http://localhost:4000",
description="LiteLLM proxy base URL (e.g. http://localhost:4000)",
description="LiteLLM proxy base URL",
)
litellm_master_key: str = Field(
default="",
@@ -46,20 +46,6 @@ class Tools:
description="When True, attempts to fetch live pricing from /v1/model/info for the Audit Engine.",
)
# Fallback registry for 2026 standard tier pricing
MODEL_PRICING_FALLBACK = {
"gemini/gemini-3.5-flash": {
"input": 1.50 / 1_000_000,
"input_cached": 0.15 / 1_000_000,
"output": 9.00 / 1_000_000,
},
"gemini/gemini-3-flash-preview": {
"input": 0.50 / 1_000_000,
"input_cached": 0.05 / 1_000_000,
"output": 3.00 / 1_000_000,
}
}
def _headers(self) -> dict:
if not self.valves.litellm_master_key:
return {}
@@ -72,7 +58,6 @@ class Tools:
"""Sync internal price cache with LiteLLM's live model registry."""
if not self.valves.enable_dynamic_pricing or self._price_cache:
return
try:
url = f"{self._base_url()}/v1/model/info"
resp = requests.get(url, headers=self._headers(), timeout=10)
@@ -88,36 +73,7 @@ class Tools:
"output": info.get("output_cost_per_token", 0),
"reasoning": info.get("output_cost_per_reasoning_token", info.get("output_cost_per_token", 0))
}
except Exception:
pass
def _audit_cost(self, model: str, prompt: int, cached: int, completion: int, reported_cost: float, reasoning: int = 0) -> dict:
"""Audit reported cost with dynamic or fallback pricing."""
self._fetch_live_prices()
pricing = self._price_cache.get(model) or self.MODEL_PRICING_FALLBACK.get(model)
if not pricing:
return {"status": "unsupported", "display": ""}
uncached = max(0, prompt - cached)
# Handle reasoning tokens separately if pricing allows
actual_output = max(0, completion - reasoning)
calc = (cached * pricing.get("input_cached", 0)) + \
(uncached * pricing.get("input", 0)) + \
(actual_output * pricing.get("output", 0)) + \
(reasoning * pricing.get("reasoning", pricing.get("output", 0)))
delta = reported_cost - calc
status = "mismatch" if abs(delta) > 0.005 else "ok"
symbol = "⚠️" if status == "mismatch" else "✓"
return {
"audited": calc,
"delta": delta,
"status": status,
"display": f" [calc: ${calc:.4f} {symbol}]"
}
except Exception: pass
def _resolve_date(self, date_str: str) -> str:
date_str = date_str.strip().lower()
@@ -125,163 +81,157 @@ class Tools:
today = datetime.now(ZoneInfo(self.valves.timezone)).date()
except Exception:
today = datetime.utcnow().date()
if date_str == "today": return today.isoformat()
if date_str == "yesterday": return (today - timedelta(days=1)).isoformat()
if "days ago" in date_str:
try:
n = int(date_str.split()[0])
return (today - timedelta(days=n)).isoformat()
except Exception: pass
try:
datetime.strptime(date_str, "%Y-%m-%d")
return date_str
except ValueError: pass
return date_str
def _audit_cost(self, model: str, prompt: int, cached: int, completion: int, reported_cost: float, reasoning: int = 0) -> str:
"""Audit reported cost with dynamic pricing."""
self._fetch_live_prices()
pricing = self._price_cache.get(model)
if not pricing: return ""
uncached = max(0, prompt - cached)
actual_output = max(0, completion - reasoning)
calc = (cached * pricing.get("input_cached", 0)) + \
(uncached * pricing.get("input", 0)) + \
(actual_output * pricing.get("output", 0)) + \
(reasoning * pricing.get("reasoning", pricing.get("output", 0)))
delta = reported_cost - calc
symbol = "⚠️" if abs(delta) > 0.005 else "✓"
return f" [calc: ${calc:.4f} {symbol}]"
def litellm_status(self) -> str:
"""
Get an ultra-lean snapshot of proxy health and total spend.
Uses /health/readiness and /global/spend/models for maximum token efficiency.
Comprehensive system check: Proxy readiness, DB connectivity, and Model health.
Use this to verify the infrastructure is up and which models are available.
"""
lines = [f"LiteLLM Status Snapshot — {self._base_url()}"]
lines.append("=" * 40)
lines = [f"LiteLLM Infrastructure Status — {self._base_url()}", "=" * 50]
# 1. Readiness (Infrastructure)
# 1. Readiness
try:
r = requests.get(f"{self._base_url()}/health/readiness", timeout=5)
if r.status_code == 200:
data = r.json()
lines.append(f"Proxy: ✅ {data.get('status')} | DB: {data.get('db')} | v{data.get('litellm_version')}")
else:
lines.append(f"Proxy Readiness: ⚠️ Status {r.status_code}")
except Exception as e:
lines.append(f"Proxy Readiness: ❌ Connection Error")
d = r.json()
lines.append(f"Proxy: ✅ {d.get('status')} | DB: {d.get('db')} | v{d.get('litellm_version')}")
else: lines.append(f"Proxy Readiness: ⚠️ Status {r.status_code}")
except Exception: lines.append("Proxy Readiness: ❌ Connection Error")
# 2. Spend Snapshot (Token Efficient)
if self.valves.litellm_master_key:
try:
s = requests.get(f"{self._base_url()}/global/spend/models", headers=self._headers(), timeout=5)
if s.status_code == 200:
models = s.json()
total = sum(m.get("total_spend", 0) for m in models)
top = sorted(models, key=lambda x: x.get("total_spend", 0), reverse=True)[:3]
lines.append(f"Lifetime Spend: ${total:.2f}")
if top:
lines.append("Top Models: " + ", ".join([f"{m['model']}: ${m['total_spend']:.2f}" for m in top]))
except Exception:
pass
return "\n".join(lines)
def litellm_health(self) -> str:
"""Check LiteLLM proxy health and authentication status."""
url = f"{self._base_url()}/health"
# 2. Model Health
try:
resp = requests.get(url, headers=self._headers(), timeout=15)
except Exception as e:
return f"❌ Connection failed: {str(e)}"
if resp.status_code != 200:
return f"❌ Error {resp.status_code}: {resp.text[:200]}"
data = resp.json()
healthy = data.get("healthy_endpoints", [])
unhealthy = data.get("unhealthy_endpoints", [])
h = requests.get(f"{self._base_url()}/health", headers=self._headers(), timeout=10)
if h.status_code == 200:
hd = h.json()
healthy = hd.get("healthy_endpoints", [])
unhealthy = hd.get("unhealthy_endpoints", [])
lines.append(f"Models: {len(healthy)} healthy, {len(unhealthy)} unhealthy")
for ep in healthy[:15]: # Cap display to save tokens
m = ep.get("model", "unknown")
lines.append(f" ✅ {m:25s} | rpm: {ep.get('rpm','?'):>3}")
if len(healthy) > 15: lines.append(f" ... and {len(healthy)-15} more healthy models.")
for ep in unhealthy:
lines.append(f" ❌ {ep.get('model', 'unknown')}")
except Exception: lines.append("Model Health: ❌ Failed to fetch")
lines = [f"LiteLLM Health Check — {self._base_url()}"]
lines.append("=" * 50)
lines.append(f"Healthy: {len(healthy)} | Unhealthy: {len(unhealthy)}")
for ep in healthy:
m = ep.get("model", "unknown")
lines.append(f" ✅ {m:25s} | rpm: {ep.get('rpm','?'):>3} | timeout: {ep.get('timeout','?')}s")
for ep in unhealthy:
lines.append(f" ❌ {ep.get('model', 'unknown')}")
return "\n".join(lines)
def litellm_cost(self, start_date: str, end_date: str, hide_zero_spend: bool = False) -> str:
"""Get cost breakdown with Sparkline trend visualization."""
if not self.valves.litellm_master_key:
return "❌ Master key required."
effective_hide = hide_zero_spend or self.valves.default_hide_zero_spend
def litellm_usage(self, start_date: str = "today", end_date: str = "today", detail_level: Literal["summary", "full"] = "summary") -> str:
"""
Get fiscal usage and spend data.
:param start_date: YYYY-MM-DD or shortcut (today, yesterday, 7 days ago)
:param end_date: YYYY-MM-DD or shortcut
:param detail_level: 'summary' for quick spend snapshot, 'full' for daily breakdown and sparklines.
"""
if not self.valves.litellm_master_key: return "❌ Master key required."
r_start = self._resolve_date(start_date)
r_end = self._resolve_date(end_date)
url = f"{self._base_url()}/user/daily/activity"
if detail_level == "summary":
try:
s = requests.get(f"{self._base_url()}/global/spend/models", headers=self._headers(), timeout=10)
if s.status_code == 200:
models = sorted(s.json(), key=lambda x: x.get("total_spend", 0), reverse=True)
total = sum(m.get("total_spend", 0) for m in models)
lines = [f"LiteLLM Spend Summary (Lifetime)", "=" * 40, f"Total Lifetime Spend: ${total:.2f}"]
for m in models[:5]:
lines.append(f" - {m['model']}: ${m['total_spend']:.2f}")
return "\n".join(lines)
except Exception: pass
# Full Report
try:
url = f"{self._base_url()}/user/daily/activity"
resp = requests.get(url, headers=self._headers(), params={"start_date": r_start, "end_date": r_end}, timeout=30)
if resp.status_code != 200: return f"❌ Error {resp.status_code}"
data = resp.json()
except Exception as e: return f"❌ Failed: {str(e)}"
results = data.get("results", [])
if not results: return "No data found."
# Sparkline Generation
if not results: return f"No usage found for {r_start} to {r_end}."
spends = [d.get("metrics", {}).get("spend", 0) for d in results]
def get_spark(vals):
if not vals: return ""
if len(vals) < 2: return ""
chars = " ▂▃▄▅▆▇█"
low, high = min(vals), max(vals)
rng = (high - low) or 1
return "".join(chars[int((v - low) / rng * 7)] for v in vals)
return " [" + "".join(chars[int((v - low) / rng * 7)] for v in vals) + "]"
meta = data.get("metadata", {})
lines = [f"LiteLLM Cost Report: {r_start} to {r_end}"]
lines.append("=" * 60)
lines.append(f"Total Spend: ${meta.get('total_spend', 0):.2f} [{get_spark(spends)}]")
lines.append(f"Requests: {meta.get('total_successful_requests', 0)} ok, {meta.get('total_failed_requests', 0)} fail")
lines = [f"LiteLLM Usage Report: {r_start} to {r_end}", "=" * 60]
lines.append(f"Spend: ${meta.get('total_spend', 0):.2f}{get_spark(spends)}")
lines.append(f"Requests: {meta.get('total_successful_requests', 0)} ok, {meta.get('total_failed_requests', 0)} fail")
# Endpoint breakdown
lines.append("\nBy Endpoint:")
lines.append("\nBreakdown by Endpoint:")
ends = results[0].get("breakdown", {}).get("endpoints", {}) if results else {}
for path, edata in ends.items():
m = edata.get("metrics", {})
lines.append(f" {path:20s} | ${m.get('spend',0):.4f} | {m.get('successful_requests',0)} reqs")
return "\n".join(lines)
def litellm_history(self, limit: int = 20, show_failed: bool = True) -> str:
"""Request history with Thought Token detection and Dynamic Audit."""
def litellm_history(self, limit: int = 20, filter_model: Optional[str] = None) -> str:
"""
Get recent request logs with audit data and thought detection.
:param limit: Number of requests (1-50)
:param filter_model: Optional string to filter by model name (e.g. 'gpt-4')
"""
if not self.valves.litellm_master_key: return "❌ Master key required."
limit = max(1, min(50, limit))
try:
resp = requests.get(f"{self._base_url()}/spend/logs", headers=self._headers(), timeout=30)
if resp.status_code != 200: return f"❌ Error {resp.status_code}"
logs = sorted(resp.json(), key=lambda x: x.get("startTime", ""), reverse=True)
except Exception as e: return f"❌ Failed: {str(e)}"
if not show_failed:
logs = [l for l in logs if not l.get("metadata", {}).get("exception")]
if filter_model:
logs = [l for l in logs if filter_model.lower() in l.get("model", "").lower()]
recent = logs[:limit]
lines = [f"LiteLLM History (Most Recent {len(recent)})", "=" * 90]
recent = logs[:max(1, min(50, limit))]
lines = [f"LiteLLM History (Most Recent {len(recent)})", "=" * 95]
for i, entry in enumerate(recent, 1):
ts = entry.get("startTime", "?")[5:19]
ts = entry.get("startTime", "?")[5:16]
model = entry.get("model", "unknown")[:25]
spend = entry.get("spend", 0)
prompt = entry.get("prompt_tokens", 0) or 0
completion = entry.get("completion_tokens", 0) or 0
# Thought / Reasoning tokens
usage = entry.get("usage", {}) or {}
thought = usage.get("completion_tokens_details", {}).get("reasoning_tokens", 0)
thought_str = f" [💭{thought}]" if thought else ""
# Cache
cached = entry.get("cache_read_input_tokens", 0) or 0
cache_str = f" 💾{int(cached/prompt*100)}%" if prompt and cached else ""
audit = self._audit_cost(entry.get("model"), prompt, cached, completion, spend, reasoning=thought)
audit_display = self._audit_cost(entry.get("model"), prompt, cached, completion, spend, reasoning=thought)
status = "❌" if entry.get("metadata", {}).get("exception") else "✅"
line = f"{i:2d}. {ts} | {model:25s} | ${spend:.4f}{audit['display']} | {prompt}→{completion}{thought_str}{cache_str}"
lines.append(line)
lines.append(f"{i:2d}. {ts} {status} | {model:25s} | ${spend:.4f}{audit_display} | {prompt}→{completion}{thought_str}{cache_str}")
return "\n".join(lines)
tests/integration_test.py
+26 -36
View File
@@ -1,13 +1,11 @@
import unittest
import os
import sys
from litellm_dashboard import Tools
# Note: Integration key is NOT stored in repo. It is read from /tmp at runtime.
INTEGRATION_KEY_PATH = "/tmp/litellm-integration-testing-key.txt"
LIVE_URL = "http://mon-serv1.captcouch.xyz:4001"
class TestLiteLLMTool(unittest.TestCase):
class TestLiteLLMToolConsolidated(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tools = Tools()
@@ -15,49 +13,41 @@ class TestLiteLLMTool(unittest.TestCase):
if os.path.exists(INTEGRATION_KEY_PATH):
with open(INTEGRATION_KEY_PATH, 'r') as f:
cls.tools.valves.litellm_master_key = f.read().strip()
else:
print("Warning: Integration key not found. Some tests will fail.")
def test_litellm_status_integration(self):
"""Test lean status snapshot against live instance."""
def test_litellm_status(self):
"""Test consolidated infrastructure + model health check."""
output = self.tools.litellm_status()
print("\n--- Status Output ---")
print(output)
self.assertIn("LiteLLM Status Snapshot", output)
self.assertIn("Infrastructure Status", output)
self.assertIn("Proxy: ✅", output)
if self.tools.valves.litellm_master_key:
self.assertIn("Lifetime Spend:", output)
self.assertIn("healthy", output.lower())
def test_litellm_health_integration(self):
"""Test health check against live instance."""
output = self.tools.litellm_health()
self.assertIn("LiteLLM Health Check", output)
self.assertIn("Healthy:", output)
def test_litellm_cost_visualization_integration(self):
"""Test cost report and sparkline logic."""
output = self.tools.litellm_cost(start_date="3 days ago", end_date="today")
print("\n--- Cost Output ---")
def test_litellm_usage_summary(self):
"""Test token-efficient usage snapshot."""
output = self.tools.litellm_usage(detail_level="summary")
print("\n--- Usage Summary ---")
print(output)
self.assertIn("Total Spend:", output)
self.assertIn("[", output) # Part of sparkline check
self.assertIn("]", output)
self.assertIn("Spend Summary", output)
self.assertIn("Total Lifetime Spend", output)
def test_litellm_history_audit_integration(self):
"""Test request history and Dynamic Audit Engine."""
output = self.tools.litellm_history(limit=5)
def test_litellm_usage_full(self):
"""Test detailed fiscal report with sparklines."""
output = self.tools.litellm_usage(start_date="7 days ago", detail_level="full")
print("\n--- Usage Full ---")
print(output)
self.assertIn("Usage Report", output)
self.assertIn("Breakdown by Endpoint", output)
def test_litellm_history_filtered(self):
"""Test filtered request history."""
output = self.tools.litellm_history(limit=5, filter_model="gemini")
print("\n--- History Output ---")
print(output)
self.assertIn("LiteLLM History", output)
# Check if audit engine is firing (either calc ok or mismatch)
self.assertTrue("calc:" in output or "History" in output)
def test_date_resolution(self):
"""Unit test for date resolver."""
d1 = self.tools._resolve_date("today")
d2 = self.tools._resolve_date("yesterday")
self.assertNotEqual(d1, d2)
self.assertEqual(len(d1), 10) # YYYY-MM-DD
self.assertIn("History", output)
# Verify filtering if logs exist
if "gemini" in output.lower():
self.assertTrue(True)
if __name__ == "__main__":
unittest.main()