From 49b95b1f62bd4f00beebf259d373fd3c6ea02003 Mon Sep 17 00:00:00 2001 From: VAI Date: Sat, 27 Jun 2026 09:53:32 +0000 Subject: [PATCH] feat(core): restore original high-density architecture and specialized power-functions for v1.6.0 --- litellm_dashboard.py | 231 +++++++++++++++++++++++++++---------------- 1 file changed, 144 insertions(+), 87 deletions(-) diff --git a/litellm_dashboard.py b/litellm_dashboard.py index 9162a9f..de8b2b5 100644 --- a/litellm_dashboard.py +++ b/litellm_dashboard.py @@ -2,10 +2,10 @@ title: LiteLLM Dashboard author: Kagamine Rin & VAI author_url: https://rin.captcouch.xyz -description: Professional LiteLLM observability — High-density health, cost breakdown, and request history. +description: High-performance LiteLLM observability — Comprehensive health, cost breakdowns, and request history. required_open_webui_version: 0.4.0 requirements: requests -version: 1.5.1 +version: 1.6.0 licence: MIT """ @@ -14,7 +14,7 @@ import json from datetime import datetime, timedelta from zoneinfo import ZoneInfo from pydantic import BaseModel, Field -from typing import Optional, List, Dict, Any, Literal +from typing import Optional, List, Dict, Any class Tools: @@ -35,11 +35,11 @@ class Tools: ) default_hide_zero_spend: bool = Field( default=True, - description="Default behavior for zero-spend model filtering.", + description="Default behavior for zero-spend model filtering. When True, models with sub-$0.001 spend are hidden by default.", ) timezone: str = Field( default="UTC", - description="IANA Timezone name (e.g. America/New_York).", + description="IANA Timezone name (e.g. America/New_York) for date resolution.", ) enable_dynamic_pricing: bool = Field( default=True, @@ -55,6 +55,7 @@ class Tools: return self.valves.litellm_url.rstrip("/") def _fetch_live_prices(self): + """Sync internal price cache with LiteLLM's live model registry.""" if not self.valves.enable_dynamic_pricing or self._price_cache: return try: @@ -87,6 +88,14 @@ class Tools: n = int(date_str.split()[0]) return (today - timedelta(days=n)).isoformat() except Exception: pass + + # Day names + days = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"] + if date_str in days: + target_day = days.index(date_str) + days_back = (today.weekday() - target_day) % 7 + return (today - timedelta(days=days_back)).isoformat() + try: datetime.strptime(date_str, "%Y-%m-%d") return date_str @@ -94,88 +103,107 @@ class Tools: return date_str def _audit_cost(self, model: str, prompt: int, cached: int, completion: int, reported_cost: float, reasoning: int = 0) -> str: + """Audit reported cost against dynamic pricing.""" self._fetch_live_prices() pricing = self._price_cache.get(model) if not pricing: return "" + uncached = max(0, prompt - cached) actual_output = max(0, completion - reasoning) + calc = (cached * (pricing.get("input_cached") or 0)) + \ (uncached * (pricing.get("input") or 0)) + \ (actual_output * (pricing.get("output") or 0)) + \ (reasoning * (pricing.get("reasoning") or pricing.get("output") or 0)) + delta = reported_cost - calc symbol = "⚠️" if abs(delta) > 0.005 else "✓" return f" [calc: ${calc:.4f} {symbol}]" - def litellm_status(self) -> str: + def litellm_health(self) -> str: """ - Comprehensive system check: Proxy readiness, DB connectivity, and Model health. - Use this to verify the infrastructure is up and which models are available. + Check LiteLLM proxy health and authentication status. + Returns detailed model health, rate limits, and infrastructure readiness. """ - lines = [f"LiteLLM Infrastructure Status — {self._base_url()}"] + lines = [f"LiteLLM Health Check — {self._base_url()}"] lines.append("=" * 60) + # Readiness Header try: r = requests.get(f"{self._base_url()}/health/readiness", timeout=5) if r.status_code == 200: d = r.json() lines.append(f"✅ Proxy: {d.get('status')} | DB: {d.get('db')} | v{d.get('litellm_version')}") - else: lines.append(f"⚠️ Proxy Readiness: Status {r.status_code}") - except Exception: lines.append("❌ Proxy Readiness: Connection Error") + except Exception: pass + + if not self.valves.litellm_master_key: + lines.append("⚠️ No master key configured — showing public health only") + else: + lines.append("✅ Authenticated with master key") try: - h = requests.get(f"{self._base_url()}/health", headers=self._headers(), timeout=10) - if h.status_code == 200: - hd = h.json() - healthy = hd.get("healthy_endpoints", []) - unhealthy = hd.get("unhealthy_endpoints", []) - lines.append(f"\nHealthy Models: {len(healthy)} | Unhealthy: {len(unhealthy)}") - lines.append("-" * 60) + url = f"{self._base_url()}/health" + resp = requests.get(url, headers=self._headers(), timeout=15) + if resp.status_code != 200: + return f"❌ Unexpected status {resp.status_code}: {resp.text[:500]}" + + data = resp.json() + healthy = data.get("healthy_endpoints", []) + unhealthy = data.get("unhealthy_endpoints", []) + + lines.append(f"\nHealthy: {len(healthy)} | Unhealthy: {len(unhealthy)}") + + if healthy: + lines.append(f"\n--- Healthy Models ({len(healthy)}) ---") for ep in healthy: - m = ep.get("model", "unknown") - display = m.replace("gemini/", "").replace("openai/", "") - lines.append(f" ✅ {display:30s} — rpm: {ep.get('rpm','?'):>3}, timeout: {ep.get('timeout','?')}s") + model = ep.get("model", "unknown") + display = model.replace("gemini/", "").replace("openai/", "") + rpm = ep.get("rpm", "?") + timeout = ep.get("timeout", "?") + lines.append(f" ✅ {display:30s} — rpm: {rpm:>3}, timeout: {timeout}s") + + if unhealthy: + lines.append(f"\n--- Unhealthy Models ({len(unhealthy)}) ---") for ep in unhealthy: lines.append(f" ❌ {ep.get('model', 'unknown')}") - except Exception as e: lines.append(f"\n❌ Health Check Failed: {str(e)}") - + + except Exception as e: + return f"❌ Connection failed: {str(e)}" + return "\n".join(lines) - def litellm_usage(self, start_date: str = "today", end_date: str = "today", detail_level: Literal["summary", "full"] = "summary") -> str: + def litellm_cost(self, start_date: str, end_date: str, hide_zero_spend: bool = False) -> str: """ - Get fiscal usage and spend data. - :param detail_level: 'summary' for quick spend snapshot, 'full' for daily breakdown and sparklines. + Get LiteLLM cost breakdown for a date range. + Returns per-model and per-day spend, token counts, cache hit rate, and trend sparklines. """ - if not self.valves.litellm_master_key: return "❌ Master key required." + if not self.valves.litellm_master_key: + return "❌ Master key required for cost data." + + effective_hide = hide_zero_spend or self.valves.default_hide_zero_spend r_start = self._resolve_date(start_date) r_end = self._resolve_date(end_date) - if detail_level == "summary": - try: - s = requests.get(f"{self._base_url()}/global/spend/models", headers=self._headers(), timeout=10) - if s.status_code == 200: - models = sorted(s.json(), key=lambda x: x.get("total_spend", 0), reverse=True) - total = sum(m.get("total_spend", 0) for m in models) - lines = [f"LiteLLM Spend Snapshot (Lifetime)", "=" * 50, f"Total Lifetime Spend: ${total:.2f}\n"] - lines.append(f"{'Model':30s} | {'Spend':>10s}") - lines.append("-" * 45) - for m in models[:8]: - display = m['model'].split("/")[-1] - lines.append(f" {display:28s} | ${m['total_spend']:9.2f}") - return "\n".join(lines) - except Exception: pass - + url = f"{self._base_url()}/user/daily/activity" try: - url = f"{self._base_url()}/user/daily/activity" resp = requests.get(url, headers=self._headers(), params={"start_date": r_start, "end_date": r_end}, timeout=30) - if resp.status_code != 200: return f"❌ Error {resp.status_code}" + if resp.status_code != 200: return f"❌ Unexpected status {resp.status_code}" data = resp.json() - except Exception as e: return f"❌ Failed: {str(e)}" + except Exception as e: return f"❌ Connection failed: {str(e)}" results = data.get("results", []) - if not results: return f"No usage found for {r_start} to {r_end}." + metadata = data.get("metadata", {}) + if not results: return f"No cost data found for {r_start} to {r_end}." + + # Totals + total_spend = metadata.get("total_spend", 0) + total_prompt = metadata.get("total_prompt_tokens", 0) + total_completion = metadata.get("total_completion_tokens", 0) + total_tokens = metadata.get("total_tokens", 0) + cache_read = metadata.get("total_cache_read_input_tokens", 0) - spends = [d.get("metrics", {}).get("spend", 0) for d in results] + # Sparkline + spends = [day.get("metrics", {}).get("spend", 0) for day in results] def get_spark(vals): if len(vals) < 2: return "" chars = " ▂▃▄▅▆▇█" @@ -183,80 +211,109 @@ class Tools: rng = (high - low) or 1 return " [" + "".join(chars[int((v - low) / rng * 7)] for v in vals) + "]" - meta = data.get("metadata", {}) - lines = [f"LiteLLM Usage Report: {r_start} to {r_end}"] + lines = [f"LiteLLM Cost Report: {r_start} to {r_end}"] lines.append("=" * 75) - lines.append(f"Total Spend: ${meta.get('total_spend', 0):.2f}{get_spark(spends)}") - lines.append(f"Total Tokens: {meta.get('total_tokens', 0):,} ({meta.get('total_prompt_tokens',0):,} prompt + {meta.get('total_completion_tokens',0):,} completion)") - lines.append(f"Requests: {meta.get('total_successful_requests', 0)} successful, {meta.get('total_failed_requests', 0)} failed") + lines.append(f"Total Spend: ${total_spend:.2f}{get_spark(spends)}") + lines.append(f"Total Tokens: {total_tokens:,} ({total_prompt:,} prompt + {total_completion:,} completion)") - # Model Breakdown + if total_prompt > 0 and cache_read > 0: + hit_rate = (cache_read / total_prompt) * 100 + lines.append(f"Cache Hit Rate: {hit_rate:.1f}% ({cache_read:,} / {total_prompt:,} input tokens)") + + lines.append(f"Requests: {metadata.get('total_successful_requests', 0)} ok, {metadata.get('total_failed_requests', 0)} fail") + + # Model Aggregation model_totals = {} for day in results: models = day.get("breakdown", {}).get("models", {}) for m_name, m_data in models.items(): - metrics = m_data.get("metrics", {}) + m_metrics = m_data.get("metrics", {}) if m_name not in model_totals: - model_totals[m_name] = {"spend": 0, "reqs": 0, "tokens": 0} - model_totals[m_name]["spend"] += metrics.get("spend", 0) - model_totals[m_name]["reqs"] += metrics.get("successful_requests", 0) - model_totals[m_name]["tokens"] += metrics.get("total_tokens", 0) + model_totals[m_name] = {"spend": 0, "reqs": 0, "tokens": 0, "cache": 0} + model_totals[m_name]["spend"] += m_metrics.get("spend", 0) + model_totals[m_name]["reqs"] += m_metrics.get("successful_requests", 0) + model_totals[m_name]["tokens"] += m_metrics.get("total_tokens", 0) + model_totals[m_name]["cache"] += m_metrics.get("cache_read_input_tokens", 0) - lines.append(f"\n{'Model':30s} | {'Spend':>10s} | {'Requests':>12s} | {'Tokens':>12s}") + lines.append(f"\n{'Model':30s} | {'Spend':>10s} | {'Requests':>14s} | {'Tokens':>12s}") lines.append("-" * 75) + + hidden_count = 0 sorted_models = sorted(model_totals.items(), key=lambda x: x[1]["spend"], reverse=True) - hidden = 0 - for m_name, m_stats in sorted_models: - if m_stats["spend"] < 0.001 and self.valves.default_hide_zero_spend: - hidden += 1 + for name, mt in sorted_models: + if effective_hide and mt["spend"] < 0.001: + hidden_count += 1 continue - display = m_name.split("/")[-1][:30] - lines.append(f" {display:28s} | ${m_stats['spend']:9.2f} | {m_stats['reqs']:10d} ok | {m_stats['tokens']:10,d}") - if hidden: lines.append(f" ... ({hidden} models with sub-$0.001 spend hidden)") + display = name.replace("gemini/", "").replace("openai/", "")[:30] + cache_info = "" + if mt["cache"] > 0: + hit = (mt["cache"] / (mt["tokens"] or 1) * 100) + cache_info = f" 💾{hit:.0f}%" + + lines.append(f" {display:28s} | ${mt['spend']:9.2f} | {mt['reqs']:6d} ok | {mt['tokens']:10,d}{cache_info}") + + if hidden_count: lines.append(f" ... ({hidden_count} model(s) with sub-$0.001 spend hidden)") - lines.append(f"\n{'Endpoint':30s} | {'Spend':>10s} | {'Requests':>12s}") - lines.append("-" * 60) - ends = results[0].get("breakdown", {}).get("endpoints", {}) if results else {} - for path, edata in sorted(ends.items(), key=lambda x: x[1].get("metrics",{}).get("spend",0), reverse=True): - m = edata.get("metrics", {}) - lines.append(f" {path:28s} | ${m.get('spend',0):9.4f} | {m.get('successful_requests',0):10d} ok") + # Date breakdown + lines.append("\nBy Date:") + for day in results: + m = day.get("metrics", {}) + lines.append(f" {day.get('date')}: ${m.get('spend',0):.2f} | {m.get('successful_requests',0)} ok | {m.get('total_tokens',0):,} tokens") return "\n".join(lines) - def litellm_history(self, limit: int = 20, filter_model: Optional[str] = None) -> str: - """Get recent request logs with audit data and thought detection.""" - if not self.valves.litellm_master_key: return "❌ Master key required." + def litellm_history(self, limit: int = 20, show_failed: bool = True) -> str: + """ + Get recent LiteLLM request history with audit data and thought detection. + Returns a high-density table of recent requests. + """ + if not self.valves.litellm_master_key: + return "❌ Master key required for request history." + try: resp = requests.get(f"{self._base_url()}/spend/logs", headers=self._headers(), timeout=30) - if resp.status_code != 200: return f"❌ Error {resp.status_code}" + if resp.status_code != 200: return f"❌ Unexpected status {resp.status_code}" logs = sorted(resp.json(), key=lambda x: x.get("startTime", ""), reverse=True) - except Exception as e: return f"❌ Failed: {str(e)}" + except Exception as e: return f"❌ Connection failed: {str(e)}" - if filter_model: - logs = [l for l in logs if filter_model.lower() in l.get("model", "").lower()] + if not show_failed: + logs = [l for l in logs if not l.get("metadata", {}).get("exception")] recent = logs[:max(1, min(50, limit))] - lines = [f"LiteLLM History — {self._base_url()}", "=" * 105] + lines = [f"LiteLLM Request History — {self._base_url()}"] + lines.append(f"Showing {len(recent)} of {len(logs)} total requests") + lines.append("=" * 105) for i, entry in enumerate(recent, 1): - ts = entry.get("startTime", "?")[5:16] - model = entry.get("model", "unknown").split("/")[-1][:25] + start_time = entry.get("startTime", "?")[5:16] # MM-DD HH:MM + model = entry.get("model", "unknown").split("/")[-1][:28] spend = entry.get("spend", 0) prompt = entry.get("prompt_tokens", 0) or 0 completion = entry.get("completion_tokens", 0) or 0 + # Metadata & Status + meta = entry.get("metadata", {}) or {} + is_failed = bool(meta.get("exception")) or meta.get("status") == "failure" + status = "❌ FAIL" if is_failed else "✅ OK" + + # Advanced Tokens usage = entry.get("usage", {}) or {} thought = usage.get("completion_tokens_details", {}).get("reasoning_tokens", 0) thought_str = f" [💭{thought}]" if thought else "" cached = entry.get("cache_read_input_tokens", 0) or 0 cache_str = f" 💾{int(cached/prompt*100)}%" if prompt and cached else "" - audit = self._audit_cost(entry.get("model"), prompt, cached, completion, spend, reasoning=thought) - status = "✅" if not entry.get("metadata", {}).get("exception") else "❌" - + # Format numbers p_s = f"{prompt/1000:.1f}k" if prompt >= 1000 else str(prompt) c_s = f"{completion/1000:.1f}k" if completion >= 1000 else str(completion) - lines.append(f"{i:2d}. {ts} {status} | {model:25s} | ${spend:.4f}{audit:18s} | {p_s:>6s}→{c_s:<6s}{thought_str}{cache_str}") - + audit = self._audit_cost(entry.get("model"), prompt, cached, completion, spend, reasoning=thought) + + line = f" {i:2d}. {start_time} | {model:28s} | {status} | ${spend:.4f}{audit:18s} | {p_s:>6s}→{c_s:<6s}{thought_str}{cache_str}" + lines.append(line) + + if is_failed: + err = meta.get("exception", "unknown error") + lines.append(f" └─ {err[:80]}") + return "\n".join(lines)