diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9384bae..4e43bd7 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -255,8 +255,29 @@ Showing 20 of 4560 total requests (most recent first) These are documented but not in scope for v1. The two-function design is the right scope. +## Cost Verification & Audit Engine (TASK-004) + +**Introduced in v1.3.0** + +To defend against model-level cost over-reporting or regression bugs (e.g., LiteLLM Issue #19681 / #11156), the tool includes an internal **Hardcore FinOps Audit Engine**. + +### How it works: +1. **Pricing Registry**: A static mapping of model identifiers to actual provider rates (including uncached input, cached input, and output rates). +2. **Audit Helper**: The `_audit_cost` method calculates the "True Cost" dynamically: + ```python + uncached_tokens = max(0, prompt_tokens - cached_tokens) + calculated_cost = (cached_tokens * input_cached_rate) + (uncached_tokens * input_uncached_rate) + (completion_tokens * output_rate) + ``` +3. **Discrepancy Flagging**: If the delta between LiteLLM's reported cost and our calculated cost is greater than half a cent ($0.005), it flags it with a warning `⚠️`. Otherwise, it marks it as validated `✓`. + +This ensures that we have immediate, live verification of context caching discounts (like Google's 90% implicit caching discount) on a per-request basis. + ## Changelog +### v1.3.0 (2026-06-26) +- **feat:** Hardcore FinOps Audit Engine (TASK-004) — added `_audit_cost` helper and static `MODEL_PRICING` registry to verify reported spend against actual provider caching rates. +- **feat:** Real-time request cost auditing in `litellm_history` output. + ### v1.2.0 (2026-06-13) - **feat:** Natural language date shortcuts for `litellm_cost()` — `today`, `yesterday`, `monday`, `last week`, etc. - **feat:** Two-level zero-spend filtering — valve default (`default_hide_zero_spend`, default True) + per-call override (`hide_zero_spend`) diff --git a/README.md b/README.md index f72c9d6..8b412ae 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,11 @@ Both valves are passed at runtime — no hardcoded keys. The tool is portable to ## Changelog +### v1.3.2 (2026-06-26) +- **feat:** Hardcore FinOps Audit Engine (TASK-004) — added `_audit_cost` helper and static `MODEL_PRICING` registry to verify reported spend against actual provider caching rates. +- **feat:** Real-time request cost auditing in `litellm_history` output. +- **fix:** Robust cache-extraction logic supporting nested metadata structures for Gemini/z.ai models. + ### v1.2.0 (2026-06-13) - **feat:** Natural language date shortcuts for `litellm_cost()` — `today`, `yesterday`, `monday`, `last week`, etc. - **feat:** `hide_zero_spend` parameter for `litellm_cost()` — default True hides sub-$0.001 models with summary line; per-call parameter can override diff --git a/litellm_dashboard.py b/litellm_dashboard.py index a32c033..0dad457 100644 --- a/litellm_dashboard.py +++ b/litellm_dashboard.py @@ -5,7 +5,7 @@ author_url: https://rin.captcouch.xyz description: Check LiteLLM proxy health + auth status and get cost breakdowns with configurable timerange. Two functions, that's it. required_open_webui_version: 0.4.0 requirements: requests -version: 1.2.2 +version: 1.3.2 licence: MIT """ @@ -20,6 +20,56 @@ class Tools: def __init__(self): self.valves = self.Valves() + # TASK-004: Hardcore FinOps Audit Registry (Standard Tier Pricing) + MODEL_PRICING = { + "gemini/gemini-3.5-flash": { + "input_uncached": 1.50 / 1_000_000, + "input_cached": 0.15 / 1_000_000, + "output": 9.00 / 1_000_000, + }, + "gemini/gemini-3-flash-preview": { + "input_uncached": 0.50 / 1_000_000, + "input_cached": 0.05 / 1_000_000, + "output": 3.00 / 1_000_000, + }, + "gemini/gemini-2.5-flash": { + "input_uncached": 0.30 / 1_000_000, + "input_cached": 0.03 / 1_000_000, + "output": 2.50 / 1_000_000, + }, + "gemini/gemini-3.1-flash-lite": { + "input_uncached": 0.25 / 1_000_000, + "input_cached": 0.025 / 1_000_000, + "output": 1.50 / 1_000_000, + }, + "gemini/gemini-3.1-pro-preview": { + "input_uncached": 2.00 / 1_000_000, + "input_cached": 0.20 / 1_000_000, + "output": 12.00 / 1_000_000, + } + } + + def _audit_cost(self, model: str, prompt: int, cached: int, completion: int, reported_cost: float) -> dict: + """Audit reported cost against our pricing registry, factoring in context caching discounts.""" + pricing = self.MODEL_PRICING.get(model) + if not pricing: + return {"audited": reported_cost, "delta": 0.0, "status": "unsupported", "display": ""} + + # Subtract cached tokens from prompt before applying standard rate + uncached = max(0, prompt - cached) + calculated = (cached * pricing["input_cached"]) + (uncached * pricing["input_uncached"]) + (completion * pricing["output"]) + delta = reported_cost - calculated + + # Flag discrepancies greater than half a cent + if abs(delta) > 0.005: + status = "mismatch" + display = f" [calc: ${calculated:.4f} ⚠️]" + else: + status = "ok" + display = f" [calc: ${calculated:.4f} ✓]" + + return {"audited": calculated, "delta": delta, "status": status, "display": display} + class Valves(BaseModel): """LiteLLM proxy connection settings.""" litellm_url: str = Field( @@ -279,6 +329,7 @@ class Tools: model_totals[model_name] = { "spend": 0, "requests": 0, "prompt": 0, "completion": 0, "total": 0, "failed": 0, + "cache_read": 0, "cache_created": 0 } model_totals[model_name]["spend"] += m_metrics.get("spend", 0) model_totals[model_name]["requests"] += m_metrics.get("successful_requests", 0) @@ -286,6 +337,8 @@ class Tools: model_totals[model_name]["prompt"] += m_metrics.get("prompt_tokens", 0) model_totals[model_name]["completion"] += m_metrics.get("completion_tokens", 0) model_totals[model_name]["total"] += m_metrics.get("total_tokens", 0) + model_totals[model_name]["cache_read"] += m_metrics.get("cache_read_input_tokens", 0) + model_totals[model_name]["cache_created"] += m_metrics.get("cache_creation_input_tokens", 0) # Build per-date breakdown date_lines = [] @@ -316,16 +369,32 @@ class Tools: max_name = 24 if len(display) > max_name: display = display[:max_name - 1] + "…" + + # Build token string with split and cache indicator + p_tokens = mt["prompt"] + c_tokens = mt["completion"] + + # Compact token numbers + p_str = f"{p_tokens / 1000:.1f}k" if p_tokens >= 1000 else str(p_tokens) + c_str = f"{c_tokens / 1000:.1f}k" if c_tokens >= 1000 else str(c_tokens) + token_display = f"{p_str}→{c_str}" + + # Cache indicator + cache_info = "" + if mt["cache_read"] > 0: + hit_rate = (mt["cache_read"] / p_tokens * 100) if p_tokens > 0 else 0 + cache_info = f" 💾{hit_rate:.0f}%" + model_lines.append( f" {display:25s} ${mt['spend']:7.2f} | " f"{mt['requests']:3d} ok, {mt['failed']:d} fail | " - f"{mt['total']:>10,} tokens" + f"{token_display:>10s}{cache_info}" ) # Assemble report lines = [] lines.append(f"LiteLLM Cost Report: {resolved_start} to {resolved_end}") - lines.append("=" * 60) + lines.append("=" * 70) lines.append(f"Total Spend: ${total_spend:.2f}") lines.append(f"Total Tokens: {total_tokens:,} ({total_prompt:,} prompt + {total_completion:,} completion)") lines.append(f"Cache Hit Rate: {cache_str}") @@ -441,8 +510,17 @@ class Tools: completion_tokens = entry.get("completion_tokens", 0) or 0 total_tokens = entry.get("total_tokens", 0) or 0 - # Cache + # Cache extraction (handles Anthropic/OpenAI top-level and Gemini/z.ai nested metadata structures) cache_read = entry.get("cache_read_input_tokens", 0) or 0 + if cache_read == 0: + metadata = entry.get("metadata", {}) or {} + usage_obj = metadata.get("usage_object", {}) or {} + if isinstance(usage_obj, dict): + cache_read = usage_obj.get("cache_read_input_tokens", 0) or 0 + if cache_read == 0: + prompt_details = entry.get("prompt_tokens_details", {}) or {} + if isinstance(prompt_details, dict): + cache_read = prompt_details.get("cached_tokens", 0) or 0 # Spend spend = entry.get("spend", 0) or 0 @@ -470,9 +548,15 @@ class Tools: base += f" | {token_str}" - # Cost + # Cost & Audit if show_cost and spend > 0: - base += f" | ${spend:.4f}" + audit = self._audit_cost(entry.get("model", ""), prompt_tokens, cache_read, completion_tokens, spend) + if audit["status"] == "mismatch": + base += f" | ${spend:.4f} (calc: ${audit['audited']:.4f} ⚠️)" + elif audit["status"] == "ok": + base += f" | ${spend:.4f} ✓" + else: + base += f" | ${spend:.4f}" # Error info if is_failed: