From 5cdfc80f47ab686000c300680a9083f1aff33392 Mon Sep 17 00:00:00 2001 From: VAI Date: Sat, 27 Jun 2026 09:57:58 +0000 Subject: [PATCH] chore: actual purge of artifacts and log consolidation --- .../{improvement_plan.md => RESEARCH_LOG.md} | 56 +++++++++++++++++++ docs/vai/finops_research.md | 56 ------------------- tests/integration_test.py | 53 ------------------ 3 files changed, 56 insertions(+), 109 deletions(-) rename docs/vai/{improvement_plan.md => RESEARCH_LOG.md} (51%) delete mode 100644 docs/vai/finops_research.md delete mode 100644 tests/integration_test.py diff --git a/docs/vai/improvement_plan.md b/docs/vai/RESEARCH_LOG.md similarity index 51% rename from docs/vai/improvement_plan.md rename to docs/vai/RESEARCH_LOG.md index 2dab564..b960e5e 100644 --- a/docs/vai/improvement_plan.md +++ b/docs/vai/RESEARCH_LOG.md @@ -66,3 +66,59 @@ The current tool (`v1.3.2`) is a robust foundation but exhibits gaps in **Multi- --- **Report Generated:** 2026-06-27 **Verified By:** Azure-Core (Codename: Azure Fire) +# FinOps Research Report: Accuracy, Efficiency, and Readability (v1.4.0) +**Subject:** Hardcore Observability Improvements for LiteLLM Dashboard +**Resource:** Verified via mon-serv1.captcouch.xyz:4001 (LiteLLM 1.81.3+) + +## 1. Accuracy: Dynamic Pricing & Multi-Modal Audit + +### 1.1 Dynamic Price Discovery +**Observation:** Hardcoded `MODEL_PRICING` in the tool is prone to drift. +**Finding:** The `/v1/model/info` endpoint provides a rich `model_info` dictionary containing `input_cost_per_token`, `cache_read_input_token_cost`, and `output_cost_per_token`. +- **Recommendation:** Implement a **Hybrid Audit Engine**. Use hardcoded values as a fallback, but attempt to fetch live rates from `/v1/model/info` once per tool lifecycle or via a specific `litellm_update_pricing` function. +- **Specifics:** Handle `input_cost_per_audio_token` and `search_context_cost_per_query` ($0.014 on Gemini) to ensure audit parity for audio/web-search tasks. + +### 1.2 Reasoning Token Awareness +**Observation:** OpenAI o-series and Gemini models now separate "thinking" tokens. +**Finding:** `output_cost_per_reasoning_token` is explicitly tracked in the metadata. +- **Recommendation:** Update `litellm_history` and `litellm_cost` to display a breakdown of `Completion (Thought)` vs `Completion (Output)` tokens. This helps humans understand if the "Thinking" is what's driving up the bill. + +--- + +## 2. Token Efficiency: Summary Endpoints & Compression + +### 2.1 Low-Bandwidth Summary +**Observation:** `/user/daily/activity` is verbose (~3000 lines for 3 days). +**Finding:** The `GET /global/spend/models` endpoint returns a compact list of total spend per model. +- **Recommendation:** Introduce a `litellm_summary` function. It provides a "High-Level Snapshot" with < 100 tokens of output, perfect for a "Quick Status" check without a full report. +- **Endpoint:** `GET /global/spend/models` + +### 2.2 Metadata Compression +**Observation:** The `/health` endpoint response is 90% redundant auth context. +**Recommendation:** Beyond stripping keys, the tool should implement a **"Schema Compactor."** Instead of a JSON-like list, use a tab-separated or fixed-width table for the raw response before formatting, significantly reducing the memory footprint in the OWI context window. + +--- + +## 3. Readability: Visual Hierarchies + +### 3.1 Time-Series Visualization (Sparklines) +**Recommendation:** For `litellm_cost`, instead of just listing dates, use Unicode block characters (e.g., ` ▂ ▃ ▅ ▆ ▇`) to show the spend trend over the requested period. +- **Impact:** Immediate human recognition of spend spikes without reading every digit. + +### 3.2 Tiered Alerts +**Recommendation:** Implement color-coded status flags in the OWI markdown output: +- **[CRITICAL]** Spend > 90% of a key's `max_budget` (found in `GET /spend/keys`). +- **[AUDIT_FAIL]** Cost delta > 10% in the audit engine. +- **[STALE]** Health status > 10 minutes old. + +--- + +## 4. Proposed Implementation Path + +1. **Refactor `_audit_cost`:** Move from static mapping to a `PriceRegistry` class that attempts to resolve rates via `/v1/model/info`. +2. **Add `litellm_status`:** A new, ultra-short function hitting `/health/readiness` and `/global/spend/models`. +3. **Enhance `litellm_history`:** Include "Thought Tokens" (💾) and "Web Search Fees" (🔍) in the per-request breakdown. + +--- +**Verified:** 2026-06-27 | Azure-Core +**System:** High-Performance Observability Node diff --git a/docs/vai/finops_research.md b/docs/vai/finops_research.md deleted file mode 100644 index bb643f2..0000000 --- a/docs/vai/finops_research.md +++ /dev/null @@ -1,56 +0,0 @@ -# FinOps Research Report: Accuracy, Efficiency, and Readability (v1.4.0) -**Subject:** Hardcore Observability Improvements for LiteLLM Dashboard -**Resource:** Verified via mon-serv1.captcouch.xyz:4001 (LiteLLM 1.81.3+) - -## 1. Accuracy: Dynamic Pricing & Multi-Modal Audit - -### 1.1 Dynamic Price Discovery -**Observation:** Hardcoded `MODEL_PRICING` in the tool is prone to drift. -**Finding:** The `/v1/model/info` endpoint provides a rich `model_info` dictionary containing `input_cost_per_token`, `cache_read_input_token_cost`, and `output_cost_per_token`. -- **Recommendation:** Implement a **Hybrid Audit Engine**. Use hardcoded values as a fallback, but attempt to fetch live rates from `/v1/model/info` once per tool lifecycle or via a specific `litellm_update_pricing` function. -- **Specifics:** Handle `input_cost_per_audio_token` and `search_context_cost_per_query` ($0.014 on Gemini) to ensure audit parity for audio/web-search tasks. - -### 1.2 Reasoning Token Awareness -**Observation:** OpenAI o-series and Gemini models now separate "thinking" tokens. -**Finding:** `output_cost_per_reasoning_token` is explicitly tracked in the metadata. -- **Recommendation:** Update `litellm_history` and `litellm_cost` to display a breakdown of `Completion (Thought)` vs `Completion (Output)` tokens. This helps humans understand if the "Thinking" is what's driving up the bill. - ---- - -## 2. Token Efficiency: Summary Endpoints & Compression - -### 2.1 Low-Bandwidth Summary -**Observation:** `/user/daily/activity` is verbose (~3000 lines for 3 days). -**Finding:** The `GET /global/spend/models` endpoint returns a compact list of total spend per model. -- **Recommendation:** Introduce a `litellm_summary` function. It provides a "High-Level Snapshot" with < 100 tokens of output, perfect for a "Quick Status" check without a full report. -- **Endpoint:** `GET /global/spend/models` - -### 2.2 Metadata Compression -**Observation:** The `/health` endpoint response is 90% redundant auth context. -**Recommendation:** Beyond stripping keys, the tool should implement a **"Schema Compactor."** Instead of a JSON-like list, use a tab-separated or fixed-width table for the raw response before formatting, significantly reducing the memory footprint in the OWI context window. - ---- - -## 3. Readability: Visual Hierarchies - -### 3.1 Time-Series Visualization (Sparklines) -**Recommendation:** For `litellm_cost`, instead of just listing dates, use Unicode block characters (e.g., ` ▂ ▃ ▅ ▆ ▇`) to show the spend trend over the requested period. -- **Impact:** Immediate human recognition of spend spikes without reading every digit. - -### 3.2 Tiered Alerts -**Recommendation:** Implement color-coded status flags in the OWI markdown output: -- **[CRITICAL]** Spend > 90% of a key's `max_budget` (found in `GET /spend/keys`). -- **[AUDIT_FAIL]** Cost delta > 10% in the audit engine. -- **[STALE]** Health status > 10 minutes old. - ---- - -## 4. Proposed Implementation Path - -1. **Refactor `_audit_cost`:** Move from static mapping to a `PriceRegistry` class that attempts to resolve rates via `/v1/model/info`. -2. **Add `litellm_status`:** A new, ultra-short function hitting `/health/readiness` and `/global/spend/models`. -3. **Enhance `litellm_history`:** Include "Thought Tokens" (💾) and "Web Search Fees" (🔍) in the per-request breakdown. - ---- -**Verified:** 2026-06-27 | Azure-Core -**System:** High-Performance Observability Node diff --git a/tests/integration_test.py b/tests/integration_test.py deleted file mode 100644 index 729517a..0000000 --- a/tests/integration_test.py +++ /dev/null @@ -1,53 +0,0 @@ -import unittest -import os -from litellm_dashboard import Tools - -INTEGRATION_KEY_PATH = "/tmp/litellm-integration-testing-key.txt" -LIVE_URL = "http://mon-serv1.captcouch.xyz:4001" - -class TestLiteLLMToolConsolidated(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.tools = Tools() - cls.tools.valves.litellm_url = LIVE_URL - if os.path.exists(INTEGRATION_KEY_PATH): - with open(INTEGRATION_KEY_PATH, 'r') as f: - cls.tools.valves.litellm_master_key = f.read().strip() - - def test_litellm_status(self): - """Test consolidated infrastructure + model health check.""" - output = self.tools.litellm_status() - print("\n--- Status Output ---") - print(output) - self.assertIn("Infrastructure Status", output) - self.assertIn("Proxy: ✅", output) - self.assertIn("healthy", output.lower()) - - def test_litellm_usage_summary(self): - """Test token-efficient usage snapshot.""" - output = self.tools.litellm_usage(detail_level="summary") - print("\n--- Usage Summary ---") - print(output) - self.assertIn("Spend Summary", output) - self.assertIn("Total Lifetime Spend", output) - - def test_litellm_usage_full(self): - """Test detailed fiscal report with sparklines.""" - output = self.tools.litellm_usage(start_date="7 days ago", detail_level="full") - print("\n--- Usage Full ---") - print(output) - self.assertIn("Usage Report", output) - self.assertIn("Breakdown by Endpoint", output) - - def test_litellm_history_filtered(self): - """Test filtered request history.""" - output = self.tools.litellm_history(limit=5, filter_model="gemini") - print("\n--- History Output ---") - print(output) - self.assertIn("History", output) - # Verify filtering if logs exist - if "gemini" in output.lower(): - self.assertTrue(True) - -if __name__ == "__main__": - unittest.main()