From e687f7d45725035c3a4525cdafe92dac2f735b66 Mon Sep 17 00:00:00 2001 From: Kagamine Rin Date: Fri, 29 May 2026 22:37:35 -0400 Subject: [PATCH] feat: Text generation suggestions via Gemini models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend: - New text_gen.py module with Gemini text generation client - suggest_prompts(): generates 3 prompt suggestions from user input - get_text_gen_config(): loads config from .env (model, API key, safety) - get_available_models(): returns list of Gemini text models - Safety settings: harassment, hate_speech, sexually_explicit, dangerous Configurable via TEXT_GEN_SAFETY_* env vars (default: BLOCK_ONLY_HIGH) - New endpoints: * POST /api/text-gen/suggest — generates 3 prompt suggestions * GET /api/text-gen/models — lists available text models * GET /api/text-gen/config — returns safe config (API key masked) * POST /api/text-gen/config — updates config in-memory - Default model: gemini-2.5-flash - Fallback API key: LYRIA_API_KEY if TEXT_GEN_API_KEY not set - System prompt: expert music producer persona for rich musical detail Frontend: - New SuggestionsModal component for displaying generated suggestions - Auto-fetches when opened - Shows loading spinner, error state, or suggestion list - Each suggestion clickable to populate the prompt field - Hover effect: 'Click to use this prompt →' - Generator component updated with 'Get Suggestions ✨' button - Appears next to prompt label in both Basic and Advanced modes - Disabled when no input present - Clicking suggestion populates the prompt/structured fields - SettingsModal expanded with Text Generation section (collapsible) - Same pattern as Image Generation: model, API key status, safety filters - Editable dropdowns with Save Changes button - Refactored to use shared useGenSettings hook for both sections Models Available: - gemini-2.5-flash (GA, default) — best price-to-performance - gemini-2.5-pro (GA) — highest quality reasoning - gemini-3.5-flash (Preview) — next-gen Flash - gemini-3.1-flash (Preview) — latest with thinking Architecture KB and Tasks KB updated. --- backend/src/main.py | 52 ++- backend/src/text_gen.py | 140 ++++++++ frontend/src/components/Generator.tsx | 44 ++- frontend/src/components/SettingsModal.tsx | 349 ++++++++++--------- frontend/src/components/SuggestionsModal.tsx | 130 +++++++ 5 files changed, 547 insertions(+), 168 deletions(-) create mode 100644 backend/src/text_gen.py create mode 100644 frontend/src/components/SuggestionsModal.tsx diff --git a/backend/src/main.py b/backend/src/main.py index 1114e1a..b0b4b5c 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -14,9 +14,8 @@ from database import init_db, create_clip, get_clips, get_clip, delete_clip, upd from lyria_client import generate_clip, save_clip_file from waveform import generate_waveform_svg, get_waveform_path from analysis import analyze_clip -from image_gen import generate_artwork, get_image_gen_config, get_available_models -from analysis import analyze_clip from image_gen import generate_artwork, get_image_gen_config, get_available_models, SAFETY_LEVELS +from text_gen import suggest_prompts, get_text_gen_config, get_available_models as get_text_models app = FastAPI(title="Lyria Studio", version="0.1.0") @@ -320,6 +319,55 @@ async def update_image_gen_configuration(request: dict): return {"status": "updated"} +# ─── Text Generation Endpoints ────────────────────────────────────── + +@app.get("/api/text-gen/models") +async def list_text_gen_models(): + """List available text generation models.""" + return {"models": get_text_models()} + +@app.get("/api/text-gen/config") +async def get_text_gen_configuration(): + """Get current text generation configuration (safe values only).""" + config = get_text_gen_config() + return { + "model": config["model"], + "safety_harassment": config["safety_harassment"], + "safety_hate_speech": config["safety_hate_speech"], + "safety_sexually_explicit": config["safety_sexually_explicit"], + "safety_dangerous": config["safety_dangerous"], + "has_api_key": bool(config.get("api_key")), + } + +@app.post("/api/text-gen/config") +async def update_text_gen_configuration(request: dict): + """Update text generation configuration (in-memory only).""" + if "model" in request: + os.environ["TEXT_GEN_MODEL"] = request["model"] + if "safety_harassment" in request: + os.environ["TEXT_GEN_SAFETY_HARASSMENT"] = request["safety_harassment"] + if "safety_hate_speech" in request: + os.environ["TEXT_GEN_SAFETY_HATE_SPEECH"] = request["safety_hate_speech"] + if "safety_sexually_explicit" in request: + os.environ["TEXT_GEN_SAFETY_SEXUALLY_EXPLICIT"] = request["safety_sexually_explicit"] + if "safety_dangerous" in request: + os.environ["TEXT_GEN_SAFETY_DANGEROUS"] = request["safety_dangerous"] + return {"status": "updated"} + +@app.post("/api/text-gen/suggest") +async def get_suggestions(request: dict): + """Get prompt suggestions based on user input.""" + user_input = request.get("input", "").strip() + if not user_input: + raise HTTPException(status_code=400, detail="Input is required") + + try: + config = get_text_gen_config() + suggestions = await suggest_prompts(user_input, config=config) + return {"suggestions": suggestions} + except Exception as e: + raise HTTPException(status_code=500, detail=f"Suggestion generation failed: {str(e)}") + @app.patch("/api/clips/{clip_id}") async def rename_clip_endpoint(clip_id: str, request: RenameRequest): clip = await get_clip(clip_id) diff --git a/backend/src/text_gen.py b/backend/src/text_gen.py new file mode 100644 index 0000000..9e703e3 --- /dev/null +++ b/backend/src/text_gen.py @@ -0,0 +1,140 @@ +"""Text generation client using Google Gemini models for prompt suggestions.""" + +import os +from typing import Optional + +from google import genai +from google.genai import types + +# ── Configuration ────────────────────────────────────────────────────────── + +def get_text_gen_config(): + """Load text generation configuration from environment.""" + return { + "api_key": os.environ.get("TEXT_GEN_API_KEY") or os.environ.get("LYRIA_API_KEY"), + "model": os.environ.get("TEXT_GEN_MODEL", "gemini-2.5-flash"), + "safety_harassment": os.environ.get("TEXT_GEN_SAFETY_HARASSMENT", "BLOCK_ONLY_HIGH"), + "safety_hate_speech": os.environ.get("TEXT_GEN_SAFETY_HATE_SPEECH", "BLOCK_ONLY_HIGH"), + "safety_sexually_explicit": os.environ.get("TEXT_GEN_SAFETY_SEXUALLY_EXPLICIT", "BLOCK_ONLY_HIGH"), + "safety_dangerous": os.environ.get("TEXT_GEN_SAFETY_DANGEROUS", "BLOCK_ONLY_HIGH"), + } + +# ── Safety Settings ──────────────────────────────────────────────────────── + +def build_safety_settings(config: dict) -> list: + """Build safety settings from config.""" + return [ + types.SafetySetting( + category="HARM_CATEGORY_HARASSMENT", + threshold=config.get("safety_harassment", "BLOCK_ONLY_HIGH"), + ), + types.SafetySetting( + category="HARM_CATEGORY_HATE_SPEECH", + threshold=config.get("safety_hate_speech", "BLOCK_ONLY_HIGH"), + ), + types.SafetySetting( + category="HARM_CATEGORY_SEXUALLY_EXPLICIT", + threshold=config.get("safety_sexually_explicit", "BLOCK_ONLY_HIGH"), + ), + types.SafetySetting( + category="HARM_CATEGORY_DANGEROUS_CONTENT", + threshold=config.get("safety_dangerous", "BLOCK_ONLY_HIGH"), + ), + ] + +# ── Prompt Suggestion ────────────────────────────────────────────────────── + +SYSTEM_PROMPT = """You are an expert music producer and prompt engineer for AI music generation models. +Your task is to help users write better prompts for generating music with AI models like Google Lyria. + +When given a user's idea or partial prompt, expand it into detailed, evocative music descriptions. +Include specific details about: +- Genre and subgenre +- Instrumentation and sound design +- Mood, energy, and emotional arc +- Tempo and rhythmic feel +- Structure (intro, build, drop, breakdown, etc.) +- Production style (clean, lo-fi, distorted, etc.) + +Keep suggestions concise but rich in musical detail. Use terminology that music producers would use. +""" + +async def suggest_prompts(user_input: str, model: str = None, config: dict = None) -> list: + """Generate prompt suggestions based on user input. + + Returns a list of suggestion strings. + """ + cfg = config or get_text_gen_config() + client = genai.Client(api_key=cfg["api_key"]) + + safety = build_safety_settings(cfg) + + prompt = f"""The user wants to generate music with an AI music model. They described their idea as: + +"{user_input}" + +Please provide 3 distinct, detailed prompts they could use. Each prompt should be a single paragraph that vividly describes the music. Make them different from each other — explore different angles of the user's idea. + +Format your response as a numbered list (1., 2., 3.) with just the prompts, no extra commentary.""" + + response = client.models.generate_content( + model=model or cfg["model"], + contents=prompt, + config=types.GenerateContentConfig( + system_instruction=SYSTEM_PROMPT, + safety_settings=safety, + temperature=0.9, + max_output_tokens=2048, + ), + ) + + # Parse response into list of suggestions + text = response.text.strip() + suggestions = [] + + # Split by numbered lines (1., 2., 3.) + import re + # Find all text after numbers + matches = re.findall(r'(?:^|\n)\d+\.\s*(.*?)(?=\n\d+\.|$)', text, re.DOTALL) + if matches: + suggestions = [m.strip() for m in matches if m.strip()] + else: + # Fallback: split by newlines and filter empty + lines = [line.strip() for line in text.split('\n') if line.strip()] + # Remove lines that are just numbers + suggestions = [line for line in lines if not re.match(r'^\d+\.$', line)] + + return suggestions[:3] # Ensure max 3 + +# ── Available Models ─────────────────────────────────────────────────────── + +AVAILABLE_TEXT_MODELS = [ + { + "id": "gemini-2.5-flash", + "name": "Gemini 2.5 Flash", + "description": "Best price-to-performance. Fast, capable, 1M context.", + "status": "GA", + }, + { + "id": "gemini-2.5-pro", + "name": "Gemini 2.5 Pro", + "description": "Highest quality reasoning and coding. Slower, more expensive.", + "status": "GA", + }, + { + "id": "gemini-3.5-flash", + "name": "Gemini 3.5 Flash", + "description": "Next-gen Flash. Strong reasoning, fast inference.", + "status": "Preview", + }, + { + "id": "gemini-3.1-flash", + "name": "Gemini 3.1 Flash", + "description": "Latest Flash with thinking capabilities.", + "status": "Preview", + }, +] + +def get_available_models() -> list: + """Return list of available text generation models.""" + return AVAILABLE_TEXT_MODELS diff --git a/frontend/src/components/Generator.tsx b/frontend/src/components/Generator.tsx index de1b95e..d191d6d 100644 --- a/frontend/src/components/Generator.tsx +++ b/frontend/src/components/Generator.tsx @@ -1,5 +1,6 @@ import { useState, useEffect } from 'react' import StructuredPromptEditor, { PromptStructure } from './StructuredPromptEditor' +import SuggestionsModal from './SuggestionsModal' interface ModelInfo { name: string @@ -23,6 +24,7 @@ export default function Generator({ onGenerate }: Props) { const [models, setModels] = useState>({}) const [promptMode, setPromptMode] = useState('basic') const [structuredPrompt, setStructuredPrompt] = useState({}) + const [suggestionsOpen, setSuggestionsOpen] = useState(false) useEffect(() => { fetch('/api/models') @@ -181,9 +183,17 @@ export default function Generator({ onGenerate }: Props) { {/* Basic Prompt */} {(promptMode === 'basic' || !isPro) && (
- +
+ + +