feat: Text generation suggestions via Gemini models
Backend:
- New text_gen.py module with Gemini text generation client
- suggest_prompts(): generates 3 prompt suggestions from user input
- get_text_gen_config(): loads config from .env (model, API key, safety)
- get_available_models(): returns list of Gemini text models
- Safety settings: harassment, hate_speech, sexually_explicit, dangerous
Configurable via TEXT_GEN_SAFETY_* env vars (default: BLOCK_ONLY_HIGH)
- New endpoints:
* POST /api/text-gen/suggest — generates 3 prompt suggestions
* GET /api/text-gen/models — lists available text models
* GET /api/text-gen/config — returns safe config (API key masked)
* POST /api/text-gen/config — updates config in-memory
- Default model: gemini-2.5-flash
- Fallback API key: LYRIA_API_KEY if TEXT_GEN_API_KEY not set
- System prompt: expert music producer persona for rich musical detail
Frontend:
- New SuggestionsModal component for displaying generated suggestions
- Auto-fetches when opened
- Shows loading spinner, error state, or suggestion list
- Each suggestion clickable to populate the prompt field
- Hover effect: 'Click to use this prompt →'
- Generator component updated with 'Get Suggestions ✨' button
- Appears next to prompt label in both Basic and Advanced modes
- Disabled when no input present
- Clicking suggestion populates the prompt/structured fields
- SettingsModal expanded with Text Generation section (collapsible)
- Same pattern as Image Generation: model, API key status, safety filters
- Editable dropdowns with Save Changes button
- Refactored to use shared useGenSettings hook for both sections
Models Available:
- gemini-2.5-flash (GA, default) — best price-to-performance
- gemini-2.5-pro (GA) — highest quality reasoning
- gemini-3.5-flash (Preview) — next-gen Flash
- gemini-3.1-flash (Preview) — latest with thinking
Architecture KB and Tasks KB updated.
This commit is contained in:
1 parent
70beb89ad9
commit
e687f7d457
5 files changed
+547
-168
No files matched your search
+50
-2
@@ -14,9 +14,8 @@ from database import init_db, create_clip, get_clips, get_clip, delete_clip, upd
|
||||
from lyria_client import generate_clip, save_clip_file
|
||||
from waveform import generate_waveform_svg, get_waveform_path
|
||||
from analysis import analyze_clip
|
||||
from image_gen import generate_artwork, get_image_gen_config, get_available_models
|
||||
from analysis import analyze_clip
|
||||
from image_gen import generate_artwork, get_image_gen_config, get_available_models, SAFETY_LEVELS
|
||||
from text_gen import suggest_prompts, get_text_gen_config, get_available_models as get_text_models
|
||||
|
||||
app = FastAPI(title="Lyria Studio", version="0.1.0")
|
||||
|
||||
@@ -320,6 +319,55 @@ async def update_image_gen_configuration(request: dict):
|
||||
|
||||
return {"status": "updated"}
|
||||
|
||||
# ─── Text Generation Endpoints ──────────────────────────────────────
|
||||
|
||||
@app.get("/api/text-gen/models")
|
||||
async def list_text_gen_models():
|
||||
"""List available text generation models."""
|
||||
return {"models": get_text_models()}
|
||||
|
||||
@app.get("/api/text-gen/config")
|
||||
async def get_text_gen_configuration():
|
||||
"""Get current text generation configuration (safe values only)."""
|
||||
config = get_text_gen_config()
|
||||
return {
|
||||
"model": config["model"],
|
||||
"safety_harassment": config["safety_harassment"],
|
||||
"safety_hate_speech": config["safety_hate_speech"],
|
||||
"safety_sexually_explicit": config["safety_sexually_explicit"],
|
||||
"safety_dangerous": config["safety_dangerous"],
|
||||
"has_api_key": bool(config.get("api_key")),
|
||||
}
|
||||
|
||||
@app.post("/api/text-gen/config")
|
||||
async def update_text_gen_configuration(request: dict):
|
||||
"""Update text generation configuration (in-memory only)."""
|
||||
if "model" in request:
|
||||
os.environ["TEXT_GEN_MODEL"] = request["model"]
|
||||
if "safety_harassment" in request:
|
||||
os.environ["TEXT_GEN_SAFETY_HARASSMENT"] = request["safety_harassment"]
|
||||
if "safety_hate_speech" in request:
|
||||
os.environ["TEXT_GEN_SAFETY_HATE_SPEECH"] = request["safety_hate_speech"]
|
||||
if "safety_sexually_explicit" in request:
|
||||
os.environ["TEXT_GEN_SAFETY_SEXUALLY_EXPLICIT"] = request["safety_sexually_explicit"]
|
||||
if "safety_dangerous" in request:
|
||||
os.environ["TEXT_GEN_SAFETY_DANGEROUS"] = request["safety_dangerous"]
|
||||
return {"status": "updated"}
|
||||
|
||||
@app.post("/api/text-gen/suggest")
|
||||
async def get_suggestions(request: dict):
|
||||
"""Get prompt suggestions based on user input."""
|
||||
user_input = request.get("input", "").strip()
|
||||
if not user_input:
|
||||
raise HTTPException(status_code=400, detail="Input is required")
|
||||
|
||||
try:
|
||||
config = get_text_gen_config()
|
||||
suggestions = await suggest_prompts(user_input, config=config)
|
||||
return {"suggestions": suggestions}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Suggestion generation failed: {str(e)}")
|
||||
|
||||
@app.patch("/api/clips/{clip_id}")
|
||||
async def rename_clip_endpoint(clip_id: str, request: RenameRequest):
|
||||
clip = await get_clip(clip_id)
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Text generation client using Google Gemini models for prompt suggestions."""
|
||||
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
# ── Configuration ──────────────────────────────────────────────────────────
|
||||
|
||||
def get_text_gen_config():
|
||||
"""Load text generation configuration from environment."""
|
||||
return {
|
||||
"api_key": os.environ.get("TEXT_GEN_API_KEY") or os.environ.get("LYRIA_API_KEY"),
|
||||
"model": os.environ.get("TEXT_GEN_MODEL", "gemini-2.5-flash"),
|
||||
"safety_harassment": os.environ.get("TEXT_GEN_SAFETY_HARASSMENT", "BLOCK_ONLY_HIGH"),
|
||||
"safety_hate_speech": os.environ.get("TEXT_GEN_SAFETY_HATE_SPEECH", "BLOCK_ONLY_HIGH"),
|
||||
"safety_sexually_explicit": os.environ.get("TEXT_GEN_SAFETY_SEXUALLY_EXPLICIT", "BLOCK_ONLY_HIGH"),
|
||||
"safety_dangerous": os.environ.get("TEXT_GEN_SAFETY_DANGEROUS", "BLOCK_ONLY_HIGH"),
|
||||
}
|
||||
|
||||
# ── Safety Settings ────────────────────────────────────────────────────────
|
||||
|
||||
def build_safety_settings(config: dict) -> list:
|
||||
"""Build safety settings from config."""
|
||||
return [
|
||||
types.SafetySetting(
|
||||
category="HARM_CATEGORY_HARASSMENT",
|
||||
threshold=config.get("safety_harassment", "BLOCK_ONLY_HIGH"),
|
||||
),
|
||||
types.SafetySetting(
|
||||
category="HARM_CATEGORY_HATE_SPEECH",
|
||||
threshold=config.get("safety_hate_speech", "BLOCK_ONLY_HIGH"),
|
||||
),
|
||||
types.SafetySetting(
|
||||
category="HARM_CATEGORY_SEXUALLY_EXPLICIT",
|
||||
threshold=config.get("safety_sexually_explicit", "BLOCK_ONLY_HIGH"),
|
||||
),
|
||||
types.SafetySetting(
|
||||
category="HARM_CATEGORY_DANGEROUS_CONTENT",
|
||||
threshold=config.get("safety_dangerous", "BLOCK_ONLY_HIGH"),
|
||||
),
|
||||
]
|
||||
|
||||
# ── Prompt Suggestion ──────────────────────────────────────────────────────
|
||||
|
||||
SYSTEM_PROMPT = """You are an expert music producer and prompt engineer for AI music generation models.
|
||||
Your task is to help users write better prompts for generating music with AI models like Google Lyria.
|
||||
|
||||
When given a user's idea or partial prompt, expand it into detailed, evocative music descriptions.
|
||||
Include specific details about:
|
||||
- Genre and subgenre
|
||||
- Instrumentation and sound design
|
||||
- Mood, energy, and emotional arc
|
||||
- Tempo and rhythmic feel
|
||||
- Structure (intro, build, drop, breakdown, etc.)
|
||||
- Production style (clean, lo-fi, distorted, etc.)
|
||||
|
||||
Keep suggestions concise but rich in musical detail. Use terminology that music producers would use.
|
||||
"""
|
||||
|
||||
async def suggest_prompts(user_input: str, model: str = None, config: dict = None) -> list:
|
||||
"""Generate prompt suggestions based on user input.
|
||||
|
||||
Returns a list of suggestion strings.
|
||||
"""
|
||||
cfg = config or get_text_gen_config()
|
||||
client = genai.Client(api_key=cfg["api_key"])
|
||||
|
||||
safety = build_safety_settings(cfg)
|
||||
|
||||
prompt = f"""The user wants to generate music with an AI music model. They described their idea as:
|
||||
|
||||
"{user_input}"
|
||||
|
||||
Please provide 3 distinct, detailed prompts they could use. Each prompt should be a single paragraph that vividly describes the music. Make them different from each other — explore different angles of the user's idea.
|
||||
|
||||
Format your response as a numbered list (1., 2., 3.) with just the prompts, no extra commentary."""
|
||||
|
||||
response = client.models.generate_content(
|
||||
model=model or cfg["model"],
|
||||
contents=prompt,
|
||||
config=types.GenerateContentConfig(
|
||||
system_instruction=SYSTEM_PROMPT,
|
||||
safety_settings=safety,
|
||||
temperature=0.9,
|
||||
max_output_tokens=2048,
|
||||
),
|
||||
)
|
||||
|
||||
# Parse response into list of suggestions
|
||||
text = response.text.strip()
|
||||
suggestions = []
|
||||
|
||||
# Split by numbered lines (1., 2., 3.)
|
||||
import re
|
||||
# Find all text after numbers
|
||||
matches = re.findall(r'(?:^|\n)\d+\.\s*(.*?)(?=\n\d+\.|$)', text, re.DOTALL)
|
||||
if matches:
|
||||
suggestions = [m.strip() for m in matches if m.strip()]
|
||||
else:
|
||||
# Fallback: split by newlines and filter empty
|
||||
lines = [line.strip() for line in text.split('\n') if line.strip()]
|
||||
# Remove lines that are just numbers
|
||||
suggestions = [line for line in lines if not re.match(r'^\d+\.$', line)]
|
||||
|
||||
return suggestions[:3] # Ensure max 3
|
||||
|
||||
# ── Available Models ───────────────────────────────────────────────────────
|
||||
|
||||
AVAILABLE_TEXT_MODELS = [
|
||||
{
|
||||
"id": "gemini-2.5-flash",
|
||||
"name": "Gemini 2.5 Flash",
|
||||
"description": "Best price-to-performance. Fast, capable, 1M context.",
|
||||
"status": "GA",
|
||||
},
|
||||
{
|
||||
"id": "gemini-2.5-pro",
|
||||
"name": "Gemini 2.5 Pro",
|
||||
"description": "Highest quality reasoning and coding. Slower, more expensive.",
|
||||
"status": "GA",
|
||||
},
|
||||
{
|
||||
"id": "gemini-3.5-flash",
|
||||
"name": "Gemini 3.5 Flash",
|
||||
"description": "Next-gen Flash. Strong reasoning, fast inference.",
|
||||
"status": "Preview",
|
||||
},
|
||||
{
|
||||
"id": "gemini-3.1-flash",
|
||||
"name": "Gemini 3.1 Flash",
|
||||
"description": "Latest Flash with thinking capabilities.",
|
||||
"status": "Preview",
|
||||
},
|
||||
]
|
||||
|
||||
def get_available_models() -> list:
|
||||
"""Return list of available text generation models."""
|
||||
return AVAILABLE_TEXT_MODELS
|
||||
Reference in new issue
Block a user