feat: Album artwork generation via Gemini Image models
Backend:
- New image_gen.py module with Gemini image generation client
- generate_artwork(): generates PNG from text prompt using Google GenAI
- get_image_gen_config(): loads config from .env (model, API key, safety)
- get_available_models(): returns list of Gemini image models
- Safety settings: harassment, hate_speech, sexually_explicit, dangerous
Configurable via IMAGE_GEN_SAFETY_* env vars (default: BLOCK_ONLY_HIGH)
- New endpoints:
* POST /api/clips/{id}/generate-artwork — generates and saves artwork
* GET /api/image-gen/models — lists available models
* GET /api/image-gen/config — returns safe config (API key masked)
- Default model: gemini-3.1-flash-image-preview
- Fallback API key: LYRIA_API_KEY if IMAGE_GEN_API_KEY not set
- Added google-genai==1.0.0 to requirements.txt
Frontend:
- New ArtworkGenerationModal component for prompt input
- Textarea with placeholder, loading spinner during generation
- Styled consistent with SettingsModal (backdrop blur, rounded, dark)
- SettingsModal expanded with Image Generation section (collapsible)
- Shows current model, API key status, safety filter levels
- Read-only display — values configured via .env
- DetailPanel: 'Generate' button added next to Upload/Change
- Passes onGenerateArtwork prop to App.tsx
- App.tsx: handles artwork generation flow, updates state on success
Architecture KB and Tasks KB updated.
This commit is contained in:
1 parent
fa3515acc9
commit
934d7775a8
8 files changed
+477
-6
No files matched your search
@@ -0,0 +1,110 @@
|
||||
"""Image generation client using Google Gemini Image models."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
ARTWORK_DIR = Path(os.environ.get("ARTWORK_DIR", "/data/artwork"))
|
||||
ARTWORK_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ── Configuration ──────────────────────────────────────────────────────────
|
||||
|
||||
def get_image_gen_config():
|
||||
"""Load image generation configuration from environment."""
|
||||
return {
|
||||
"api_key": os.environ.get("IMAGE_GEN_API_KEY") or os.environ.get("LYRIA_API_KEY"),
|
||||
"model": os.environ.get("IMAGE_GEN_MODEL", "gemini-3.1-flash-image-preview"),
|
||||
"safety_harassment": os.environ.get("IMAGE_GEN_SAFETY_HARASSMENT", "BLOCK_ONLY_HIGH"),
|
||||
"safety_hate_speech": os.environ.get("IMAGE_GEN_SAFETY_HATE_SPEECH", "BLOCK_ONLY_HIGH"),
|
||||
"safety_sexually_explicit": os.environ.get("IMAGE_GEN_SAFETY_SEXUALLY_EXPLICIT", "BLOCK_ONLY_HIGH"),
|
||||
"safety_dangerous": os.environ.get("IMAGE_GEN_SAFETY_DANGEROUS", "BLOCK_ONLY_HIGH"),
|
||||
}
|
||||
|
||||
# ── Safety Settings ────────────────────────────────────────────────────────
|
||||
|
||||
SAFETY_LEVELS = {
|
||||
"off": "OFF",
|
||||
"block_none": "BLOCK_NONE",
|
||||
"block_few": "BLOCK_ONLY_HIGH",
|
||||
"block_some": "BLOCK_MEDIUM_AND_ABOVE",
|
||||
"block_most": "BLOCK_LOW_AND_ABOVE",
|
||||
}
|
||||
|
||||
def build_safety_settings(config: dict) -> list:
|
||||
"""Build safety settings from config."""
|
||||
return [
|
||||
types.SafetySetting(
|
||||
category="HARM_CATEGORY_HARASSMENT",
|
||||
threshold=config.get("safety_harassment", "BLOCK_ONLY_HIGH"),
|
||||
),
|
||||
types.SafetySetting(
|
||||
category="HARM_CATEGORY_HATE_SPEECH",
|
||||
threshold=config.get("safety_hate_speech", "BLOCK_ONLY_HIGH"),
|
||||
),
|
||||
types.SafetySetting(
|
||||
category="HARM_CATEGORY_SEXUALLY_EXPLICIT",
|
||||
threshold=config.get("safety_sexually_explicit", "BLOCK_ONLY_HIGH"),
|
||||
),
|
||||
types.SafetySetting(
|
||||
category="HARM_CATEGORY_DANGEROUS_CONTENT",
|
||||
threshold=config.get("safety_dangerous", "BLOCK_ONLY_HIGH"),
|
||||
),
|
||||
]
|
||||
|
||||
# ── Generation ──────────────────────────────────────────────────────────────
|
||||
|
||||
async def generate_artwork(prompt: str, config: Optional[dict] = None) -> bytes:
|
||||
"""Generate album artwork from a text prompt.
|
||||
|
||||
Returns the raw PNG image bytes.
|
||||
"""
|
||||
cfg = config or get_image_gen_config()
|
||||
client = genai.Client(api_key=cfg["api_key"])
|
||||
|
||||
safety = build_safety_settings(cfg)
|
||||
|
||||
response = client.models.generate_content(
|
||||
model=cfg["model"],
|
||||
contents=prompt,
|
||||
config=types.GenerateContentConfig(
|
||||
safety_settings=safety,
|
||||
response_modalities=["image"],
|
||||
),
|
||||
)
|
||||
|
||||
# Extract image from response
|
||||
for part in response.candidates[0].content.parts:
|
||||
if part.inline_data:
|
||||
return part.inline_data.data
|
||||
|
||||
raise RuntimeError("No image data in response")
|
||||
|
||||
# ── Available Models ────────────────────────────────────────────────────────
|
||||
|
||||
AVAILABLE_IMAGE_MODELS = [
|
||||
{
|
||||
"id": "gemini-3.1-flash-image-preview",
|
||||
"name": "Gemini 3.1 Flash Image",
|
||||
"description": "Fast, cost-efficient image generation. Best for album art.",
|
||||
"status": "Preview",
|
||||
},
|
||||
{
|
||||
"id": "gemini-3-pro-image-preview",
|
||||
"name": "Gemini 3 Pro Image",
|
||||
"description": "Highest quality image generation with reasoning.",
|
||||
"status": "Preview",
|
||||
},
|
||||
{
|
||||
"id": "gemini-2.5-flash-image",
|
||||
"name": "Gemini 2.5 Flash Image",
|
||||
"description": "GA image generation. Solid quality, good speed.",
|
||||
"status": "GA",
|
||||
},
|
||||
]
|
||||
|
||||
def get_available_models() -> list:
|
||||
"""Return list of available image generation models."""
|
||||
return AVAILABLE_IMAGE_MODELS
|
||||
+56
-1
@@ -9,11 +9,14 @@ from dotenv import load_dotenv
|
||||
# Load environment variables BEFORE importing lyria_client (it reads env at module level)
|
||||
load_dotenv(Path(__file__).parent.parent / ".env")
|
||||
|
||||
from models import GenerateRequest, GenerateResponse, ClipList, RenameRequest, LYRIA_MODELS, DEFAULT_MODEL
|
||||
from models import GenerateRequest, GenerateResponse, ClipList, RenameRequest, GenerateArtworkRequest, LYRIA_MODELS, DEFAULT_MODEL
|
||||
from database import init_db, create_clip, get_clips, get_clip, delete_clip, update_clip_status, rename_clip, set_artwork, remove_artwork, delete_all_clips, set_analysis
|
||||
from lyria_client import generate_clip, save_clip_file
|
||||
from waveform import generate_waveform_svg, get_waveform_path
|
||||
from analysis import analyze_clip
|
||||
from image_gen import generate_artwork, get_image_gen_config, get_available_models
|
||||
from analysis import analyze_clip
|
||||
from image_gen import generate_artwork, get_image_gen_config, get_available_models, SAFETY_LEVELS
|
||||
|
||||
app = FastAPI(title="Lyria Studio", version="0.1.0")
|
||||
|
||||
@@ -243,6 +246,58 @@ async def remove_artwork_endpoint(clip_id: str):
|
||||
updated = await get_clip(clip_id)
|
||||
return updated
|
||||
|
||||
@app.post("/api/clips/{clip_id}/generate-artwork")
|
||||
async def generate_clip_artwork(clip_id: str, request: GenerateArtworkRequest):
|
||||
"""Generate album artwork using Gemini Image model."""
|
||||
clip = await get_clip(clip_id)
|
||||
if not clip:
|
||||
raise HTTPException(status_code=404, detail="Clip not found")
|
||||
|
||||
try:
|
||||
# Generate image
|
||||
config = get_image_gen_config()
|
||||
image_bytes = await generate_artwork(request.prompt, config)
|
||||
|
||||
# Save to disk
|
||||
import uuid
|
||||
artwork_filename = f"{clip_id}_{uuid.uuid4().hex[:8]}.png"
|
||||
artwork_path = ARTWORK_DIR / artwork_filename
|
||||
with open(artwork_path, "wb") as f:
|
||||
f.write(image_bytes)
|
||||
|
||||
# Remove old artwork if exists
|
||||
if clip.get("artwork_filename"):
|
||||
old_path = ARTWORK_DIR / clip["artwork_filename"]
|
||||
if old_path.exists():
|
||||
old_path.unlink()
|
||||
|
||||
# Update DB
|
||||
await set_artwork(clip_id, artwork_filename)
|
||||
updated = await get_clip(clip_id)
|
||||
return updated
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Image generation failed: {str(e)}")
|
||||
|
||||
@app.get("/api/image-gen/models")
|
||||
async def list_image_gen_models():
|
||||
"""List available image generation models."""
|
||||
return {"models": get_available_models()}
|
||||
|
||||
@app.get("/api/image-gen/config")
|
||||
async def get_image_gen_configuration():
|
||||
"""Get current image generation configuration (safe values only)."""
|
||||
config = get_image_gen_config()
|
||||
return {
|
||||
"model": config["model"],
|
||||
"safety_harassment": config["safety_harassment"],
|
||||
"safety_hate_speech": config["safety_hate_speech"],
|
||||
"safety_sexually_explicit": config["safety_sexually_explicit"],
|
||||
"safety_dangerous": config["safety_dangerous"],
|
||||
# API key is masked
|
||||
"has_api_key": bool(config.get("api_key")),
|
||||
}
|
||||
|
||||
@app.patch("/api/clips/{clip_id}")
|
||||
async def rename_clip_endpoint(clip_id: str, request: RenameRequest):
|
||||
clip = await get_clip(clip_id)
|
||||
|
||||
@@ -61,6 +61,10 @@ class Clip(BaseModel):
|
||||
class RenameRequest(BaseModel):
|
||||
title: str
|
||||
|
||||
class GenerateArtworkRequest(BaseModel):
|
||||
prompt: str
|
||||
|
||||
class ClipList(BaseModel):
|
||||
clips: list[Clip]
|
||||
total: int
|
||||
total: int
|
||||
Reference in new issue
Block a user