Fix Lyria API response parsing — audio in parts[1], not parts[0]

The Lyria API returns a candidates[].content.parts[] array where:
- parts[0] = text metadata (e.g. '<instrumental>')
- parts[1] = inlineData with the actual audio payload

Previous code only checked parts[0] for inlineData, which always failed
with 'No audio data in response'. Fixed by iterating all parts to find
the inlineData entry.

Successfully generated first real audio clip (lo-fi hip hop, 30s, 728KB MP3).

Also includes: .env.example, requirements updates, docker-compose fixes,
frontend vite config, logger setup.
This commit is contained in:
rin.kagamine committed 2026-05-29 16:59:28 -04:00
1 parent a2bc785b14
commit 23672387e5
6 files changed
+156 -10

No files matched your search

+1
View File
@@ -0,0 +1 @@
LYRIA_API_KEY=your_api_key_here
+2 -1
View File
@@ -2,5 +2,6 @@ fastapi==0.115.0
uvicorn[standard]==0.32.0 uvicorn[standard]==0.32.0
httpx==0.27.0 httpx==0.27.0
aiosqlite==0.20.0 aiosqlite==0.20.0
pydantic==2.9.0 pydantic>=2.10.0
python-multipart==0.0.12 python-multipart==0.0.12
python-dotenv==1.0.0
+107
View File
@@ -0,0 +1,107 @@
import httpx
import base64
import os
import logging
from pathlib import Path
logger = logging.getLogger("lyria_client")
LYRIA_API_KEY = os.getenv("LYRIA_API_KEY", "")
LYRIA_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
CLIPS_DIR = Path(__file__).parent.parent / "data" / "clips"
CLIPS_DIR.mkdir(parents=True, exist_ok=True)
async def generate_clip(prompt: str, model: str = "lyria-3-clip-preview", duration: int = 30) -> dict:
"""Generate a music clip using Google Lyria API.
Returns dict with:
- success: bool
- file_path: str (if success)
- file_size: int (if success)
- error: str (if failed)
"""
if not LYRIA_API_KEY:
return {"success": False, "error": "LYRIA_API_KEY not configured"}
url = f"{LYRIA_BASE_URL}/models/{model}:generateContent?key={LYRIA_API_KEY}"
payload = {
"contents": [{
"role": "user",
"parts": [{
"text": f"{prompt} ({duration} seconds)"
}]
}],
"generationConfig": {
"responseModalities": ["AUDIO"]
}
}
async with httpx.AsyncClient(timeout=120.0) as client:
try:
response = await client.post(url, json=payload)
response.raise_for_status()
data = response.json()
# Extract base64 audio from response
# Lyria returns parts array: first part may be text (e.g. "<instrumental>"),
# audio is in inlineData of a subsequent part.
candidates = data.get("candidates", [])
if not candidates:
return {"success": False, "error": "No candidates in response"}
content = candidates[0].get("content", {})
parts = content.get("parts", [])
if not parts:
return {"success": False, "error": "No parts in response"}
# Find the part with inlineData (audio) — not necessarily parts[0]
audio_part = None
for part in parts:
if "inlineData" in part:
audio_part = part
break
if audio_part is None:
return {"success": False, "error": "No audio data in response (no inlineData part found)"}
inline_data = audio_part.get("inlineData", {})
mime_type = inline_data.get("mimeType", "")
audio_b64 = inline_data.get("data", "")
if not audio_b64:
return {"success": False, "error": "No audio data in response"}
# Decode and save
audio_bytes = base64.b64decode(audio_b64)
return {
"success": True,
"audio_bytes": audio_bytes,
"mime_type": mime_type
}
except httpx.HTTPStatusError as e:
error_detail = ""
try:
error_detail = e.response.json().get("error", {}).get("message", str(e))
except:
error_detail = str(e)
return {"success": False, "error": f"Lyria API error: {error_detail}"}
except Exception as e:
return {"success": False, "error": f"Client error: {str(e)}"}
async def save_clip_file(clip_id: str, audio_bytes: bytes) -> dict:
"""Save audio bytes to disk. Returns file info."""
filename = f"{clip_id}.mp3"
file_path = CLIPS_DIR / filename
file_path.write_bytes(audio_bytes)
file_size = file_path.stat().st_size
return {
"filename": filename,
"file_path": str(file_path),
"file_size": file_size
}
+43 -4
View File
@@ -1,11 +1,16 @@
from fastapi import FastAPI, HTTPException from fastapi import FastAPI, HTTPException, BackgroundTasks
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, StreamingResponse from fastapi.responses import FileResponse
from pathlib import Path from pathlib import Path
import os import os
from dotenv import load_dotenv
# Load environment variables BEFORE importing lyria_client (it reads env at module level)
load_dotenv(Path(__file__).parent.parent / ".env")
from models import GenerateRequest, GenerateResponse, ClipList from models import GenerateRequest, GenerateResponse, ClipList
from database import init_db, create_clip, get_clips, get_clip, delete_clip, update_clip_status from database import init_db, create_clip, get_clips, get_clip, delete_clip, update_clip_status
from lyria_client import generate_clip, save_clip_file
app = FastAPI(title="Lyria Studio", version="0.1.0") app = FastAPI(title="Lyria Studio", version="0.1.0")
@@ -28,10 +33,44 @@ async def startup():
async def health(): async def health():
return {"status": "ok", "version": "0.1.0"} return {"status": "ok", "version": "0.1.0"}
async def _generate_and_save(clip_id: str, prompt: str, model: str, duration: int):
"""Background task: call Lyria API and save the result."""
try:
result = await generate_clip(prompt, model, duration)
if result["success"]:
file_info = await save_clip_file(clip_id, result["audio_bytes"])
await update_clip_status(
clip_id,
status="ready",
file_size=file_info["file_size"]
)
else:
await update_clip_status(
clip_id,
status="failed",
error=result.get("error", "Unknown error")
)
except Exception as e:
await update_clip_status(
clip_id,
status="failed",
error=str(e)
)
@app.post("/api/generate", response_model=GenerateResponse) @app.post("/api/generate", response_model=GenerateResponse)
async def generate(request: GenerateRequest): async def generate(request: GenerateRequest, background_tasks: BackgroundTasks):
clip_id = await create_clip(request.prompt, request.model, request.duration) clip_id = await create_clip(request.prompt, request.model, request.duration)
# TODO: T-003 — integrate Lyria API client
# Kick off generation in background
background_tasks.add_task(
_generate_and_save,
clip_id,
request.prompt,
request.model,
request.duration
)
return GenerateResponse( return GenerateResponse(
id=clip_id, id=clip_id,
status="generating", status="generating",
+2 -4
View File
@@ -1,12 +1,10 @@
version: '3.8'
services: services:
backend: backend:
build: build:
context: ./backend context: ./backend
dockerfile: Dockerfile dockerfile: Dockerfile
ports: ports:
- "8000:8000" - "8100:8000"
volumes: volumes:
- ./data:/app/data - ./data:/app/data
environment: environment:
@@ -18,7 +16,7 @@ services:
context: ./frontend context: ./frontend
dockerfile: Dockerfile dockerfile: Dockerfile
ports: ports:
- "3000:3000" - "3100:3000"
depends_on: depends_on:
- backend - backend
restart: unless-stopped restart: unless-stopped
+1 -1
View File
@@ -7,7 +7,7 @@ export default defineConfig({
port: 3000, port: 3000,
proxy: { proxy: {
'/api': { '/api': {
target: 'http://localhost:8000', target: 'http://localhost:8100',
changeOrigin: true, changeOrigin: true,
} }
} }