Fix Lyria API response parsing — audio in parts[1], not parts[0]
The Lyria API returns a candidates[].content.parts[] array where: - parts[0] = text metadata (e.g. '<instrumental>') - parts[1] = inlineData with the actual audio payload Previous code only checked parts[0] for inlineData, which always failed with 'No audio data in response'. Fixed by iterating all parts to find the inlineData entry. Successfully generated first real audio clip (lo-fi hip hop, 30s, 728KB MP3). Also includes: .env.example, requirements updates, docker-compose fixes, frontend vite config, logger setup.
This commit is contained in:
1 parent
a2bc785b14
commit
23672387e5
6 files changed
+156
-10
No files matched your search
@@ -0,0 +1 @@
|
|||||||
|
LYRIA_API_KEY=your_api_key_here
|
||||||
@@ -2,5 +2,6 @@ fastapi==0.115.0
|
|||||||
uvicorn[standard]==0.32.0
|
uvicorn[standard]==0.32.0
|
||||||
httpx==0.27.0
|
httpx==0.27.0
|
||||||
aiosqlite==0.20.0
|
aiosqlite==0.20.0
|
||||||
pydantic==2.9.0
|
pydantic>=2.10.0
|
||||||
python-multipart==0.0.12
|
python-multipart==0.0.12
|
||||||
|
python-dotenv==1.0.0
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
import httpx
|
||||||
|
import base64
|
||||||
|
import os
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
logger = logging.getLogger("lyria_client")
|
||||||
|
|
||||||
|
LYRIA_API_KEY = os.getenv("LYRIA_API_KEY", "")
|
||||||
|
LYRIA_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
|
||||||
|
|
||||||
|
CLIPS_DIR = Path(__file__).parent.parent / "data" / "clips"
|
||||||
|
CLIPS_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
async def generate_clip(prompt: str, model: str = "lyria-3-clip-preview", duration: int = 30) -> dict:
|
||||||
|
"""Generate a music clip using Google Lyria API.
|
||||||
|
|
||||||
|
Returns dict with:
|
||||||
|
- success: bool
|
||||||
|
- file_path: str (if success)
|
||||||
|
- file_size: int (if success)
|
||||||
|
- error: str (if failed)
|
||||||
|
"""
|
||||||
|
if not LYRIA_API_KEY:
|
||||||
|
return {"success": False, "error": "LYRIA_API_KEY not configured"}
|
||||||
|
|
||||||
|
url = f"{LYRIA_BASE_URL}/models/{model}:generateContent?key={LYRIA_API_KEY}"
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"contents": [{
|
||||||
|
"role": "user",
|
||||||
|
"parts": [{
|
||||||
|
"text": f"{prompt} ({duration} seconds)"
|
||||||
|
}]
|
||||||
|
}],
|
||||||
|
"generationConfig": {
|
||||||
|
"responseModalities": ["AUDIO"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||||
|
try:
|
||||||
|
response = await client.post(url, json=payload)
|
||||||
|
response.raise_for_status()
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
# Extract base64 audio from response
|
||||||
|
# Lyria returns parts array: first part may be text (e.g. "<instrumental>"),
|
||||||
|
# audio is in inlineData of a subsequent part.
|
||||||
|
candidates = data.get("candidates", [])
|
||||||
|
if not candidates:
|
||||||
|
return {"success": False, "error": "No candidates in response"}
|
||||||
|
|
||||||
|
content = candidates[0].get("content", {})
|
||||||
|
parts = content.get("parts", [])
|
||||||
|
if not parts:
|
||||||
|
return {"success": False, "error": "No parts in response"}
|
||||||
|
|
||||||
|
# Find the part with inlineData (audio) — not necessarily parts[0]
|
||||||
|
audio_part = None
|
||||||
|
for part in parts:
|
||||||
|
if "inlineData" in part:
|
||||||
|
audio_part = part
|
||||||
|
break
|
||||||
|
|
||||||
|
if audio_part is None:
|
||||||
|
return {"success": False, "error": "No audio data in response (no inlineData part found)"}
|
||||||
|
|
||||||
|
inline_data = audio_part.get("inlineData", {})
|
||||||
|
mime_type = inline_data.get("mimeType", "")
|
||||||
|
audio_b64 = inline_data.get("data", "")
|
||||||
|
|
||||||
|
if not audio_b64:
|
||||||
|
return {"success": False, "error": "No audio data in response"}
|
||||||
|
|
||||||
|
# Decode and save
|
||||||
|
audio_bytes = base64.b64decode(audio_b64)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"audio_bytes": audio_bytes,
|
||||||
|
"mime_type": mime_type
|
||||||
|
}
|
||||||
|
|
||||||
|
except httpx.HTTPStatusError as e:
|
||||||
|
error_detail = ""
|
||||||
|
try:
|
||||||
|
error_detail = e.response.json().get("error", {}).get("message", str(e))
|
||||||
|
except:
|
||||||
|
error_detail = str(e)
|
||||||
|
return {"success": False, "error": f"Lyria API error: {error_detail}"}
|
||||||
|
except Exception as e:
|
||||||
|
return {"success": False, "error": f"Client error: {str(e)}"}
|
||||||
|
|
||||||
|
async def save_clip_file(clip_id: str, audio_bytes: bytes) -> dict:
|
||||||
|
"""Save audio bytes to disk. Returns file info."""
|
||||||
|
filename = f"{clip_id}.mp3"
|
||||||
|
file_path = CLIPS_DIR / filename
|
||||||
|
|
||||||
|
file_path.write_bytes(audio_bytes)
|
||||||
|
file_size = file_path.stat().st_size
|
||||||
|
|
||||||
|
return {
|
||||||
|
"filename": filename,
|
||||||
|
"file_path": str(file_path),
|
||||||
|
"file_size": file_size
|
||||||
|
}
|
||||||
+43
-4
@@ -1,11 +1,16 @@
|
|||||||
from fastapi import FastAPI, HTTPException
|
from fastapi import FastAPI, HTTPException, BackgroundTasks
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
from fastapi.responses import FileResponse, StreamingResponse
|
from fastapi.responses import FileResponse
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import os
|
import os
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
|
# Load environment variables BEFORE importing lyria_client (it reads env at module level)
|
||||||
|
load_dotenv(Path(__file__).parent.parent / ".env")
|
||||||
|
|
||||||
from models import GenerateRequest, GenerateResponse, ClipList
|
from models import GenerateRequest, GenerateResponse, ClipList
|
||||||
from database import init_db, create_clip, get_clips, get_clip, delete_clip, update_clip_status
|
from database import init_db, create_clip, get_clips, get_clip, delete_clip, update_clip_status
|
||||||
|
from lyria_client import generate_clip, save_clip_file
|
||||||
|
|
||||||
app = FastAPI(title="Lyria Studio", version="0.1.0")
|
app = FastAPI(title="Lyria Studio", version="0.1.0")
|
||||||
|
|
||||||
@@ -28,10 +33,44 @@ async def startup():
|
|||||||
async def health():
|
async def health():
|
||||||
return {"status": "ok", "version": "0.1.0"}
|
return {"status": "ok", "version": "0.1.0"}
|
||||||
|
|
||||||
|
async def _generate_and_save(clip_id: str, prompt: str, model: str, duration: int):
|
||||||
|
"""Background task: call Lyria API and save the result."""
|
||||||
|
try:
|
||||||
|
result = await generate_clip(prompt, model, duration)
|
||||||
|
|
||||||
|
if result["success"]:
|
||||||
|
file_info = await save_clip_file(clip_id, result["audio_bytes"])
|
||||||
|
await update_clip_status(
|
||||||
|
clip_id,
|
||||||
|
status="ready",
|
||||||
|
file_size=file_info["file_size"]
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
await update_clip_status(
|
||||||
|
clip_id,
|
||||||
|
status="failed",
|
||||||
|
error=result.get("error", "Unknown error")
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
await update_clip_status(
|
||||||
|
clip_id,
|
||||||
|
status="failed",
|
||||||
|
error=str(e)
|
||||||
|
)
|
||||||
|
|
||||||
@app.post("/api/generate", response_model=GenerateResponse)
|
@app.post("/api/generate", response_model=GenerateResponse)
|
||||||
async def generate(request: GenerateRequest):
|
async def generate(request: GenerateRequest, background_tasks: BackgroundTasks):
|
||||||
clip_id = await create_clip(request.prompt, request.model, request.duration)
|
clip_id = await create_clip(request.prompt, request.model, request.duration)
|
||||||
# TODO: T-003 — integrate Lyria API client
|
|
||||||
|
# Kick off generation in background
|
||||||
|
background_tasks.add_task(
|
||||||
|
_generate_and_save,
|
||||||
|
clip_id,
|
||||||
|
request.prompt,
|
||||||
|
request.model,
|
||||||
|
request.duration
|
||||||
|
)
|
||||||
|
|
||||||
return GenerateResponse(
|
return GenerateResponse(
|
||||||
id=clip_id,
|
id=clip_id,
|
||||||
status="generating",
|
status="generating",
|
||||||
|
|||||||
+2
-4
@@ -1,12 +1,10 @@
|
|||||||
version: '3.8'
|
|
||||||
|
|
||||||
services:
|
services:
|
||||||
backend:
|
backend:
|
||||||
build:
|
build:
|
||||||
context: ./backend
|
context: ./backend
|
||||||
dockerfile: Dockerfile
|
dockerfile: Dockerfile
|
||||||
ports:
|
ports:
|
||||||
- "8000:8000"
|
- "8100:8000"
|
||||||
volumes:
|
volumes:
|
||||||
- ./data:/app/data
|
- ./data:/app/data
|
||||||
environment:
|
environment:
|
||||||
@@ -18,7 +16,7 @@ services:
|
|||||||
context: ./frontend
|
context: ./frontend
|
||||||
dockerfile: Dockerfile
|
dockerfile: Dockerfile
|
||||||
ports:
|
ports:
|
||||||
- "3000:3000"
|
- "3100:3000"
|
||||||
depends_on:
|
depends_on:
|
||||||
- backend
|
- backend
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
@@ -7,7 +7,7 @@ export default defineConfig({
|
|||||||
port: 3000,
|
port: 3000,
|
||||||
proxy: {
|
proxy: {
|
||||||
'/api': {
|
'/api': {
|
||||||
target: 'http://localhost:8000',
|
target: 'http://localhost:8100',
|
||||||
changeOrigin: true,
|
changeOrigin: true,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user