feat: BPM and musical key detection for all clips
Backend:
- New analysis.py module using librosa for BPM + key detection
- detect_bpm(): librosa.beat.beat_track() returns tempo
- detect_key(): Krumhansl-Schmuckler key profiles on chroma features
- analyze_clip(): combined call returning {bpm, key}
- New dependency: librosa==0.10.2 + soundfile==0.12.1
- Database migrations: bpm (REAL) and key (TEXT) columns added
- New set_analysis() function in database.py
- New endpoint: GET /api/clips/{id}/analysis
- Returns cached analysis if available
- Computes on-demand if missing, caches result
- 400 if clip not ready, 404 if no audio file
Frontend:
- Clip interface extended with bpm?: number | null, key?: string | null
- Gallery: BPM and key badges shown in ribbon and tile views
- BPM badge: lyria-accent background
- Key badge: white/5 background
- DetailPanel: BPM and key shown in metadata grid when available
- App.tsx: handleSelectClip() fetches analysis on-demand when opening
detail panel if not already cached
Docs:
- Architecture KB updated with Audio Analysis section
- Tasks KB: T-028 marked In Progress → will mark Done after commit
This commit is contained in:
1 parent
2d47f77820
commit
1e21b7dcc8
7 files changed
+187
-3
No files matched your search
@@ -6,3 +6,5 @@ pydantic>=2.10.0
|
|||||||
python-multipart==0.0.12
|
python-multipart==0.0.12
|
||||||
python-dotenv==1.0.0
|
python-dotenv==1.0.0
|
||||||
numpy==2.2.0
|
numpy==2.2.0
|
||||||
|
librosa==0.10.2
|
||||||
|
soundfile==0.12.1
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
"""Audio analysis: BPM and musical key detection using librosa."""
|
||||||
|
|
||||||
|
import librosa
|
||||||
|
import numpy as np
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# Krumhansl-Schmuckler key profiles (normalized)
|
||||||
|
# Each profile is a 12-element vector representing the strength of each pitch class
|
||||||
|
# in the key. Index 0 = C, 1 = C#, ..., 11 = B
|
||||||
|
|
||||||
|
MAJOR_PROFILE = np.array([
|
||||||
|
6.35, 2.23, 3.48, 2.33, 4.38, 4.09, 2.52, 5.19, 2.39, 3.66, 2.29, 2.88
|
||||||
|
])
|
||||||
|
|
||||||
|
MINOR_PROFILE = np.array([
|
||||||
|
6.33, 2.68, 3.52, 5.38, 2.60, 3.53, 2.54, 4.75, 3.98, 2.69, 3.34, 3.17
|
||||||
|
])
|
||||||
|
|
||||||
|
KEY_NAMES = [
|
||||||
|
'C major', 'C# major', 'D major', 'D# major', 'E major', 'F major',
|
||||||
|
'F# major', 'G major', 'G# major', 'A major', 'A# major', 'B major',
|
||||||
|
'C minor', 'C# minor', 'D minor', 'D# minor', 'E minor', 'F minor',
|
||||||
|
'F# minor', 'G minor', 'G# minor', 'A minor', 'A# minor', 'B minor',
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def detect_bpm(audio_path: str | Path) -> float:
|
||||||
|
"""Detect BPM (tempo) of an audio file using librosa."""
|
||||||
|
y, sr = librosa.load(str(audio_path), sr=None)
|
||||||
|
tempo, _ = librosa.beat.beat_track(y=y, sr=sr)
|
||||||
|
# librosa returns tempo as a numpy array or scalar depending on version
|
||||||
|
if isinstance(tempo, np.ndarray):
|
||||||
|
return float(tempo[0])
|
||||||
|
return float(tempo)
|
||||||
|
|
||||||
|
|
||||||
|
def detect_key(audio_path: str | Path) -> str:
|
||||||
|
"""Detect musical key using Krumhansl-Schmuckler key profiles."""
|
||||||
|
y, sr = librosa.load(str(audio_path), sr=None)
|
||||||
|
|
||||||
|
# Extract chroma features
|
||||||
|
chroma = librosa.feature.chroma_stft(y=y, sr=sr)
|
||||||
|
|
||||||
|
# Average chroma across time
|
||||||
|
chroma_avg = np.mean(chroma, axis=1)
|
||||||
|
|
||||||
|
# Normalize
|
||||||
|
chroma_avg = chroma_avg / (np.sum(chroma_avg) + 1e-10)
|
||||||
|
|
||||||
|
# Correlate with all 24 key profiles (12 major + 12 minor, each rotated)
|
||||||
|
correlations = []
|
||||||
|
|
||||||
|
for i in range(12):
|
||||||
|
# Rotate profiles to test each tonic
|
||||||
|
major_rotated = np.roll(MAJOR_PROFILE, i)
|
||||||
|
minor_rotated = np.roll(MINOR_PROFILE, i)
|
||||||
|
|
||||||
|
# Pearson correlation
|
||||||
|
major_corr = np.corrcoef(chroma_avg, major_rotated)[0, 1]
|
||||||
|
minor_corr = np.corrcoef(chroma_avg, minor_rotated)[0, 1]
|
||||||
|
|
||||||
|
correlations.append(major_corr)
|
||||||
|
correlations.append(minor_corr)
|
||||||
|
|
||||||
|
# Find best match
|
||||||
|
best_idx = int(np.argmax(correlations))
|
||||||
|
return KEY_NAMES[best_idx]
|
||||||
|
|
||||||
|
|
||||||
|
def analyze_clip(audio_path: str | Path) -> dict:
|
||||||
|
"""Full analysis: BPM and key in one call."""
|
||||||
|
audio_path = Path(audio_path)
|
||||||
|
if not audio_path.exists():
|
||||||
|
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||||
|
|
||||||
|
bpm = detect_bpm(audio_path)
|
||||||
|
key = detect_key(audio_path)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'bpm': round(bpm, 1),
|
||||||
|
'key': key,
|
||||||
|
}
|
||||||
@@ -68,6 +68,16 @@ async def init_db():
|
|||||||
await db.execute("ALTER TABLE clips ADD COLUMN artwork_filename TEXT")
|
await db.execute("ALTER TABLE clips ADD COLUMN artwork_filename TEXT")
|
||||||
except Exception:
|
except Exception:
|
||||||
pass # Column already exists
|
pass # Column already exists
|
||||||
|
# Migration: add bpm column if it doesn't exist
|
||||||
|
try:
|
||||||
|
await db.execute("ALTER TABLE clips ADD COLUMN bpm REAL")
|
||||||
|
except Exception:
|
||||||
|
pass # Column already exists
|
||||||
|
# Migration: add key column if it doesn't exist
|
||||||
|
try:
|
||||||
|
await db.execute("ALTER TABLE clips ADD COLUMN key TEXT")
|
||||||
|
except Exception:
|
||||||
|
pass # Column already exists
|
||||||
await db.commit()
|
await db.commit()
|
||||||
|
|
||||||
async def create_clip(prompt: str, model: str, duration: int) -> str:
|
async def create_clip(prompt: str, model: str, duration: int) -> str:
|
||||||
@@ -148,6 +158,14 @@ async def remove_artwork(clip_id: str):
|
|||||||
)
|
)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
|
|
||||||
|
async def set_analysis(clip_id: str, bpm: float, key: str):
|
||||||
|
async with aiosqlite.connect(DB_PATH) as db:
|
||||||
|
await db.execute(
|
||||||
|
"UPDATE clips SET bpm = ?, key = ? WHERE id = ?",
|
||||||
|
(bpm, key, clip_id)
|
||||||
|
)
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
async def delete_clip(clip_id: str):
|
async def delete_clip(clip_id: str):
|
||||||
async with aiosqlite.connect(DB_PATH) as db:
|
async with aiosqlite.connect(DB_PATH) as db:
|
||||||
await db.execute("DELETE FROM clips WHERE id = ?", (clip_id,))
|
await db.execute("DELETE FROM clips WHERE id = ?", (clip_id,))
|
||||||
|
|||||||
+28
-1
@@ -10,9 +10,10 @@ from dotenv import load_dotenv
|
|||||||
load_dotenv(Path(__file__).parent.parent / ".env")
|
load_dotenv(Path(__file__).parent.parent / ".env")
|
||||||
|
|
||||||
from models import GenerateRequest, GenerateResponse, ClipList, RenameRequest, LYRIA_MODELS, DEFAULT_MODEL
|
from models import GenerateRequest, GenerateResponse, ClipList, RenameRequest, LYRIA_MODELS, DEFAULT_MODEL
|
||||||
from database import init_db, create_clip, get_clips, get_clip, delete_clip, update_clip_status, rename_clip, set_artwork, remove_artwork, delete_all_clips
|
from database import init_db, create_clip, get_clips, get_clip, delete_clip, update_clip_status, rename_clip, set_artwork, remove_artwork, delete_all_clips, set_analysis
|
||||||
from lyria_client import generate_clip, save_clip_file
|
from lyria_client import generate_clip, save_clip_file
|
||||||
from waveform import generate_waveform_svg, get_waveform_path
|
from waveform import generate_waveform_svg, get_waveform_path
|
||||||
|
from analysis import analyze_clip
|
||||||
|
|
||||||
app = FastAPI(title="Lyria Studio", version="0.1.0")
|
app = FastAPI(title="Lyria Studio", version="0.1.0")
|
||||||
|
|
||||||
@@ -228,6 +229,32 @@ async def rename_clip_endpoint(clip_id: str, request: RenameRequest):
|
|||||||
|
|
||||||
# ─── Admin Endpoints ──────────────────────────────────────────────
|
# ─── Admin Endpoints ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
@app.get("/api/clips/{clip_id}/analysis")
|
||||||
|
async def get_clip_analysis(clip_id: str):
|
||||||
|
"""Get BPM and key analysis for a clip. Computes on-demand if not cached."""
|
||||||
|
clip = await get_clip(clip_id)
|
||||||
|
if not clip:
|
||||||
|
raise HTTPException(status_code=404, detail="Clip not found")
|
||||||
|
if clip.get("status") != "ready":
|
||||||
|
raise HTTPException(status_code=400, detail="Clip is not ready for analysis")
|
||||||
|
|
||||||
|
# Return cached analysis if available
|
||||||
|
if clip.get("bpm") and clip.get("key"):
|
||||||
|
return {"bpm": clip["bpm"], "key": clip["key"]}
|
||||||
|
|
||||||
|
# Compute analysis
|
||||||
|
mp3_path = CLIPS_DIR / clip["filename"]
|
||||||
|
if not mp3_path.exists():
|
||||||
|
raise HTTPException(status_code=404, detail="Audio file not found")
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = analyze_clip(mp3_path)
|
||||||
|
# Cache in DB
|
||||||
|
await set_analysis(clip_id, result["bpm"], result["key"])
|
||||||
|
return result
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(status_code=500, detail=f"Analysis failed: {str(e)}")
|
||||||
|
|
||||||
@app.post("/api/admin/clear-waveforms")
|
@app.post("/api/admin/clear-waveforms")
|
||||||
async def clear_waveforms():
|
async def clear_waveforms():
|
||||||
"""Delete all cached waveform SVGs. They regenerate on next view."""
|
"""Delete all cached waveform SVGs. They regenerate on next view."""
|
||||||
|
|||||||
+20
-1
@@ -15,6 +15,8 @@ interface Clip {
|
|||||||
error_message?: string
|
error_message?: string
|
||||||
file_size_bytes?: number
|
file_size_bytes?: number
|
||||||
artwork_filename?: string | null
|
artwork_filename?: string | null
|
||||||
|
bpm?: number | null
|
||||||
|
key?: string | null
|
||||||
created_at: string
|
created_at: string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -119,6 +121,23 @@ function App() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const handleSelectClip = async (clip: Clip) => {
|
||||||
|
setSelectedClip(clip)
|
||||||
|
// Fetch analysis if not already cached
|
||||||
|
if (!clip.bpm || !clip.key) {
|
||||||
|
try {
|
||||||
|
const res = await fetch(`/api/clips/${clip.id}/analysis`)
|
||||||
|
if (res.ok) {
|
||||||
|
const analysis = await res.json()
|
||||||
|
setClips(prev => prev.map(c => c.id === clip.id ? { ...c, bpm: analysis.bpm, key: analysis.key } : c))
|
||||||
|
setSelectedClip({ ...clip, bpm: analysis.bpm, key: analysis.key })
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Failed to fetch analysis:', err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const handleClearWaveforms = async () => {
|
const handleClearWaveforms = async () => {
|
||||||
try {
|
try {
|
||||||
await fetch('/api/admin/clear-waveforms', { method: 'POST' })
|
await fetch('/api/admin/clear-waveforms', { method: 'POST' })
|
||||||
@@ -174,7 +193,7 @@ function App() {
|
|||||||
onPlay={handlePlay}
|
onPlay={handlePlay}
|
||||||
onDelete={handleDelete}
|
onDelete={handleDelete}
|
||||||
onRename={handleRename}
|
onRename={handleRename}
|
||||||
onSelect={(clip) => setSelectedClip(clip)}
|
onSelect={handleSelectClip}
|
||||||
onUploadArtwork={handleUploadArtwork}
|
onUploadArtwork={handleUploadArtwork}
|
||||||
currentClipId={currentClip?.id}
|
currentClipId={currentClip?.id}
|
||||||
/>
|
/>
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ interface Clip {
|
|||||||
error_message?: string
|
error_message?: string
|
||||||
file_size_bytes?: number
|
file_size_bytes?: number
|
||||||
artwork_filename?: string | null
|
artwork_filename?: string | null
|
||||||
|
bpm?: number | null
|
||||||
|
key?: string | null
|
||||||
created_at: string
|
created_at: string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -258,6 +260,18 @@ export default function DetailPanel({ clip, onClose, onRename, onDelete, onUploa
|
|||||||
<p className="text-sm text-white">{(clip.file_size_bytes / 1024).toFixed(0)} KB</p>
|
<p className="text-sm text-white">{(clip.file_size_bytes / 1024).toFixed(0)} KB</p>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
{clip.bpm && (
|
||||||
|
<div>
|
||||||
|
<label className="text-xs text-lyria-muted uppercase tracking-wider block">BPM</label>
|
||||||
|
<p className="text-sm text-white">{Math.round(clip.bpm)}</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{clip.key && (
|
||||||
|
<div>
|
||||||
|
<label className="text-xs text-lyria-muted uppercase tracking-wider block">Key</label>
|
||||||
|
<p className="text-sm text-white">{clip.key}</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
<div>
|
<div>
|
||||||
<label className="text-xs text-lyria-muted uppercase tracking-wider block">Created</label>
|
<label className="text-xs text-lyria-muted uppercase tracking-wider block">Created</label>
|
||||||
<p className="text-sm text-white">{formatDate(clip.created_at)}</p>
|
<p className="text-sm text-white">{formatDate(clip.created_at)}</p>
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ interface Clip {
|
|||||||
error_message?: string
|
error_message?: string
|
||||||
file_size_bytes?: number
|
file_size_bytes?: number
|
||||||
artwork_filename?: string | null
|
artwork_filename?: string | null
|
||||||
|
bpm?: number | null
|
||||||
|
key?: string | null
|
||||||
created_at: string
|
created_at: string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -244,6 +246,16 @@ export default function Gallery({ clips, onPlay, onDelete, onRename, onSelect, o
|
|||||||
<span className="text-xs text-lyria-muted">{clip.duration_seconds}s</span>
|
<span className="text-xs text-lyria-muted">{clip.duration_seconds}s</span>
|
||||||
<span className="text-xs text-lyria-muted">{modelName(clip.model)}</span>
|
<span className="text-xs text-lyria-muted">{modelName(clip.model)}</span>
|
||||||
<span className="text-xs text-lyria-muted">{formatDate(clip.created_at)}</span>
|
<span className="text-xs text-lyria-muted">{formatDate(clip.created_at)}</span>
|
||||||
|
{clip.bpm && (
|
||||||
|
<span className="text-xs px-1.5 py-0.5 rounded bg-lyria-accent/10 text-lyria-accent">
|
||||||
|
{Math.round(clip.bpm)} BPM
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
{clip.key && (
|
||||||
|
<span className="text-xs px-1.5 py-0.5 rounded bg-white/5 text-lyria-muted">
|
||||||
|
{clip.key}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
{clip.status === 'failed' && clip.error_message && (
|
{clip.status === 'failed' && clip.error_message && (
|
||||||
<p className="text-xs text-red-400/80 mt-1 break-words">
|
<p className="text-xs text-red-400/80 mt-1 break-words">
|
||||||
@@ -362,7 +374,7 @@ export default function Gallery({ clips, onPlay, onDelete, onRename, onSelect, o
|
|||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Meta */}
|
{/* Meta */}
|
||||||
<div className="flex items-center gap-1.5 mt-auto">
|
<div className="flex items-center gap-1.5 mt-auto flex-wrap">
|
||||||
<span className={`text-[10px] px-1 py-0.5 rounded ${
|
<span className={`text-[10px] px-1 py-0.5 rounded ${
|
||||||
clip.status === 'ready'
|
clip.status === 'ready'
|
||||||
? 'bg-green-500/20 text-green-400'
|
? 'bg-green-500/20 text-green-400'
|
||||||
@@ -374,6 +386,16 @@ export default function Gallery({ clips, onPlay, onDelete, onRename, onSelect, o
|
|||||||
</span>
|
</span>
|
||||||
<span className="text-[10px] text-lyria-muted">{clip.duration_seconds}s</span>
|
<span className="text-[10px] text-lyria-muted">{clip.duration_seconds}s</span>
|
||||||
<span className="text-[10px] text-lyria-muted">{modelName(clip.model)}</span>
|
<span className="text-[10px] text-lyria-muted">{modelName(clip.model)}</span>
|
||||||
|
{clip.bpm && (
|
||||||
|
<span className="text-[10px] px-1 py-0.5 rounded bg-lyria-accent/10 text-lyria-accent">
|
||||||
|
{Math.round(clip.bpm)} BPM
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
{clip.key && (
|
||||||
|
<span className="text-[10px] px-1 py-0.5 rounded bg-white/5 text-lyria-muted">
|
||||||
|
{clip.key}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Error */}
|
{/* Error */}
|
||||||
|
|||||||
Reference in new issue
Block a user