Add audio-to-MIDI import via Basic Pitch (MP3, WAV, M4A, OGG, FLAC, AAC), fix piano remount on view change
This commit is contained in:
Fabricio Giglio
@@ -30,6 +30,7 @@ export function TransportControls() {
|
||||
loadMidiFile,
|
||||
loadMusicXmlFile,
|
||||
loadPdfFile,
|
||||
loadAudioFile,
|
||||
isLoadingDocument,
|
||||
loadingMessage,
|
||||
exportMidi,
|
||||
@@ -87,6 +88,7 @@ export function TransportControls() {
|
||||
if (file.name.match(/\.midi?$/i)) loadMidiFile(file);
|
||||
else if (file.name.match(/\.(xml|mxl)$/i)) loadMusicXmlFile(file);
|
||||
else if (file.name.match(/\.pdf$/i)) loadPdfFile(file);
|
||||
else if (file.name.match(/\.(mp3|wav|m4a|ogg|flac|aac)$/i)) loadAudioFile(file);
|
||||
};
|
||||
|
||||
const total = doc?.totalDuration ?? 0;
|
||||
@@ -209,8 +211,8 @@ export function TransportControls() {
|
||||
>
|
||||
{/* ── Zone 1: File import - accent-styled to stand out ── */}
|
||||
<label
|
||||
title="Open MIDI, MusicXML, or PDF sheet music"
|
||||
aria-label="Import MIDI, MusicXML, or PDF file"
|
||||
title="Open MIDI, MusicXML, PDF sheet music, or audio recording (MP3/WAV/etc.)"
|
||||
aria-label="Import MIDI, MusicXML, PDF, or audio file"
|
||||
style={{
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
@@ -243,7 +245,7 @@ export function TransportControls() {
|
||||
)}
|
||||
<input
|
||||
type="file"
|
||||
accept=".mid,.midi,.xml,.mxl,.pdf"
|
||||
accept=".mid,.midi,.xml,.mxl,.pdf,.mp3,.wav,.m4a,.ogg,.flac,.aac"
|
||||
className="hidden"
|
||||
onChange={(e) => {
|
||||
const f = e.target.files?.[0];
|
||||
|
||||
@@ -140,6 +140,7 @@ export interface AppState {
|
||||
loadMidiFile: (file: File) => Promise<void>;
|
||||
loadMusicXmlFile: (file: File) => Promise<void>;
|
||||
loadPdfFile: (file: File) => Promise<void>;
|
||||
loadAudioFile: (file: File) => Promise<void>;
|
||||
generateFingering: () => Promise<void>;
|
||||
exportMidi: () => void;
|
||||
exportMusicXml: () => void;
|
||||
@@ -436,6 +437,50 @@ export const useAppStore = create<AppState>((set, get) => {
|
||||
}
|
||||
},
|
||||
|
||||
loadAudioFile: async (file: File) => {
|
||||
const id = crypto.randomUUID();
|
||||
const rawTitle = file.name.replace(/\.(mp3|wav|m4a|ogg|flac|aac)$/i, "");
|
||||
const title = rawTitle.replace(/[\x00-\x1f\x7f/\\]/g, "").slice(0, 200) || "Untitled";
|
||||
|
||||
set({
|
||||
isLoadingDocument: true,
|
||||
loadingMessage: "Transcribing audio…",
|
||||
loadError: null,
|
||||
});
|
||||
|
||||
try {
|
||||
const form = new FormData();
|
||||
form.append("file", file, file.name);
|
||||
|
||||
// Basic Pitch can take 10-60 s on the backend; no client-side fetch
|
||||
// timeout - the server's subprocess timeout is the effective limit.
|
||||
const res = await fetch("/api/transcribe/mp3", { method: "POST", body: form });
|
||||
|
||||
if (!res.ok) {
|
||||
const body = (await res.json().catch(() => ({}))) as { detail?: string };
|
||||
throw new Error(body.detail ?? `Server error ${res.status}`);
|
||||
}
|
||||
|
||||
// Endpoint returns a raw MIDI file (audio/midi) - parse it through the
|
||||
// same Web Worker pipeline as a regular .mid import.
|
||||
const midiBuffer = await res.arrayBuffer();
|
||||
const bufferForDoc = midiBuffer.slice(0);
|
||||
const doc = await parseMidiInWorker(midiBuffer, id, title);
|
||||
|
||||
set({ document: { ...doc, midiBuffer: bufferForDoc } });
|
||||
await syncEngine.loadDocument({ ...doc, midiBuffer: bufferForDoc });
|
||||
|
||||
// Kick off MusicXML transcription so the sheet-music view eventually
|
||||
// populates, mirroring loadMidiFile's pattern. Fire-and-forget.
|
||||
fetchMusicXml(bufferForDoc.slice(0), doc, id).catch(console.warn);
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
set({ loadError: msg });
|
||||
} finally {
|
||||
set({ isLoadingDocument: false, loadingMessage: null });
|
||||
}
|
||||
},
|
||||
|
||||
generateFingering: async () => {
|
||||
const { document: doc } = get();
|
||||
if (!doc?.musicXml) return;
|
||||
|
||||
@@ -32,6 +32,7 @@ export function PracticeView() {
|
||||
loadMidiFile,
|
||||
loadMusicXmlFile,
|
||||
loadPdfFile,
|
||||
loadAudioFile,
|
||||
settings,
|
||||
updateSettings,
|
||||
status,
|
||||
@@ -117,8 +118,9 @@ export function PracticeView() {
|
||||
if (file.name.match(/\.midi?$/i)) loadMidiFile(file);
|
||||
else if (file.name.match(/\.(xml|mxl)$/i)) loadMusicXmlFile(file);
|
||||
else if (file.name.match(/\.pdf$/i)) loadPdfFile(file);
|
||||
else if (file.name.match(/\.(mp3|wav|m4a|ogg|flac|aac)$/i)) loadAudioFile(file);
|
||||
},
|
||||
[loadMidiFile, loadMusicXmlFile, loadPdfFile]
|
||||
[loadMidiFile, loadMusicXmlFile, loadPdfFile, loadAudioFile]
|
||||
);
|
||||
|
||||
const handleDrop = useCallback(
|
||||
@@ -265,40 +267,31 @@ export function PracticeView() {
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{/* ── Main content - layout-aware ────────────────────────────────────── */}
|
||||
{/* ── Main content - layout-aware ──────────────────────────────────────
|
||||
PianoKeyboardView is a SINGLETON at the bottom of the main column so
|
||||
PixiJS doesn't re-init on every view switch (skeleton flash + perf cost).
|
||||
Same for FallingNotesView in falling/all modes - kept mounted across
|
||||
the two modes that show it. Only the conditional top-content elements
|
||||
(PianoModeBackground / SheetMusicView / sheet sidebar) mount and
|
||||
unmount on layoutMode change. */}
|
||||
<main className="flex flex-1 overflow-hidden min-h-0 relative">
|
||||
{layoutMode === "piano" && (
|
||||
<div className="flex flex-col flex-1 min-h-0 overflow-hidden">
|
||||
<PianoModeBackground />
|
||||
<PianoKeyboardView />
|
||||
</div>
|
||||
)}
|
||||
{layoutMode === "falling" && (
|
||||
<div className="flex flex-col flex-1 min-h-0 overflow-hidden">
|
||||
<FallingNotesView />
|
||||
<PianoKeyboardView />
|
||||
</div>
|
||||
)}
|
||||
{layoutMode === "sheet" && (
|
||||
<div className="flex flex-col flex-1 min-h-0 overflow-hidden">
|
||||
<SheetMusicView />
|
||||
<PianoKeyboardView />
|
||||
</div>
|
||||
)}
|
||||
{/* Optional sheet sidebar (only in "all" mode) */}
|
||||
{layoutMode === "all" && (
|
||||
<>
|
||||
<div
|
||||
className="flex flex-col flex-shrink-0 min-h-0 overflow-hidden relative"
|
||||
style={{ flex: "0 0 42%", minWidth: 260, maxWidth: 440 }}
|
||||
>
|
||||
<SheetMusicView />
|
||||
</div>
|
||||
<div className="flex flex-col flex-1 min-h-0 overflow-hidden">
|
||||
<FallingNotesView />
|
||||
<PianoKeyboardView />
|
||||
</div>
|
||||
</>
|
||||
<div
|
||||
className="flex flex-col flex-shrink-0 min-h-0 overflow-hidden relative"
|
||||
style={{ flex: "0 0 42%", minWidth: 260, maxWidth: 440 }}
|
||||
>
|
||||
<SheetMusicView />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Main column: top content varies, piano persists across all modes */}
|
||||
<div className="flex flex-col flex-1 min-h-0 overflow-hidden">
|
||||
{layoutMode === "piano" && <PianoModeBackground />}
|
||||
{(layoutMode === "falling" || layoutMode === "all") && <FallingNotesView />}
|
||||
{layoutMode === "sheet" && <SheetMusicView />}
|
||||
<PianoKeyboardView />
|
||||
</div>
|
||||
</main>
|
||||
|
||||
{/* ── Empty state overlay ────────────────────────────────────────────── */}
|
||||
@@ -362,7 +355,7 @@ function EmptyState({ onImport }: { onImport: (file: File) => void }) {
|
||||
No file loaded
|
||||
</div>
|
||||
<div style={{ fontSize: 13, color: "var(--color-text-muted)", lineHeight: 1.6 }}>
|
||||
Import a MIDI, MusicXML, or PDF sheet music file to start practicing
|
||||
Import a MIDI, MusicXML, PDF sheet music, or audio recording to start practicing
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -406,7 +399,7 @@ function EmptyState({ onImport }: { onImport: (file: File) => void }) {
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
accept=".mid,.midi,.xml,.mxl,.pdf"
|
||||
accept=".mid,.midi,.xml,.mxl,.pdf,.mp3,.wav,.m4a,.ogg,.flac,.aac"
|
||||
className="hidden"
|
||||
onChange={(e) => {
|
||||
const f = e.target.files?.[0];
|
||||
|
||||
Reference in New Issue
Block a user