From 6de3c8d98e857d7196d1a51be406f6d487229f8e Mon Sep 17 00:00:00 2001 From: Fabricio Giglio <58532944+gigliof@users.noreply.github.com> Date: Sun, 17 May 2026 08:10:03 +0200 Subject: [PATCH] Add audio-to-MIDI import via Basic Pitch (MP3, WAV, M4A, OGG, FLAC, AAC), fix piano remount on view change --- README.md | 17 +++++- backend/requirements-transcribe.txt | 25 ++++++-- backend/routers/transcribe.py | 25 +++++++- .../Transport/TransportControls.tsx | 8 ++- frontend/src/store/useAppStore.ts | 45 ++++++++++++++ frontend/src/views/PracticeView.tsx | 61 ++++++++----------- 6 files changed, 134 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 4aaaff8..07adf29 100644 --- a/README.md +++ b/README.md @@ -29,13 +29,15 @@ A browser-based piano practice tool. Drop in a **MIDI file**, a **MusicXML score - **MIDI** (`.mid`, `.midi`): parsed in a Web Worker, converted to MusicXML on the backend for sheet rendering - **MusicXML** (`.xml`, `.mxl`): both uncompressed and compressed (zip-style) variants - **PDF sheet music** (`.pdf`): optical music recognition via [Audiveris](https://github.com/Audiveris/audiveris) to MusicXML, then to MIDI (optional setup, see below) +- **Audio recordings** (`.mp3`, `.wav`, `.m4a`, `.ogg`, `.flac`, `.aac`): audio-to-MIDI transcription via [Basic Pitch](https://github.com/spotify/basic-pitch) (optional setup, see below) - drop a recording of yourself improvising, get a playable piano roll Drag-and-drop anywhere on the window, or use the import button. -> ⚠️ **A note on import accuracy.** Both PDF-to-MIDI (optical music recognition) and MIDI-to-sheet-music (notation reconstruction) are inherently lossy conversions. Expect some inconsistencies: +> ⚠️ **A note on import accuracy.** PDF-to-MIDI (optical music recognition), MIDI-to-sheet-music (notation reconstruction), and audio-to-MIDI (pitch detection) are all inherently lossy conversions. Expect some inconsistencies: > > - **PDF imports** can mis-read notes, dynamics, ornaments, articulation, and voicing, especially with low-resolution scans, handwritten scores, or complex layouts. Audiveris is the best open-source OMR engine available, but no engine matches a careful human transcription. > - **MIDI imports** produce sheet music by re-deriving notation from raw note timings. MIDI doesn't encode key signatures, beaming, voicing, articulation, or enharmonic spelling, so the rendered sheet is an interpretation, not a faithful reproduction of the composer's original score. +> - **Audio imports** detect pitches in the mixed signal - this is pitch-detection, not source separation. Quality is excellent on solo instruments (piano, voice, guitar) and degrades on complex mixes (full-band recordings with drums + bass + vocals). For multi-track sources, expect a rough sketch rather than a clean transcription. Solo piano recordings give the best results. > > For the cleanest sheet-music experience, import a **MusicXML** file (or `.mxl`) when you have one, which preserves full notation semantics end-to-end. @@ -131,6 +133,16 @@ PDF export uses [LilyPond](https://lilypond.org/) for typesetting. It's not bund Restart the backend after installing. Without LilyPond, MIDI and MusicXML export still work; only PDF export is unavailable. +### Optional: Audio-to-MIDI via Basic Pitch + +Audio-to-MIDI transcription uses [Basic Pitch](https://github.com/spotify/basic-pitch), Spotify's open-source pitch-detection model. It's not bundled by default because it pulls in TensorFlow (~500 MB) and has tight Python-version constraints. + +- **macOS**: requires Python **3.11** specifically. `brew install python@3.11`, then create a 3.11 venv at `backend/.venv` and `pip install -r backend/requirements-transcribe.txt`. (Python 3.12+ doesn't work on macOS because `basic-pitch` requires `tensorflow-macos<2.15.1`, and no compatible wheel exists for 3.12.) +- **Linux**: Python 3.11 or 3.12. Activate the venv and `pip install -r backend/requirements-transcribe.txt`. +- **Docker**: edit `backend/Dockerfile` to also `pip install -r requirements-transcribe.txt` (or use a separate build target), then `docker compose build backend`. The image already uses Python 3.12. + +Restart the backend after installing. Without Basic Pitch, MIDI / MusicXML / PDF import still works; only audio import (MP3, WAV, etc.) is unavailable. + ## Using the app on iPad / iPhone (same network) 1. In `frontend/vite.config.ts`, add `host: true` to the `server` block (or run `pnpm --filter frontend dev --host`) @@ -189,9 +201,8 @@ Requires a Chromium-based browser (Chrome, Edge, Brave, Arc) for **Web MIDI API* The backend already exposes endpoints for several features that don't yet have UI hooks: -- **Audio-to-MIDI transcription** via [Basic Pitch](https://github.com/spotify/basic-pitch) (`/transcribe/mp3`) - **YouTube-to-MIDI** via yt-dlp + Basic Pitch (`/youtube/extract`), sync data model already in place -- **Export** to PDF / MP3 / MIDI / MusicXML (`/export/pdf`, `/export/mp3`) +- **MP3 export** (MIDI / MusicXML / PDF export already shipped; MP3 needs an OfflineAudioContext WAV-render pipeline) These will become user-facing in upcoming releases. PRs welcome. diff --git a/backend/requirements-transcribe.txt b/backend/requirements-transcribe.txt index e89773e..d16dbdd 100644 --- a/backend/requirements-transcribe.txt +++ b/backend/requirements-transcribe.txt @@ -1,11 +1,26 @@ # Optional audio-transcription dependencies (Basic Pitch). -# Requires Python 3.11 or 3.12, tensorflow-macos doesn't yet support 3.13. # -# To enable MP3 / audio-to-MIDI transcription: -# brew install python@3.12 -# /opt/homebrew/bin/python3.12 -m venv .venv +# Requires Python 3.11 on macOS. (basic-pitch's tensorflow-macos<2.15.1 +# constraint excludes 3.12+ - no compatible tensorflow-macos wheel exists.) +# Linux is more flexible; 3.11 or 3.12 both work there. +# +# To enable MP3 / audio-to-MIDI transcription on macOS: +# brew install python@3.11 +# /opt/homebrew/bin/python3.11 -m venv .venv # source .venv/bin/activate # pip install -r requirements-transcribe.txt # -r requirements.txt -basic-pitch>=0.3.6 + +# setuptools < 81 is required because basic-pitch's dependency `resampy` +# still imports `pkg_resources`, which setuptools 81 (released 2025) removed +# after years of deprecation. Until upstream resampy migrates to +# `importlib.metadata`, we pin to the last setuptools major that ships +# pkg_resources (80.x). +setuptools<81 + +# `[tf]` extra is REQUIRED since basic-pitch 0.4+ made the model backend opt-in. +# Without it, basic-pitch installs but `predict()` fails (no TensorFlow to load +# the ICASSP_2022_MODEL_PATH saved model). The `[onnx]` extra would be lighter +# (~10 MB vs TF's ~500 MB) but our service uses the TF saved-model format. +basic-pitch[tf]>=0.3.6 diff --git a/backend/routers/transcribe.py b/backend/routers/transcribe.py index bad3e7b..117da18 100644 --- a/backend/routers/transcribe.py +++ b/backend/routers/transcribe.py @@ -158,6 +158,27 @@ async def transcribe_mp3(file: UploadFile, background_tasks: BackgroundTasks): ) except HTTPException: raise - except Exception: + except Exception as e: + # Surface actionable runtime errors (e.g. missing basic-pitch) to the + # client so the user knows what to install. Match by message substring + # since exceptions may be wrapped on the way back. logger.exception("transcribe_mp3 failed") - raise HTTPException(status_code=500, detail="Transcription failed. Please try again.") + msg = str(e) + # Only match the specific "not installed" sentinel - any other error + # that happens to mention basic-pitch (e.g. file paths inside the + # package, model-load failures) should NOT be wrongly reported as + # "needs install". + if "basic-pitch is not installed" in msg.lower(): + raise HTTPException( + status_code=503, + detail=( + "Audio-to-MIDI transcription needs Basic Pitch on the backend " + "(not bundled by default).\n\n" + "Quick install: with a Python 3.11 venv active on macOS (or 3.11/3.12 " + "on Linux), run pip install -r backend/requirements-transcribe.txt. " + "For Docker, see the README section 'Optional: Audio-to-MIDI via Basic Pitch'." + ), + ) + raise HTTPException( + status_code=500, detail=f"Transcription failed: {msg or 'Unknown error'}" + ) diff --git a/frontend/src/components/Transport/TransportControls.tsx b/frontend/src/components/Transport/TransportControls.tsx index 19ac168..bede9ca 100644 --- a/frontend/src/components/Transport/TransportControls.tsx +++ b/frontend/src/components/Transport/TransportControls.tsx @@ -30,6 +30,7 @@ export function TransportControls() { loadMidiFile, loadMusicXmlFile, loadPdfFile, + loadAudioFile, isLoadingDocument, loadingMessage, exportMidi, @@ -87,6 +88,7 @@ export function TransportControls() { if (file.name.match(/\.midi?$/i)) loadMidiFile(file); else if (file.name.match(/\.(xml|mxl)$/i)) loadMusicXmlFile(file); else if (file.name.match(/\.pdf$/i)) loadPdfFile(file); + else if (file.name.match(/\.(mp3|wav|m4a|ogg|flac|aac)$/i)) loadAudioFile(file); }; const total = doc?.totalDuration ?? 0; @@ -209,8 +211,8 @@ export function TransportControls() { > {/* ── Zone 1: File import - accent-styled to stand out ── */}