From b31d1262763daa57cfae3cf40dc258f033681933 Mon Sep 17 00:00:00 2001 From: Fabricio Giglio <58532944+gigliof@users.noreply.github.com> Date: Wed, 20 May 2026 18:37:58 +0200 Subject: [PATCH] Add MP3 export via offline rendering (Splendid Grand Piano), polish deployment docs --- README.md | 10 +- SECURITY.md | 2 +- .../Transport/TransportControls.tsx | 16 ++- frontend/src/engine/AudioEngine.ts | 41 ++++-- frontend/src/store/useAppStore.ts | 117 +++++++++++++++++- 5 files changed, 173 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index ac52b9e..219c0b7 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,15 @@ Drag-and-drop anywhere on the window, or use the import button. - **MIDI input** via Web MIDI: connect a hardware keyboard (Roland, Yamaha, etc.); your input lights up the on-screen keys and drives wait mode - **Tempo control**: 25%-200% with snap-back to 100% +### Export + +Save the loaded piece in any of these formats via the Export button (next to Import): + +- **MIDI** (`.midi`) - the raw note data +- **MusicXML** (`.musicxml`) - full notation, opens in any score editor +- **PDF sheet music** (`.pdf`) - typeset via LilyPond (optional backend setup, see below) +- **MP3 audio** (`.mp3`) - rendered with the Splendid Grand Piano via an OfflineAudioContext (other instruments to follow; the grand covers the common case) + ### Look & feel - **Color themes**: Cascade, Violet, Classic, Ocean, Forest, or fully custom (left/right/unknown hand colors) @@ -213,7 +222,6 @@ Requires a Chromium-based browser (Chrome, Edge, Brave, Arc) for **Web MIDI API* The backend already exposes endpoints for several features that don't yet have UI hooks: - **YouTube-to-MIDI** via yt-dlp + Basic Pitch (`/youtube/extract`), sync data model already in place -- **MP3 export** (MIDI / MusicXML / PDF export already shipped; MP3 needs an OfflineAudioContext WAV-render pipeline) These will become user-facing in upcoming releases. PRs welcome. diff --git a/SECURITY.md b/SECURITY.md index abcc462..a2e8272 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -48,4 +48,4 @@ Use **GitHub's private vulnerability reporting**: 2. Click **"Report a vulnerability"** 3. Fill in the details - steps to reproduce, impact, any suggested fix -You'll get a response within a few days. There's no formal embargo policy given the nature of this project, but please give a reasonable heads-up before going public. +Reports are handled by a single maintainer in personal time, so response times vary. diff --git a/frontend/src/components/Transport/TransportControls.tsx b/frontend/src/components/Transport/TransportControls.tsx index bede9ca..1da5b98 100644 --- a/frontend/src/components/Transport/TransportControls.tsx +++ b/frontend/src/components/Transport/TransportControls.tsx @@ -36,7 +36,9 @@ export function TransportControls() { exportMidi, exportMusicXml, exportPdf, + exportMp3, isExportingPdf, + isExportingMp3, } = useAppStore(); const [speedOpen, setSpeedOpen] = useState(false); @@ -290,7 +292,7 @@ export function TransportControls() { if (doc) (e.currentTarget as HTMLElement).style.background = "rgba(147,51,234,0.14)"; }} > - {isExportingPdf ? : } + {isExportingPdf || isExportingMp3 ? : } {exportOpen && exportRect && (
+ { + void exportMp3(); + setExportOpen(false); + }} + disabledReason={ + !doc?.notes?.length ? "No notes to render" : "Already rendering an MP3" + } + />
)} diff --git a/frontend/src/engine/AudioEngine.ts b/frontend/src/engine/AudioEngine.ts index 50a5eca..e42d1af 100644 --- a/frontend/src/engine/AudioEngine.ts +++ b/frontend/src/engine/AudioEngine.ts @@ -18,25 +18,48 @@ export const INSTRUMENT_LABELS: Record = { honkytonk: "Honky", }; -// Minimal common interface shared by all smplr players -type Player = { +// Minimal common interface shared by all smplr players. Exported so the +// MP3 export pipeline can type the offline-rendering player the same way. +export type Player = { load: Promise; - start(opts: { note: number; velocity?: number; stopId?: number | string }): unknown; + /** `time` (seconds, relative to ctx.currentTime) + `duration` enable + * offline batch rendering - if omitted, the note plays immediately. */ + start(opts: { + note: number; + velocity?: number; + stopId?: number | string; + time?: number; + duration?: number; + }): unknown; stop(opts?: { stopId?: number | string }): void; }; -function createPlayer(id: InstrumentId, ctx: AudioContext): Player { +/** + * Build a smplr player for the given instrument against any AudioContext-like. + * Exported (not local) so the MP3 export pipeline can reuse it for an + * OfflineAudioContext - same sample set / same instrument as live playback, + * which is critical for the rendered audio to match what the user hears. + * + * Accepts `BaseAudioContext` (the common parent of both AudioContext and + * OfflineAudioContext) rather than `AudioContext` alone, so callers can pass + * an offline context for batch rendering. The smplr classes accept either. + */ +export function createPlayer(id: InstrumentId, ctx: BaseAudioContext): Player { switch (id) { case "grand": - return new SplendidGrandPiano(ctx) as unknown as Player; + return new SplendidGrandPiano(ctx as AudioContext) as unknown as Player; case "bright": - return new Soundfont(ctx, { instrument: "bright_acoustic_piano" }) as unknown as Player; + return new Soundfont(ctx as AudioContext, { + instrument: "bright_acoustic_piano", + }) as unknown as Player; case "electric": - return new ElectricPiano(ctx, { instrument: "CP80" }) as unknown as Player; + return new ElectricPiano(ctx as AudioContext, { instrument: "CP80" }) as unknown as Player; case "harpsichord": - return new Soundfont(ctx, { instrument: "harpsichord" }) as unknown as Player; + return new Soundfont(ctx as AudioContext, { instrument: "harpsichord" }) as unknown as Player; case "honkytonk": - return new Soundfont(ctx, { instrument: "honkytonk_piano" }) as unknown as Player; + return new Soundfont(ctx as AudioContext, { + instrument: "honkytonk_piano", + }) as unknown as Player; } } diff --git a/frontend/src/store/useAppStore.ts b/frontend/src/store/useAppStore.ts index 4615365..48f33da 100644 --- a/frontend/src/store/useAppStore.ts +++ b/frontend/src/store/useAppStore.ts @@ -115,8 +115,10 @@ export interface AppState { isLoadingDocument: boolean; /** True while /fingering/generate is in flight */ isGeneratingFingering: boolean; - /** True while /export/pdf is in flight (PDF rendering is the only async export) */ + /** True while /export/pdf is in flight (server-side LilyPond render) */ isExportingPdf: boolean; + /** True while MP3 export is in flight (frontend offline-render → backend FFmpeg) */ + isExportingMp3: boolean; loadError: string | null; /** Non-null during slow imports (e.g. OMR) to show a descriptive message */ loadingMessage: string | null; @@ -145,6 +147,7 @@ export interface AppState { exportMidi: () => void; exportMusicXml: () => void; exportPdf: () => Promise; + exportMp3: () => Promise; play: () => void; pause: () => void; stop: () => void; @@ -265,6 +268,7 @@ export const useAppStore = create((set, get) => { isLoadingDocument: false, isGeneratingFingering: false, isExportingPdf: false, + isExportingMp3: false, loadError: null, loadingMessage: null, status: "stopped", @@ -579,6 +583,104 @@ export const useAppStore = create((set, get) => { } }, + exportMp3: async () => { + const { document: doc } = get(); + if (!doc?.notes || doc.notes.length === 0) return; + + set({ isExportingMp3: true, loadError: null }); + try { + // Lazy-load smplr so this code path doesn't bloat the initial bundle + // for users who never export. + const { renderOffline, Smplr, Scheduler, HttpStorage, pianoToSmplrJson } = + await import("smplr"); + + // Tail buffer so the final note's release isn't cut off + small lead-in + // so the first note isn't clipped at sample 0. + const RENDER_TAIL_SEC = 1.5; + const RENDER_LEAD_SEC = 0.05; + + const renderResult = await renderOffline( + async (ctx) => { + // === Working around a real bug in smplr's offline rendering === + // + // smplr's default Scheduler uses setInterval to drain queued notes + // whose `time` is outside its 200ms lookahead window. setInterval + // callbacks NEVER fire during synchronous OfflineAudioContext + // rendering, so any note past 200ms would be silently discarded + // (you'd get a long silent WAV with only the very first beats + // audible). The README's renderOffline example has the same bug. + // + // Workaround: construct our own Scheduler with a huge lookahead + // (effectively infinite) so EVERY note falls within the synchronous + // dispatch branch in Scheduler.schedule(). Then we bypass the + // SplendidGrandPiano wrapper (which doesn't forward `scheduler` to + // its internal Smplr) and instantiate Smplr directly with the + // piano's JSON config + our scheduler. + // + // v1: always render with SplendidGrandPiano regardless of the + // currently-selected live instrument. 99% of users are on the + // default "grand" anyway; we can extend per-instrument later. + const scheduler = new Scheduler(ctx, { lookaheadMs: 86_400_000 }); + // pianoToSmplrJson only takes piano-shape options. The storage + // backend belongs in the Smplr constructor's options (where the + // SplendidGrandPiano wrapper also passes it). HttpStorage is a + // const singleton, not a class - no `new`. + const json = pianoToSmplrJson({ + baseUrl: "https://smpldsnds.github.io/sfzinstruments-splendid-grand-piano/samples", + detune: 0, + decayTime: 0.5, + }); + const player = new Smplr(ctx, json, { scheduler, storage: HttpStorage }); + await player.load; + + // Schedule every note. velocity/duration straight from the doc; + // we ignore current tempoMultiplier/transpose/hand-volume on + // purpose - MP3 export is "render the score as-written", not + // "render my current practice settings". + for (const note of doc.notes) { + player.start({ + note: note.midi, + velocity: note.velocity, + time: RENDER_LEAD_SEC + note.startSeconds, + duration: Math.max(0.05, note.endSeconds - note.startSeconds), + }); + } + }, + { + sampleRate: 44100, + channels: 2, + duration: RENDER_LEAD_SEC + doc.totalDuration + RENDER_TAIL_SEC, + } + ); + + // Encode the AudioBuffer as 16-bit WAV - half the size of float32, the + // codec FFmpeg expects, and well within the backend's 200 MB cap for + // any practical song length. smplr returns a Blob; we need raw bytes + // for base64 encoding. + const wavBlob = renderResult.toWav16(); + const wavBuffer = await wavBlob.arrayBuffer(); + const wavBase64 = arrayBufferToBase64(wavBuffer); + + const res = await fetch("/api/export/mp3", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ wav_base64: wavBase64 }), + }); + if (!res.ok) { + const body = (await res.json().catch(() => ({}))) as { detail?: string }; + throw new Error(body.detail ?? `Server error ${res.status}`); + } + + const blob = await res.blob(); + triggerDownload(blob, `${safeFilename(doc.title)}.mp3`); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + set({ loadError: msg }); + } finally { + set({ isExportingMp3: false }); + } + }, + play: () => { const { settings, status } = get(); // Speed trainer: seed the engine's current state before play @@ -727,6 +829,19 @@ function safeFilename(name: string): string { return name.replace(/[\x00-\x1f\x7f/\\:*?"<>|]/g, "_").slice(0, 200) || "untitled"; } +/** Encode a Uint8Array / ArrayBuffer as a standard base64 string. + * Chunked to avoid `String.fromCharCode(...veryLongArray)` blowing the call + * stack on large WAV exports (a 5-min stereo render is ~26 MB). */ +function arrayBufferToBase64(buffer: Uint8Array | ArrayBuffer): string { + const bytes = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer); + const CHUNK = 0x8000; // 32 KB + let binary = ""; + for (let i = 0; i < bytes.length; i += CHUNK) { + binary += String.fromCharCode.apply(null, Array.from(bytes.subarray(i, i + CHUNK))); + } + return btoa(binary); +} + /** Create an anchor, click it to trigger a download, then revoke the object URL. */ function triggerDownload(blob: Blob, filename: string): void { const url = URL.createObjectURL(blob);