From c6a0c7f6764d06d128176ac39e6c5f1d899fd607 Mon Sep 17 00:00:00 2001
From: Fabricio Giglio <58532944+gigliof@users.noreply.github.com>
Date: Sun, 10 May 2026 23:50:20 +0200
Subject: [PATCH] Add piano fingering generation and display via pianoplayer,
fix silent audio on first load
---
backend/services/pianoplayer_svc.py | 52 ++-
.../FallingNotes/FallingNotesRenderer.ts | 23 +-
.../FallingNotes/FallingNotesView.tsx | 51 ++-
.../components/Transport/SettingsPanel.tsx | 65 +++-
frontend/src/engine/AudioEngine.ts | 67 +++-
frontend/src/engine/SyncEngine.ts | 116 +++++--
frontend/src/store/useAppStore.ts | 308 ++++++++++++++----
frontend/src/views/PracticeView.tsx | 22 +-
8 files changed, 542 insertions(+), 162 deletions(-)
diff --git a/backend/services/pianoplayer_svc.py b/backend/services/pianoplayer_svc.py
index d1dfb87..7bc8d13 100644
--- a/backend/services/pianoplayer_svc.py
+++ b/backend/services/pianoplayer_svc.py
@@ -1,8 +1,8 @@
"""
pianoplayer service: annotate MusicXML with fingering (Parncutt algorithm).
-Note: defusedxml.defuse_stdlib() is called in main.py before this module
-is imported, so xml.etree (used by music21 internally) is already patched.
+Uses the pianoplayer v3 API (run_annotate) which handles part routing and
+hand assignment internally.
"""
import asyncio
import os
@@ -28,11 +28,12 @@ async def annotate_fingering(musicxml: str) -> str:
def _annotate(musicxml: str) -> str:
- from pianoplayer.hand import RightHand, LeftHand
- import music21
+ import io
+ import contextlib
+ from pianoplayer.core import run_annotate
xml_path = None
- out_tmp_name = None
+ out_path = None
try:
with tempfile.NamedTemporaryFile(
suffix=".xml", delete=False, mode="w", encoding="utf-8"
@@ -40,37 +41,24 @@ def _annotate(musicxml: str) -> str:
f.write(musicxml)
xml_path = f.name
- score = music21.converter.parse(xml_path)
-
- # pianoplayer works on individual parts
- for part in score.parts:
- notes = list(part.flatten().notes)
- if not notes:
- continue
-
- # Determine left / right hand by average register
- avg_pitch = sum(
- n.pitch.midi for n in notes if hasattr(n, "pitch")
- ) / max(len(notes), 1)
- hand_cls = RightHand if avg_pitch >= 60 else LeftHand
-
- try:
- hand = hand_cls(score, 0)
- hand.autodepth = False
- hand.verbose = False
- hand.generate(part=part.id if hasattr(part, "id") else 0)
- except Exception:
- pass # pianoplayer can fail on complex scores; skip gracefully
-
out_tmp = tempfile.NamedTemporaryFile(suffix=".xml", delete=False)
out_tmp.close()
- out_tmp_name = out_tmp.name
- score.write("musicxml", fp=out_tmp_name)
+ out_path = out_tmp.name
- return Path(out_tmp_name).read_text(encoding="utf-8")
+ # Suppress Rich progress/summary output so server logs stay clean.
+ with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
+ run_annotate(
+ xml_path,
+ outputfile=out_path,
+ n_measures=1000,
+ auto_routing=True, # let pianoplayer detect which part is L/R
+ quiet=True,
+ )
+
+ return Path(out_path).read_text(encoding="utf-8")
finally:
if xml_path:
Path(xml_path).unlink(missing_ok=True)
- if out_tmp_name:
- Path(out_tmp_name).unlink(missing_ok=True)
+ if out_path:
+ Path(out_path).unlink(missing_ok=True)
diff --git a/frontend/src/components/FallingNotes/FallingNotesRenderer.ts b/frontend/src/components/FallingNotes/FallingNotesRenderer.ts
index 8998b00..734e930 100644
--- a/frontend/src/components/FallingNotes/FallingNotesRenderer.ts
+++ b/frontend/src/components/FallingNotes/FallingNotesRenderer.ts
@@ -244,6 +244,9 @@ class NoteBar {
private label: PIXI.Text;
/** Whether this bar's label should be shown at all (text non-empty AND bar big enough). */
private labelWanted = false;
+ /** Cached at configure() time so relabel() can recompute labelWanted without re-rendering the bar. */
+ private barWidth = 0;
+ private barHeight = 0;
note: NoteEvent | null = null;
active = false;
impactFired = false;
@@ -283,6 +286,8 @@ class NoteBar {
this.active = true;
this.impactFired = false;
this.baseColor = baseColor;
+ this.barWidth = barWidth;
+ this.barHeight = barHeight;
this.container.visible = true;
this.container.x = x;
this.container.y = startY;
@@ -325,6 +330,13 @@ class NoteBar {
this.label.visible = this.labelWanted;
}
+ /** Update just the label text on an already-configured bar. Cheap: no re-rendering of bg. */
+ relabel(labelText: string) {
+ this.label.text = labelText;
+ this.labelWanted = labelText.length > 0 && this.barHeight > 6 && this.barWidth > 7;
+ this.label.visible = this.labelWanted;
+ }
+
/** Keep label clamped within the visible portion of the bar (for very tall notes). */
updateLabelY(containerY: number, barHeight: number, viewportHeight: number) {
if (!this.labelWanted) {
@@ -362,6 +374,7 @@ export interface FallingNotesOptions {
height: number;
viewportSeconds: number;
fallingNotesLabelMode?: NoteLabelMode;
+ /** When true, append the assigned finger digit to each note bar's label */
showFingering: boolean;
showHandColors: boolean;
useFlats: boolean;
@@ -983,9 +996,13 @@ export class FallingNotesRenderer {
("useFlats" in opts && opts.useFlats !== this.opts.useFlats) ||
("fallingNotesLabelMode" in opts &&
opts.fallingNotesLabelMode !== this.opts.fallingNotesLabelMode) ||
- ("showFingering" in opts && opts.showFingering !== this.opts.showFingering) ||
("showNoteOutline" in opts && opts.showNoteOutline !== this.opts.showNoteOutline);
+ // Fingering toggle is cheaper - re-label active bars in place rather than
+ // resetting (which would briefly clear all bars from the screen).
+ const fingeringChanged =
+ "showFingering" in opts && opts.showFingering !== this.opts.showFingering;
+
Object.assign(this.opts, opts);
if (!this._initialized) return;
@@ -993,6 +1010,10 @@ export class FallingNotesRenderer {
for (const g of this.gradientCache.values()) g.destroy();
this.gradientCache.clear();
this.reset();
+ } else if (fingeringChanged) {
+ for (const [, bar] of this.active) {
+ if (bar.note) bar.relabel(this.noteLabel(bar.note));
+ }
}
if ("showGrid" in opts) this.gridLayer.visible = !!opts.showGrid;
if ("impactStyle" in opts) {
diff --git a/frontend/src/components/FallingNotes/FallingNotesView.tsx b/frontend/src/components/FallingNotes/FallingNotesView.tsx
index 22de7c0..fb07ade 100644
--- a/frontend/src/components/FallingNotes/FallingNotesView.tsx
+++ b/frontend/src/components/FallingNotes/FallingNotesView.tsx
@@ -17,12 +17,16 @@ export function FallingNotesView() {
const accuracy = useAccuracy(midiEnabled && (status === "playing" || status === "paused"));
const measureStarts = useMemo(
- () => doc ? computeMeasureSeconds(doc.timeSignatures, doc.tempoMap, doc.ppq, doc.totalDuration) : [],
+ () =>
+ doc
+ ? computeMeasureSeconds(doc.timeSignatures, doc.tempoMap, doc.ppq, doc.totalDuration)
+ : [],
[doc]
);
const beatStarts = useMemo(
- () => doc ? computeBeatSeconds(doc.timeSignatures, doc.tempoMap, doc.ppq, doc.totalDuration) : [],
+ () =>
+ doc ? computeBeatSeconds(doc.timeSignatures, doc.tempoMap, doc.ppq, doc.totalDuration) : [],
[doc]
);
@@ -38,7 +42,7 @@ export function FallingNotesView() {
height,
viewportSeconds: settings.viewportSeconds,
fallingNotesLabelMode: settings.fallingNotesLabelMode,
- showFingering: settings.showFingering,
+ showFingering: settings.showFingering && settings.showFingeringOnNotes,
showHandColors: settings.showHandColors,
useFlats: settings.useFlats,
noteFilter: settings.noteFilter,
@@ -103,7 +107,7 @@ export function FallingNotesView() {
rendererRef.current?.setOptions({
viewportSeconds: settings.viewportSeconds,
fallingNotesLabelMode: settings.fallingNotesLabelMode,
- showFingering: settings.showFingering,
+ showFingering: settings.showFingering && settings.showFingeringOnNotes,
showHandColors: settings.showHandColors,
useFlats: settings.useFlats,
noteFilter: settings.noteFilter,
@@ -127,6 +131,7 @@ export function FallingNotesView() {
settings.viewportSeconds,
settings.fallingNotesLabelMode,
settings.showFingering,
+ settings.showFingeringOnNotes,
settings.showHandColors,
settings.useFlats,
settings.noteFilter,
@@ -151,7 +156,10 @@ export function FallingNotesView() {
if (!doc || !settings.scrollToSeek) return;
// Seek forward/backward by 30% of the viewport window per scroll tick
const delta = (e.deltaY > 0 ? 1 : -1) * settings.viewportSeconds * 0.3;
- const next = Math.max(0, Math.min(doc.totalDuration, useAppStore.getState().currentSeconds + delta));
+ const next = Math.max(
+ 0,
+ Math.min(doc.totalDuration, useAppStore.getState().currentSeconds + delta)
+ );
seek(next);
};
@@ -162,19 +170,25 @@ export function FallingNotesView() {
onWheel={handleWheel}
style={{
minHeight: 200,
- background: "radial-gradient(ellipse at 50% 100%, rgba(147,51,234,0.07) 0%, transparent 65%), var(--color-notes-bg)",
+ background:
+ "radial-gradient(ellipse at 50% 100%, rgba(147,51,234,0.07) 0%, transparent 65%), var(--color-notes-bg)",
borderTop: "1px solid var(--color-notes-border)",
}}
>
- {midiEnabled && (accuracy.correct + accuracy.wrong + accuracy.missed) > 0 && (
+ {midiEnabled && accuracy.correct + accuracy.wrong + accuracy.missed > 0 && (
= 80 ? "rgba(34,197,94,0.12)" : accuracy.score >= 50 ? "rgba(234,179,8,0.12)" : "rgba(239,68,68,0.12)",
- color: accuracy.score >= 80 ? "#4ade80" : accuracy.score >= 50 ? "#eab308" : "#ef4444",
- border: "1px solid currentColor",
+ background:
+ accuracy.score >= 80
+ ? "rgba(34,197,94,0.12)"
+ : accuracy.score >= 50
+ ? "rgba(234,179,8,0.12)"
+ : "rgba(239,68,68,0.12)",
+ color: accuracy.score >= 80 ? "#4ade80" : accuracy.score >= 50 ? "#eab308" : "#ef4444",
+ border: "1px solid currentColor",
}}
>
{accuracy.score}%
@@ -187,9 +201,18 @@ export function FallingNotesView() {
// ── Icons ─────────────────────────────────────────────────────────────────────
const TargetIcon = () => (
-