Merge pull request #137 from meowarex/dev

WIP | Syllable AI
2026-08-26 22:37:45 +10:00 · 2026-08-11 20:28:08 +10:00
parent d7783ac509 3a05506ce6
commit 796c045538
4 changed files with 160 additions and 17 deletions
@@ -17,6 +17,8 @@ declare global {
 		updateLyricsStyleSetting?: (value: number) => void;
 		updateRomanizeLyrics?: () => void;
 		updateRomanizeLyricsSetting?: (checked: boolean) => void;
+		updateAiSyllables?: () => void;
+		updateAiSyllablesSetting?: (checked: boolean) => void;
 	}
 }

@@ -29,6 +31,7 @@ export const settings = await ReactiveStore.getPluginStorage("RadiantLyrics", {
 	contextAwareLyrics: true,
 	bubbledLyrics: true,
 	romanizeLyrics: false,
+	aiSyllables: false,
 	stickyLyrics: false,
 	syllableStyle: 0, // MARKER: Syllable animations SETTINGS (WIP coming soon)
 	syllableLogging: false,
@@ -171,6 +174,16 @@ export const Settings = () => {
 			window.updateRomanizeLyricsSetting = undefined;
 		};
 	}, []);
+	const [aiSyllables, setAiSyllables] = React.useState(
+		settings.aiSyllables,
+	);
+	React.useEffect(() => {
+		window.updateAiSyllablesSetting = (checked: boolean) =>
+			setAiSyllables(checked);
+		return () => {
+			window.updateAiSyllablesSetting = undefined;
+		};
+	}, []);

 	// Derive props and override onChange to accept a broader first param type
 	type BaseSwitchProps = React.ComponentProps<typeof LunaSwitchSetting>;
@@ -279,6 +292,18 @@ export const Settings = () => {
 						window.updateLyricsStyle();
 					}
 				}}
+      />
+      <AnySwitch
+				title="Sticky Lyrics"
+				desc="auto-switches to Play Queue when lyrics aren't available (mirrored in lyrics dropdown)"
+				checked={stickyLyrics}
+				onChange={(_: unknown, checked: boolean) => {
+					settings.stickyLyrics = checked;
+					setStickyLyrics(checked);
+					if (window.updateStickyLyricsFeature) {
+						window.updateStickyLyricsFeature();
+					}
+				}}
 			/>
 			<AnySwitch
 				title="Romanize Lyrics"
@@ -293,14 +318,14 @@ export const Settings = () => {
 				}}
 			/>
 			<AnySwitch
-				title="Sticky Lyrics"
-				desc="auto-switches to Play Queue when lyrics aren't available (mirrored in lyrics dropdown)"
-				checked={stickyLyrics}
+				title="WIP | AI Generated Syllables"
+				desc="Radiant AI generates word & syllable timings from the Line timings"
+				checked={aiSyllables}
 				onChange={(_: unknown, checked: boolean) => {
-					settings.stickyLyrics = checked;
-					setStickyLyrics(checked);
-					if (window.updateStickyLyricsFeature) {
-						window.updateStickyLyricsFeature();
+					settings.aiSyllables = checked;
+					setAiSyllables(checked);
+					if (window.updateAiSyllables) {
+						window.updateAiSyllables();
 					}
 				}}
 			/>
@@ -43,12 +43,13 @@ function query(
 	title: string,
 	artist: string,
 	isrc: string | undefined,
-	options?: { romanize?: boolean; flush?: boolean },
+	options?: { romanize?: boolean; flush?: boolean; synthesize?: boolean },
 ): string {
 	let q = `?title=${encodeURIComponent(title)}&artist=${encodeURIComponent(artist)}`;
 	if (isrc) q += `&isrc=${encodeURIComponent(isrc)}`;
 	if (options?.romanize) q += "&romanize=true";
 	if (options?.flush) q += "&flush=true";
+	if (options?.synthesize) q += "&synthesize=true";
 	q += `&${platformQs}`;
 	return q;
 }
@@ -61,6 +62,8 @@ export interface WordTiming {
 	duration: number;
 	isBackground: boolean;
 	romanized?: string;
+	/** 0..1 model confidence for synthesized timings; unset for provider data */
+	confidence?: number;
 }

 export interface WordLine {
@@ -107,6 +110,10 @@ export interface WordLyricsResponse {
 		songParts?: Array<{ name: string; time: number; duration: number }>;
 	};
 	_cached?: boolean;
+	/** true when timings came from the hosted Radiant AI model (?synthesize=true) */
+	_synthesized?: boolean;
+	/** original line-level data, kept so AI timings can be toggled off without a refetch */
+	lines?: ApiLine[];
 }

 export interface LineLyricsResponse {
@@ -137,8 +144,9 @@ export async function fetchLyrics(
 	artist: string,
 	isrc: string | undefined,
 	romanize: boolean,
+	synthesize = false,
 ): Promise<LyricsApiResponse | null> {
-	const params = query(title, artist, isrc, { romanize });
+	const params = query(title, artist, isrc, { romanize, synthesize });
 	const atomixUrl = `https://api.atomix.one/rl-api${params}`;
 	const fallbackUrl = `https://rl-api.kineticsand.net/lyrics${params}`;

@@ -11,6 +11,7 @@ import {
 } from "@luna/lib";
 import {
 	type ApiLine,
+	type LineLyricsResponse,
 	type LyricsApiResponse,
 	type WordLine,
 	type WordTiming,
@@ -1368,6 +1369,19 @@ const ensureStickyDropdown = (): HTMLElement => {
 				<input type="checkbox" data-setting="stickyLyrics" ${settings.stickyLyrics ? "checked" : ""}>
 				<span class="sticky-lyrics-slider"></span>
 			</label>
+		</div>		<div class="sticky-lyrics-dropdown-row">
+			<span class="sticky-lyrics-label">Romanization</span>
+			<label class="sticky-lyrics-switch">
+				<input type="checkbox" data-setting="romanizeLyrics" ${settings.romanizeLyrics ? "checked" : ""}>
+				<span class="sticky-lyrics-slider"></span>
+			</label>
+		</div>
+		<div class="sticky-lyrics-dropdown-row">
+			<span class="sticky-lyrics-label">AI Syllables | WIP</span>
+			<label class="sticky-lyrics-switch">
+				<input type="checkbox" data-setting="aiSyllables" ${settings.aiSyllables ? "checked" : ""}>
+				<span class="sticky-lyrics-slider"></span>
+			</label>
 		</div>
 		<div class="sticky-lyrics-dropdown-row rl-style-row">
 			<div class="rl-seg-control">
@@ -1389,6 +1403,26 @@ const ensureStickyDropdown = (): HTMLElement => {
 		}
 	});

+	// AI Generated Syllables (Radiant AI [RL API])
+	const synthCheckbox = dropdown.querySelector(
+		'input[data-setting="aiSyllables"]',
+	) as HTMLInputElement;
+	synthCheckbox.addEventListener("change", () => {
+		settings.aiSyllables = synthCheckbox.checked;
+		(window as any).updateAiSyllablesSetting?.(synthCheckbox.checked);
+		updateAiSyllablesFromSettings();
+	});
+
+	// Romanization (mirrors the Romanize Lyrics setting)
+	const romanizeCheckbox = dropdown.querySelector(
+		'input[data-setting="romanizeLyrics"]',
+	) as HTMLInputElement;
+	romanizeCheckbox.addEventListener("change", () => {
+		settings.romanizeLyrics = romanizeCheckbox.checked;
+		(window as any).updateRomanizeLyricsSetting?.(romanizeCheckbox.checked);
+		updateRomanizeLyricsFromSettings();
+	});
+
 	const styleNames = ["Line", "Word", "Syllable"];
 	const segButtons = dropdown.querySelectorAll(".rl-seg-btn");
 	for (const btn of segButtons) {
@@ -1622,6 +1656,7 @@ interface SyntheticNativeLyricsState {
 let trackChangeToken = 0;
 let lyricsData: WordLine[] | null = null;
 let lyricsResponse: LyricsApiResponse | null = null;
+let lyricsIsAiGenerated = false;
 let lyricsMode: LyricsOverlayMode = "none";
 let tickLoopUnload: LunaUnload | null = null;
 let isActive = false;
@@ -2418,16 +2453,41 @@ const fetchLyrics = async (
 	artist: string,
 	isrc?: string,
 ): Promise<LyricsApiResponse | null> => {
-	const cacheKey = `${title}\0${artist}\0${isrc ?? ""}\0${settings.romanizeLyrics ? "r" : ""}`;
+	const cacheKey = `${title}\0${artist}\0${isrc ?? ""}\0${settings.romanizeLyrics ? "r" : ""}${settings.aiSyllables ? "a" : ""}`;
 	if (cachedLyricsKey === cacheKey) {
 		sylLog(`[RL-Syllable] Cache hit for "${title}" by "${artist}"`);
 		return cachedLyricsData;
 	}
+
+	// AI toggled off mid song, AI response has the original lines (no refetch)
+	if (
+		!settings.aiSyllables &&
+		cachedLyricsKey === `${cacheKey}a` &&
+		cachedLyricsData?.type === "Word" &&
+		cachedLyricsData._synthesized &&
+		cachedLyricsData.lines
+	) {
+		const derived: LineLyricsResponse = {
+			type: "Line",
+			data: cachedLyricsData.lines,
+			metadata: {
+				...cachedLyricsData.metadata,
+				source: cachedLyricsData.metadata.source.replace(/ • AI$/, ""),
+			},
+			_cached: cachedLyricsData._cached,
+		};
+		cachedLyricsKey = cacheKey;
+		cachedLyricsData = derived;
+		sylLog(`[RL-Syllable] AI off — using line timings from the AI response`);
+		return derived;
+	}
+
 	const data = await fetchLyricsApi(
 		title,
 		artist,
 		isrc,
 		settings.romanizeLyrics,
+		settings.aiSyllables,
 	);
 	cachedLyricsKey = cacheKey;
 	cachedLyricsData = data;
@@ -2689,6 +2749,7 @@ const buildWordSpans = (): {
 	if (settings.blurInactive && scrollSynced && blurActivated)
 		wbwContainer.classList.add("rl-blur-active");
 	if (settings.bubbledLyrics) wbwContainer.classList.add("rl-bubbled");
+	if (lyricsIsAiGenerated) wbwContainer.classList.add("rl-ai");
 	const effectiveStyle = getLyricsStyle();
 	const allowWordSylStyles = isWordMode();
 	// MARKER: Syllable animations (WIP coming soon)
@@ -2804,6 +2865,7 @@ const buildWordSpans = (): {
 			text: string,
 			seekMs: number,
 			bg: boolean,
+			confidence?: number,
 		): HTMLSpanElement => {
 			const span = document.createElement("span");
 			span.className = "rl-wbw-word";
@@ -2813,6 +2875,12 @@ const buildWordSpans = (): {
 				span.textContent = text;
 			}
 			forceStyle(span, WORD_SPAN_STYLE);
+			// Confidence-proportional wipe feather: crisp edges where the
+			// synthesizer had real evidence, soft where it guessed. SUPER WIP
+			if (confidence !== undefined) {
+				const feather = 0.55 + (1 - Math.max(0, Math.min(1, confidence))) * 1.3;
+				span.style.setProperty("--rl-wipe-feather", `${feather.toFixed(2)}em`);
+			}
 			if (bg) span.classList.add("rl-wbw-bg");
 			span.addEventListener("click", () => {
 				PlayState.seek(seekMs / 1000);
@@ -2898,6 +2966,7 @@ const buildWordSpans = (): {
 							sylDisplay(syl).trimEnd(),
 							wordStartMs,
 							syl.isBackground,
+							syl.confidence,
 						);
 						span.addEventListener("mouseenter", () => {
 							for (const s of groupSpans) s.classList.add("rl-wbw-word-hover");
@@ -2925,7 +2994,7 @@ const buildWordSpans = (): {
 					const start = firstSyl.time;
 					const end = lastSyl.time + lastSyl.duration;
 					const bg = firstSyl.isBackground;
-					const span = makeSpan(mergedText, start, bg);
+					const span = makeSpan(mergedText, start, bg, firstSyl.confidence);
 					targetContainer.appendChild(span);
 					const entry: WordEntry = {
 						el: span,
@@ -3523,6 +3592,7 @@ const teardown = (): void => {
 	lyricsMode = "none";
 	lyricsData = null;
 	lyricsResponse = null;
+	lyricsIsAiGenerated = false;
 	lines = [];
 	activeWordEls.clear();
 	activeBgWordEls.clear();
@@ -4126,6 +4196,15 @@ const onTrackChange = async (): Promise<void> => {
 		);

 		unlockFlush();
+
+		// MARKER: Radiant AI Syllables [RL API]
+		if (response.type === "Word" && response._synthesized) {
+			lyricsIsAiGenerated = true;
+			sylLog(
+				`[RL-Syllable] AI generated syllable timings for ${response.data.length} lines (source: ${response.metadata.source})`,
+			);
+		}
+
 		lyricsMode = response.type === "Word" ? "word" : "line-api";
 		if (token !== trackChangeToken) return;
 		lyricsData =
@@ -4278,6 +4357,10 @@ const updateLyricsStyleFromSettings = (): void => {
 (window as any).updateLyricsStyle = updateLyricsStyleFromSettings;

 const updateRomanizeLyricsFromSettings = (): void => {
+	const checkbox = document.querySelector(
+		'input[data-setting="romanizeLyrics"]',
+	) as HTMLInputElement | null;
+	if (checkbox) checkbox.checked = settings.romanizeLyrics;
 	cachedLyricsKey = null;
 	cachedLyricsData = null;
 	cachedTidalRomanizeKey = null;
@@ -4286,6 +4369,19 @@ const updateRomanizeLyricsFromSettings = (): void => {
 };
 (window as any).updateRomanizeLyrics = updateRomanizeLyricsFromSettings;

+// Turning AI off reuses the line timings the AI response already has (no refetch). turning it on refetches with ?synthesize=true
+const updateAiSyllablesFromSettings = (): void => {
+	const checkbox = document.querySelector(
+		'input[data-setting="aiSyllables"]',
+	) as HTMLInputElement | null;
+	if (checkbox) checkbox.checked = settings.aiSyllables;
+	sylLog(
+		`[RL-Syllable] AI Generated Syllables ${settings.aiSyllables ? "enabled" : "disabled"}`,
+	);
+	toggle();
+};
+(window as any).updateAiSyllables = updateAiSyllablesFromSettings;
+
 // Update lyrics on track change (wipe cache for new song)
 onGlobalTrackChange(() => {
 	cachedLyricsKey = null;
@@ -142,6 +142,11 @@ body.rl-dropdown-open [data-test="toggle-lyrics"] {
 	gap: 10px;
 }

+/* Breathing room between stacked rows */
+.sticky-lyrics-dropdown-row + .sticky-lyrics-dropdown-row {
+	margin-top: 8px;
+}
+
 .sticky-lyrics-label {
 	font-size: 11px;
 	font-weight: 600;
@@ -599,21 +604,21 @@ body.rl-owns-lyric-scroll [class*="_syncButtonContainer_"] {
@keyframes rl-wipe {
 	from {
 		background-size:
-			0.75em 100%,
+			var(--rl-wipe-feather, 0.75em) 100%,
 			0% 100%,
 			100% 100%;
 		background-position:
-			-0.375em 0%,
+			calc(var(--rl-wipe-feather, 0.75em) / -2) 0%,
 			left,
 			left;
 	}
 	to {
 		background-size:
-			0.75em 100%,
+			var(--rl-wipe-feather, 0.75em) 100%,
 			100% 100%,
 			100% 100%;
 		background-position:
-			calc(100% + 0.375em) 0%,
+			calc(100% + var(--rl-wipe-feather, 0.75em) / 2) 0%,
 			left,
 			left;
 	}
@@ -640,11 +645,11 @@ body.rl-owns-lyric-scroll [class*="_syncButtonContainer_"] {
 		linear-gradient(90deg, var(--cl-glow1, #fff) 100%, transparent 100%),
 		linear-gradient(90deg, rgba(255, 255, 255, 0.4), rgba(255, 255, 255, 0.4));
 	background-size:
-		0.75em 100%,
+		var(--rl-wipe-feather, 0.75em) 100%,
 		0% 100%,
 		100% 100%;
 	background-position:
-		-0.375em 0%,
+		calc(var(--rl-wipe-feather, 0.75em) / -2) 0%,
 		left,
 		left;
 	/* biome-ignore lint: No glow for syllable mode */
@@ -653,6 +658,15 @@ body.rl-owns-lyric-scroll [class*="_syncButtonContainer_"] {
 	filter: none !important;
 }

+/* AI-generated timing: soften word-mode color flips so a boundary that's off
+   by a bit has no crisp edge to expose it. Per-span --rl-wipe-feather is set
+   inline from the AI model's confidence. SUPER WIP */
+.rl-wbw-container.rl-ai .rl-wbw-word {
+	transition:
+		text-shadow 0.25s ease-out,
+		color 0.3s ease-out;
+}
+
 /* Syllable finished: word stays Colorama color */
 .rl-wbw-word.rl-syl-finished {
 	/* biome-ignore lint: Kill base transitions so class swaps are instant */