feat(audioVisualization): add spotify support and linear animations

This commit is contained in:
vMohammad24
2025-06-12 15:29:32 +03:00
parent 278f6249dd
commit 24aabe67fd
2 changed files with 465 additions and 165 deletions
+21 -4
View File
@@ -1,18 +1,20 @@
import { ReactiveStore } from "@luna/core";
import { LunaSettings, LunaNumberSetting, LunaSwitchSetting, LunaTextSetting } from "@luna/ui";
import { LunaNumberSetting, LunaSettings, LunaSwitchSetting } from "@luna/ui";
import React from "react";
const isWindows = navigator.userAgent.includes("Windows"); // tidal changes it in reqs navigator supplies the default electron one
export const settings = await ReactiveStore.getPluginStorage("AudioVisualizer", {
barCount: 32,
barColor: "#ffffff",
barRounding: true,
customColors: [] as string[]
customColors: [] as string[],
spotifyAPI: isWindows
});
export const Settings = () => {
const [barCount, setBarCount] = React.useState(settings.barCount);
const [barColor, setBarColor] = React.useState(settings.barColor);
const [barRounding, setBarRounding] = React.useState(settings.barRounding);
const [spotifyAPI, setSpotifyAPI] = React.useState(settings.spotifyAPI);
const [showColorPicker, setShowColorPicker] = React.useState(false);
const [isAnimatingIn, setIsAnimatingIn] = React.useState(false);
const [shouldRender, setShouldRender] = React.useState(false);
@@ -87,11 +89,26 @@ export const Settings = () => {
const allColors = [...colorPresets, ...customColors];
return (
<LunaSettings>
<LunaSettings> <LunaSwitchSetting
title="Spotify API"
desc="Use Spotify's audio analysis API instead of real-time audio data (Required for Windows)"
// @ts-expect-error no idea why this errosr wth
checked={spotifyAPI}
disabled={isWindows} // Disable on non-Windows platforms
// @ts-expect-error no idea why this errosr wth
onChange={(_, checked) => {
setSpotifyAPI(checked);
settings.spotifyAPI = checked;
(window as any).updateAudioVisualizer?.();
}}
/>
<LunaSwitchSetting
title="Bar Roundness"
desc="Enable rounded corners on visualizer bars"
// @ts-expect-error no idea why this errosr wth
checked={barRounding}
// @ts-expect-error no idea why this errosr wth
onChange={(_, checked) => {
setBarRounding(checked);
settings.barRounding = checked;
+356 -73
View File
@@ -1,5 +1,5 @@
import { LunaUnload, Tracer } from "@luna/core";
import { StyleTag, PlayState } from "@luna/lib";
import { ftch, LunaUnload, Tracer } from "@luna/core";
import { PlayState, StyleTag } from "@luna/lib";
import { settings, Settings } from "./Settings";
// Import CSS styles for the visualizer
@@ -8,9 +8,9 @@ import visualizerStyles from "file://styles.css?minify";
export const { trace } = Tracer("[Audio Visualizer]");
// Helper function for consistent logging
const log = (message: string) => console.log(`[Audio Visualizer] ${message}`);
const warn = (message: string) => console.warn(`[Audio Visualizer] ${message}`);
const error = (message: string) => console.error(`[Audio Visualizer] ${message}`);
const log = (message: string) => trace.log(message);
const warn = (message: string) => trace.warn(message);
const error = (message: string) => trace.err(message);
export { Settings };
// Basic config with settings
@@ -33,6 +33,80 @@ export const unloads = new Set<LunaUnload>();
// StyleTag for CSS
const styleTag = new StyleTag("AudioVisualizer", unloads, visualizerStyles);
interface SpotifyAudioAnalysis {
meta: {
analyzer_version: string;
platform: string;
detailed_status: string;
status_code: number;
timestamp: number;
analysis_time: number;
input_process: string;
};
track: {
num_samples: number;
duration: number;
sample_md5: string;
offset_seconds: number;
window_seconds: number;
analysis_sample_rate: number;
analysis_channels: number;
end_of_fade_in: number;
start_of_fade_out: number;
loudness: number;
tempo: number;
tempo_confidence: number;
time_signature: number;
time_signature_confidence: number;
key: number;
key_confidence: number;
mode: number;
mode_confidence: number;
codestring: string;
code_version: number;
echoprintstring: string;
echoprint_version: number;
synchstring: string;
synch_version: number;
rhythmstring: string;
rhythm_version: number;
};
bars: Array<{
start: number;
duration: number;
confidence: number;
}>;
beats: Array<{
start: number;
duration: number;
confidence: number;
}>;
sections: Array<{
[key: string]: number;
}>;
segments: Array<{
start: number;
duration: number;
confidence: number;
loudness_start: number;
loudness_max_time: number;
loudness_max: number;
loudness_end: number;
pitches: number[];
timbre: number[];
}>;
tatums: Array<{
start: number;
duration: number;
confidence: number;
}>;
}
let spotifyAudioAnalysis: SpotifyAudioAnalysis | null = null;
let currentTrackId: string | null = null;
let lastSpotifyFetchTime = 0;
const SPOTIFY_FETCH_THROTTLE = 1000;
// Audio context and analyzer
let audioContext: AudioContext | null = null;
let analyser: AnalyserNode | null = null;
@@ -42,17 +116,60 @@ let animationId: number | null = null;
let currentAudioElement: HTMLAudioElement | null = null;
let isSourceConnected: boolean = false;
let smoothedBars: number[] = [];
let previousBars: number[] = [];
const smoothingFactor = 0.15;
// Canvas and container elements
let visualizerContainer: HTMLDivElement | null = null;
let canvas: HTMLCanvasElement | null = null;
let canvasContext: CanvasRenderingContext2D | null = null;
const fetchSpotifyAudioAnalysis = async (): Promise<void> => {
try {
const trackId = PlayState.playbackContext?.actualProductId;
if (!trackId) {
warn("No track ID available for Spotify API");
return;
}
if (currentTrackId === trackId && spotifyAudioAnalysis) {
log("Using cached Spotify audio analysis");
return;
}
const now = Date.now();
if (now - lastSpotifyFetchTime < SPOTIFY_FETCH_THROTTLE) {
log("Throttling Spotify API call");
return;
}
lastSpotifyFetchTime = now;
log(`Fetching Spotify audio analysis for track: ${trackId}`);
const data = await ftch.json<{
audioAnalysis: SpotifyAudioAnalysis;
}>(`https://api.vmohammad.dev/lyrics?tidal_id=${trackId}&filter=audioAnalysis`);
if (!data.audioAnalysis) {
warn("No audio analysis data in API response");
return;
}
spotifyAudioAnalysis = data.audioAnalysis;
currentTrackId = trackId;
log("Successfully fetched Spotify audio analysis");
} catch (err) {
error(`Failed to fetch Spotify audio analysis: ${err}`);
spotifyAudioAnalysis = null;
}
};
// Find the audio element - this is a bit of a hack but it works
const findAudioElement = (): HTMLAudioElement | null => {
// Try main selectors first
const selectors = [
'audio',
'video',
// 'video',
'audio[data-test]',
'[data-test="audio-player"] audio'
];
@@ -65,7 +182,7 @@ const findAudioElement = (): HTMLAudioElement | null => {
}
// Quick scan for any audio elements
const audioElements = document.querySelectorAll('audio, video');
const audioElements = document.querySelectorAll('audio');
for (const element of audioElements) {
const audioEl = element as HTMLAudioElement;
if (audioEl.src || audioEl.currentSrc) {
@@ -79,6 +196,10 @@ const findAudioElement = (): HTMLAudioElement | null => {
// Initialize audio visualization
const initializeAudioVisualizer = async (): Promise<void> => {
try {
if (settings.spotifyAPI) {
await fetchSpotifyAudioAnalysis();
log("Using Spotify API - skipping audio element connection");
} else {
// Find the audio element
const audioElement = findAudioElement();
if (!audioElement) {
@@ -96,7 +217,8 @@ const initializeAudioVisualizer = async (): Promise<void> => {
analyser = audioContext.createAnalyser();
analyser.fftSize = 512; // Fixed power of 2 that provides enough frequency bins
analyser.smoothingTimeConstant = config.smoothing;
dataArray = new Uint8Array(analyser.frequencyBinCount);
const buffer = new ArrayBuffer(analyser.frequencyBinCount);
dataArray = new Uint8Array(buffer);
log("Created AnalyserNode");
}
@@ -124,7 +246,8 @@ const initializeAudioVisualizer = async (): Promise<void> => {
// Resume context only if needed and don't wait for it
// (otherwise it will wait for the audio to start playing)
if (audioContext.state === 'suspended') {
audioContext.resume().catch(() => {}); // Fire and forget
audioContext.resume().catch(() => { }); // Fire and forget
}
}
// Create UI only if it doesn't exist
@@ -209,6 +332,17 @@ const removeVisualizerUI = (): void => {
}
};
const lerp = (start: number, end: number, factor: number): number => {
return start + (end - start) * factor;
};
const initializeSmoothingArrays = (barCount: number): void => {
if (smoothedBars.length !== barCount) {
smoothedBars = new Array(barCount).fill(0);
previousBars = new Array(barCount).fill(0);
}
};
// Animation loop for rendering visualizer
const animate = (): void => {
if (!canvasContext || !canvas) {
@@ -219,19 +353,30 @@ const animate = (): void => {
// Update canvas color in case it changed
canvasContext.fillStyle = config.color;
canvasContext.strokeStyle = config.color;
canvasContext.clearRect(0, 0, canvas.width, canvas.height);
if (settings.spotifyAPI && spotifyAudioAnalysis) {
switch (config.visualizerType) {
case 'bars':
drawSpotifyBars();
break;
case 'waveform':
// drawWaveform();
break;
case 'circular':
// drawCircular();
break;
}
} else {
// Check if we have real audio data - this might not be needed but its a good idea
let hasRealAudio = false;
if (analyser && dataArray) {
analyser.getByteFrequencyData(dataArray);
analyser.getByteFrequencyData(dataArray as any);
// Check if there's actual audio signal (not just silence)
const avgVolume = dataArray.reduce((sum, val) => sum + val, 0) / dataArray.length;
hasRealAudio = avgVolume > 5; // Threshold for detecting actual audio
}
// Clear canvas
canvasContext.clearRect(0, 0, canvas.width, canvas.height);
if (hasRealAudio && analyser && dataArray) {
// Draw real audio visualization
switch (config.visualizerType) {
@@ -239,16 +384,17 @@ const animate = (): void => {
drawBars();
break;
case 'waveform': // Not implemented yet
drawWaveform();
// drawWaveform();
break;
case 'circular': // Not implemented yet
drawCircular();
// drawCircular();
break;
}
} else {
// Draw cool scrolling wave effect when no audio
drawScrollingWave();
}
}
animationId = requestAnimationFrame(animate);
};
@@ -270,6 +416,8 @@ const drawScrollingWave = (): void => {
waveTime += 0.05; // Speed of wave animation
const barCount = config.barCount;
initializeSmoothingArrays(barCount);
const barWidth = canvas.width / barCount;
const maxHeight = canvas.height * 0.6;
@@ -289,16 +437,18 @@ const drawScrollingWave = (): void => {
const travelWave = Math.sin(x * Math.PI * 3 - waveTime * 2) * 0.5 + 0.5;
// Final height calculation
const barHeight = maxHeight * combinedWave * travelWave * 0.8 + 2; // Minimum height of 2px
const targetHeight = maxHeight * combinedWave * travelWave * 0.8 + 2; // Minimum height of 2px
smoothedBars[i] = lerp(smoothedBars[i], targetHeight, smoothingFactor * 2); // Faster smoothing for wave effect
const xPos = i * barWidth;
const yPos = (canvas.height - barHeight) / 2;
const yPos = (canvas.height - smoothedBars[i]) / 2;
// Draw rounded or square bars based on setting
if (config.barRounding) {
drawRoundedRect(canvasContext, xPos, yPos, barWidth - 1, barHeight, 2);
drawRoundedRect(canvasContext, xPos, yPos, barWidth - 1, smoothedBars[i], 2);
} else {
canvasContext.fillRect(xPos, yPos, barWidth - 1, barHeight);
canvasContext.fillRect(xPos, yPos, barWidth - 1, smoothedBars[i]);
}
}
};
@@ -307,82 +457,181 @@ const drawScrollingWave = (): void => {
const drawBars = (): void => {
if (!canvasContext || !dataArray || !canvas) return;
const barWidth = canvas.width / config.barCount;
const barCount = config.barCount;
initializeSmoothingArrays(barCount);
const barWidth = canvas.width / barCount;
const heightScale = canvas.height / 255;
canvasContext.fillStyle = config.color;
for (let i = 0; i < config.barCount; i++) {
const dataIndex = Math.floor(i * (dataArray.length / config.barCount));
const barHeight = (dataArray[dataIndex] * config.sensitivity * heightScale);
for (let i = 0; i < barCount; i++) {
const dataIndex = Math.floor(i * (dataArray.length / barCount));
const targetHeight = (dataArray[dataIndex] * config.sensitivity * heightScale);
smoothedBars[i] = lerp(smoothedBars[i], targetHeight, smoothingFactor);
const x = i * barWidth;
const y = canvas.height - barHeight;
const y = canvas.height - smoothedBars[i];
// Draw rounded or square bars based on setting
if (config.barRounding) {
drawRoundedRect(canvasContext, x, y, barWidth - 1, barHeight, 2);
drawRoundedRect(canvasContext, x, y, barWidth - 1, smoothedBars[i], 2);
} else {
canvasContext.fillRect(x, y, barWidth - 1, barHeight);
canvasContext.fillRect(x, y, barWidth - 1, smoothedBars[i]);
}
}
};
// Draw waveform visualization - NOT IMPLEMENTED YET
// const drawWaveform = (): void => {
// if (!canvasContext || !dataArray || !canvas) return;
let currentTime = 0;
let previousTime = 0;
let lastUpdated = 0;
const drawSpotifyBars = (): void => {
if (!canvasContext || !canvas || !spotifyAudioAnalysis) return;
// const centerY = canvas.height / 2;
// const amplitudeScale = canvas.height / 512;
const audioElement = findAudioElement();
if (audioElement && audioElement.currentTime) {
currentTime = audioElement.currentTime;
previousTime = -1;
} else {
const progressBar = document.querySelector('[data-test="progress-bar"]') as HTMLElement;
if (progressBar) {
const ariaValueNow = progressBar.getAttribute('aria-valuenow');
if (ariaValueNow !== null) {
const progressTime = Number.parseInt(ariaValueNow);
const now = Date.now();
// canvasContext.strokeStyle = config.color;
// canvasContext.lineWidth = 2;
// canvasContext.beginPath();
if (progressTime !== previousTime) {
currentTime = progressTime;
previousTime = progressTime;
lastUpdated = now;
} else if (PlayState.playing) {
const elapsedSeconds = (now - lastUpdated) / 1000;
currentTime = progressTime + elapsedSeconds;
}
} else {
warn("Progress bar not found or aria-valuenow is null");
return;
}
}
}
// for (let i = 0; i < config.barCount; i++) {
// const dataIndex = Math.floor(i * (dataArray.length / config.barCount));
// const amplitude = (dataArray[dataIndex] - 128) * config.sensitivity * amplitudeScale;
if (currentTime < 0) return;
// const x = (i / config.barCount) * canvas.width;
// const y = centerY + amplitude;
const barCount = config.barCount;
initializeSmoothingArrays(barCount);
// if (i === 0) {
// canvasContext.moveTo(x, y);
// } else {
// canvasContext.lineTo(x, y);
// }
// }
const barWidth = canvas.width / barCount;
canvasContext.fillStyle = config.color;
// canvasContext.stroke();
// };
const segments = spotifyAudioAnalysis.segments;
const beats = spotifyAudioAnalysis.beats;
// Draw circular visualization - NOT IMPLEMENTED YET
// const drawCircular = (): void => {
// if (!canvasContext || !dataArray || !canvas) return;
if (!segments || segments.length === 0) {
warn("No segments data available in Spotify audio analysis");
return;
}
// const centerX = canvas.width / 2;
// const centerY = canvas.height / 2;
// const radius = Math.min(centerX, centerY) - 10;
let currentSegmentIndex = segments.findIndex(segment =>
currentTime >= segment.start && currentTime < (segment.start + segment.duration)
);
// canvasContext.strokeStyle = config.color;
// canvasContext.lineWidth = 2;
if (currentSegmentIndex === -1) {
currentSegmentIndex = segments.reduce((closestIndex, segment, index) => {
const closestDiff = Math.abs(segments[closestIndex].start - currentTime);
const segmentDiff = Math.abs(segment.start - currentTime);
return segmentDiff < closestDiff ? index : closestIndex;
}, 0);
}
// for (let i = 0; i < config.barCount; i++) {
// const dataIndex = Math.floor(i * (dataArray.length / config.barCount));
// const amplitude = (dataArray[dataIndex] * config.sensitivity) / 255;
const currentSegment = segments[currentSegmentIndex];
if (!currentSegment) return;
// const angle = (i / config.barCount) * Math.PI * 2;
// const startX = centerX + Math.cos(angle) * radius * 0.7;
// const startY = centerY + Math.sin(angle) * radius * 0.7;
// const endX = centerX + Math.cos(angle) * radius * (0.7 + amplitude * 0.3);
// const endY = centerY + Math.sin(angle) * radius * (0.7 + amplitude * 0.3);
const nextSegment = segments[currentSegmentIndex + 1];
const segmentProgress = (currentTime - currentSegment.start) / currentSegment.duration;
const interpolationFactor = Math.max(0, Math.min(1, segmentProgress));
const currentBeat = beats?.find(beat =>
currentTime >= beat.start && currentTime < (beat.start + beat.duration)
);
let beatIntensity = 1.0;
if (currentBeat) {
const beatProgress = (currentTime - currentBeat.start) / currentBeat.duration;
beatIntensity = 1.0 + (1.0 - beatProgress) * currentBeat.confidence;
}
const getInterpolatedValue = (currentValue: number, nextValue?: number): number => {
if (!nextValue || !nextSegment) return currentValue;
return lerp(currentValue, nextValue, interpolationFactor * 0.3); // Gentle interpolation
};
const currentLoudness = currentSegment.loudness_max;
const nextLoudness = nextSegment?.loudness_max ?? currentLoudness;
const interpolatedLoudness = getInterpolatedValue(currentLoudness, nextLoudness);
const loudnessMultiplier = Math.max(0.3, Math.min(1.2, (interpolatedLoudness + 80) / 80));
for (let i = 0; i < barCount; i++) {
let targetHeight = 0;
const pitches = currentSegment.pitches;
const timbre = currentSegment.timbre;
const nextPitches = nextSegment?.pitches;
const nextTimbre = nextSegment?.timbre;
if (i < 12 && pitches.length >= 12) {
const currentPitch = pitches[i];
const nextPitch = nextPitches?.[i];
const interpolatedPitch = getInterpolatedValue(currentPitch, nextPitch);
targetHeight = Math.pow(interpolatedPitch, 1.2) * canvas.height * config.sensitivity * 0.6;
} else if (i < 24 && timbre.length >= 12) {
const timbreIndex = (i - 12) % timbre.length;
const currentTimbreValue = timbre[timbreIndex];
const nextTimbreValue = nextTimbre?.[timbreIndex];
const interpolatedTimbre = getInterpolatedValue(currentTimbreValue, nextTimbreValue);
const normalizedTimbre = Math.max(0, Math.min(1, (interpolatedTimbre + 200) / 400));
targetHeight = Math.pow(normalizedTimbre, 1.0) * canvas.height * config.sensitivity * 0.4;
} else {
const harmonicIndex = i % 12;
const harmonicMultiplier = Math.max(0.2, 1.0 - (Math.floor(i / 12) * 0.3));
const basePitch = pitches[harmonicIndex] || 0;
const nextBasePitch = nextPitches?.[harmonicIndex];
const interpolatedPitch = getInterpolatedValue(basePitch, nextBasePitch);
targetHeight = Math.pow(interpolatedPitch, 1.3) * canvas.height * config.sensitivity * harmonicMultiplier * 0.5;
}
targetHeight *= loudnessMultiplier * beatIntensity;
const frequencyVariance = 1.0 + Math.sin((i / barCount) * Math.PI * 2 + currentTime * 0.5) * 0.05;
targetHeight *= frequencyVariance;
if (targetHeight > canvas.height * 0.6) {
targetHeight = canvas.height * 0.6 + (targetHeight - canvas.height * 0.6) * 0.2;
}
targetHeight = Math.max(2, Math.min(targetHeight, canvas.height));
const changeFactor = Math.abs(targetHeight - (smoothedBars[i] || 0)) / canvas.height;
const adaptiveSmoothingFactor = smoothingFactor * (1 + changeFactor * 0.5);
smoothedBars[i] = lerp(smoothedBars[i] || 0, targetHeight, Math.min(adaptiveSmoothingFactor, 0.3));
const x = i * barWidth;
const y = canvas.height - smoothedBars[i];
if (config.barRounding) {
drawRoundedRect(canvasContext, x, y, barWidth - 1, smoothedBars[i], 2);
} else {
canvasContext.fillRect(x, y, barWidth - 1, smoothedBars[i]);
}
}
};
// canvasContext.beginPath();
// canvasContext.moveTo(startX, startY);
// canvasContext.lineTo(endX, endY);
// canvasContext.stroke();
// }
// };
// Update visualizer settings
const updateAudioVisualizer = (): void => {
@@ -390,7 +639,8 @@ const updateAudioVisualizer = (): void => {
// use a fixed size that provides enough frequency bins
analyser.fftSize = 512; // Fixed power of 2 - important
analyser.smoothingTimeConstant = config.smoothing;
dataArray = new Uint8Array(analyser.frequencyBinCount);
const buffer = new ArrayBuffer(analyser.frequencyBinCount); // buffer like ahh
dataArray = new Uint8Array(buffer);
}
if (canvas) {
@@ -400,6 +650,23 @@ const updateAudioVisualizer = (): void => {
canvas.style.height = `${config.height}px`;
}
smoothedBars = [];
previousBars = [];
if (settings.spotifyAPI) {
const currentSpotifyTrackId = PlayState.playbackContext?.actualProductId;
if (currentSpotifyTrackId && currentSpotifyTrackId !== currentTrackId) {
log("Spotify API enabled, fetching audio analysis");
fetchSpotifyAudioAnalysis().catch(err => {
error(`Failed to fetch Spotify data: ${err}`);
});
}
} else {
spotifyAudioAnalysis = null;
currentTrackId = null;
log("Spotify API disabled, cleared audio analysis data");
}
// Recreate UI if position changed
createVisualizerUI();
};
@@ -425,24 +692,35 @@ const cleanupAudioVisualizer = (): void => {
const observePlayState = (): void => {
let hasTriedInitialization = false;
let checkCount = 0;
let lastTrackIdForSpotify: string | null = null;
const checkAndInitialize = () => {
checkCount++;
if (settings.spotifyAPI) {
const currentSpotifyTrackId = PlayState.playbackContext?.actualProductId;
if (currentSpotifyTrackId && currentSpotifyTrackId !== lastTrackIdForSpotify) {
lastTrackIdForSpotify = currentSpotifyTrackId;
log(`Track changed, fetching Spotify data for: ${currentSpotifyTrackId}`);
fetchSpotifyAudioAnalysis().catch(err => {
error(`Failed to fetch Spotify data: ${err}`);
});
}
}
// Only try to initialize once when music starts playing
if (PlayState.playing && !hasTriedInitialization) {
if ((PlayState.playing || settings.spotifyAPI) && !hasTriedInitialization) {
hasTriedInitialization = true;
log("Initializing audio visualizer...");
// Initialize immediately - no delay (after audio starts playing ofc)
initializeAudioVisualizer().then(() => {
if (audioContext && analyser) {
if (settings.spotifyAPI || (audioContext && analyser)) {
log("Audio visualizer ready!");
} else {
hasTriedInitialization = false; // Allow retry if failed
}
});
} else if (!PlayState.playing && hasTriedInitialization) {
} else if (!PlayState.playing && !settings.spotifyAPI && hasTriedInitialization) {
// Reset try flag when music stops so it can try again next time (otherwise it explode)
hasTriedInitialization = false;
}
@@ -519,6 +797,11 @@ const completeCleanup = (): void => {
dataArray = null;
currentAudioElement = null;
isSourceConnected = false;
smoothedBars = [];
previousBars = [];
spotifyAudioAnalysis = null;
currentTrackId = null;
log("Cleaned up Spotify API data");
};
// Register cleanup