The Separation Pipeline

Pipeline Overview

The stem separation pipeline has several stages:

1. Input: Load audio file
2. Preprocessing: Convert to spectrogram
3. Inference: Run neural network
4. Postprocessing: Convert back to audio, cleanup
5. Output: 4 stem audio files

Preprocessing

// Full preprocessing pipeline
async function preprocess(audioBuffer) {
    // 1. Ensure proper format
    if (audioBuffer.numberOfChannels > 1) {
        audioBuffer = toMono(audioBuffer);
    }
    
    // 2. Resample if needed (Spleeter expects 44100 Hz)
    if (audioBuffer.sampleRate !== 44100) {
        audioBuffer = await resample(audioBuffer, 44100);
    }
    
    // 3. Convert to spectrogram
    const spectrogram = audioToSpectrogram(audioBuffer, {
        fftSize: 4096,
        hopSize: 1024,
        window: 'hann'
    });
    
    // 4. Log scale (better for audio)
    const logSpectrogram = logScale(spectrogram);
    
    // 5. Normalize
    const normalized = normalize(logSpectrogram);
    
    // 6. Pad to fixed size (if needed)
    const padded = padToSize(normalized, [1024, 1024]);
    
    // 7. Create tensor
    const tensor = new ort.Tensor('float32', padded.data, [1, 1, 1024, 1024]);
    
    return tensor;
}

Postprocessing

// Full postprocessing pipeline
function postprocess(modelOutput, originalAudio) {
    // 1. Extract spectrograms for each stem
    const spectrograms = {
        vocals: modelOutput[0], // Channel 0
        drums: modelOutput[1], // Channel 1
        bass: modelOutput[2], // Channel 2
        other: modelOutput[3] // Channel 3
    };
    
    // 2. Inverse log scale
    const linearSpectrograms = {};
    for (const [stem, spec] of Object.entries(spectrograms)) {
        linearSpectrograms[stem] = inverseLogScale(spec);
    }
    
    // 3. Convert back to audio (inverse STFT)
    const audioStems = {};
    for (const [stem, spec] of Object.entries(linearSpectrograms)) {
        audioStems[stem] = spectrogramToAudio(spec, {
            fftSize: 4096,
            hopSize: 1024,
            originalPhase: extractPhase(originalAudio)
        });
    }
    
    // 4. Apply cleanup filters
    const cleaned = cleanupStems(audioStems);
    
    // 5. Convert back to stereo (if original was stereo)
    const stereoStems = {};
    for (const [stem, audio] of Object.entries(cleaned)) {
        stereoStems[stem] = toStereo(audio);
    }
    
    // 6. Match original duration
    const matched = matchDuration(stereoStems, originalAudio.duration);
    
    return matched;
}

// Cleanup: remove artifacts
function cleanupStems(stems) {
    // High-pass filter on vocals (remove sub-bass rumble)
    stems.vocals = applyFilter(stems.vocals, 'highpass', 80);
    
    // Low-pass filter on bass (remove ultra-high artifacts)
    stems.bass = applyFilter(stems.bass, 'lowpass', 5000);
    
    // Smooth other (reduce harshness)
    stems.other = applyFilter(stems.other, 'peaking', 3000, -2);
    
    return stems;
}

Chunked Processing

For long audio, process in chunks to avoid memory issues:

// Chunked processing
async function processLongAudio(audioBuffer, chunkDuration = 5) {
    const chunks = splitIntoChunks(audioBuffer, chunkDuration);
    const results = {
        vocals: [],
        drums: [],
        bass: [],
        other: []
    };
    
    for (let i = 0; i < chunks.length; i++) {
        const chunk = chunks[i];
        
        // Preprocess
        const input = await preprocess(chunk);
        
        // Inference
        const output = await model.run(input);
        
        // Postprocess
        const stems = postprocess(output, chunk);
        
        // Store results
        results.vocals.push(stems.vocals);
        results.drums.push(stems.drums);
        results.bass.push(stems.bass);
        results.other.push(stems.other);
        
        // Update progress
        updateProgress((i + 1) / chunks.length * 100);
    }
    
    // Concatenate chunks
    const finalStems = {
        vocals: concatenateAudio(results.vocals),
        drums: concatenateAudio(results.drums),
        bass: concatenateAudio(results.bass),
        other: concatenateAudio(results.other)
    };
    
    return finalStems;
}

// MixMind's complete separation pipeline
class SeparationPipeline {
    constructor() {
        this.model = new SpleeterModel();
        this.progress = 0;
    }
    
    async separate(file) {
        try {
            // 1. Load audio
            this.updateProgress(5, 'Loading audio...');
            const audioBuffer = await this.loadAudio(file);
            
            // 2. Analyze
            this.updateProgress(10, 'Analyzing audio...');
            const analysis = await this.analyze(audioBuffer);
            
            // 3. Initialize model
            this.updateProgress(15, 'Initializing AI model...');
            await this.model.load();
            
            // 4. Separate stems
            this.updateProgress(20, 'Separating stems...');
            const stems = await this.model.separate(audioBuffer);
            
            // 5. Post-process
            this.updateProgress(80, 'Cleaning up...');
            const cleaned = this.cleanupStems(stems);
            
            // 6. Generate previews
            this.updateProgress(90, 'Generating previews...');
            const previews = this.generatePreviews(cleaned);
            
            // 7. Complete
            this.updateProgress(100, 'Complete!');
            
            return {
                stems: cleaned,
                analysis: analysis,
                previews: previews
            };
            
        } catch (error) {
            console.error('Separation failed:', error);
            throw error;
        }
    }
    
    updateProgress(percent, message) {
        this.progress = percent;
        // Emit progress event
        window.dispatchEvent(new CustomEvent('separation-progress', {
            detail: { percent, message }
        }));
    }
}

// Usage in MixMind
async function handleSeparation(file) {
    const pipeline = new SeparationPipeline();
    
    // Listen for progress updates
    window.addEventListener('separation-progress', (event) => {
        updateUI(event.detail);
    });
    
    // Run separation
    const result = await pipeline.separate(file);
    
    // Display results
    displayStems(result.stems);
}