similarityUtils.js

// -----------------------------------------------------
// -- Calculate cosine similarity between two vectors --
// -- (delegated to embedding-utils; re-exported for  --
// --  any deep importers of this module)             --
// -----------------------------------------------------
import { cosineSimilarity } from 'embedding-utils';
export { cosineSimilarity };

// ---------------------------------------------------------------
// -- Function to compute advanced similarities with statistics --
// ---------------------------------------------------------------
/**
 * @param {string[]} sentences - Array of sentences to compute similarities for
 * @param {object} options - Options for similarity computation
 * @param {number} [options.numSimilaritySentencesLookahead=2] - Number of sentences to look ahead for similarity comparison
 * @param {boolean} [options.logging=false] - Whether to log debug information
 * @param {function(string[]): Promise<number[][]>} embedBatch - Function to compute embeddings for array of texts
 * @returns {{similarities: number[], average: number, variance: number, embeddings: number[][]}} Object containing similarity scores, statistics, and embeddings
 */
export async function computeAdvancedSimilarities(sentences, { numSimilaritySentencesLookahead = 2, logging = false } = {}, embedBatch) {
    if (typeof embedBatch !== 'function') {
        throw new Error('embedBatch must be a function');
    }
    if (!sentences || sentences.length === 0) {
        return { similarities: [], average: 0, variance: 0, embeddings: [] };
    }
    if (logging) console.log('numSimilaritySentencesLookahead', numSimilaritySentencesLookahead);

    const embeddings = await embedBatch(sentences);
    let similarities = [];
    let similaritySum = 0;

    for (let i = 0; i < embeddings.length - 1; i++) {
        let maxSimilarity = cosineSimilarity(embeddings[i], embeddings[i + 1]);

        if (logging) {
            console.log(`\nSimilarity scores for sentence ${i}:`);
            console.log(`Base similarity with next sentence: ${maxSimilarity}`);
        }

        for (let j = i + 2; j <= i + numSimilaritySentencesLookahead && j < embeddings.length; j++) {
            const sim = cosineSimilarity(embeddings[i], embeddings[j]);
            if (logging) {
                console.log(`Similarity with sentence ${j}: ${sim}`);
            }
            maxSimilarity = Math.max(maxSimilarity, sim);
        }

        similarities.push(maxSimilarity);
        similaritySum += maxSimilarity;
    }

    const average = similaritySum / similarities.length;
    const variance = similarities.reduce((acc, sim) => acc + (sim - average) ** 2, 0) / similarities.length;

    return { similarities, average, variance, embeddings };
}

// -----------------------------------------------------------
// -- Function to dynamically adjust the similarity threshold --
// -----------------------------------------------------------
export function adjustThreshold(average, variance, baseThreshold = 0.5, lowerBound = 0.2, upperBound = 0.8) {
    if (lowerBound >= upperBound) {
        console.error("Invalid bounds: lowerBound must be less than upperBound.");
        return baseThreshold;
    }

    let adjustedThreshold = baseThreshold;
    if (variance < 0.01) {
        adjustedThreshold -= 0.1;
    } else if (variance > 0.05) {
        adjustedThreshold += 0.1;
    }

    if (average < 0.3) {
        adjustedThreshold += 0.05;
    } else if (average > 0.7) {
        adjustedThreshold -= 0.05;
    }

    return Math.min(Math.max(adjustedThreshold, lowerBound), upperBound);
}