matching_algorithms.py

# matching_algorithms.py
from fuzzywuzzy import fuzz
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from openai import AzureOpenAI
import json
import numpy as np
import os
import pickle
import faiss
from rank_bm25 import BM25Okapi
import logging
import time

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

# Initialize Azure OpenAI client
client = AzureOpenAI(
    api_key=os.getenv('AZURE_OPENAI_API_KEY'),
    api_version=os.getenv('AZURE_OPENAI_API_VERSION', '2024-05-01-preview'),
    azure_endpoint=os.getenv('AZURE_OPENAI_ENDPOINT')
)

def fuzzy_search(query, corpus, threshold=50):
    if not query or not corpus:
        return []
    matches = [(i, fuzz.token_sort_ratio(query, item)) for i, item in enumerate(corpus)]
    return sorted([(i, score) for i, score in matches if score >= threshold], key=lambda x: x[1], reverse=True)

def semantic_search_tfidf(query, corpus, threshold=0.5):
    if not query or not corpus:
        return []
    vectorizer = TfidfVectorizer().fit(corpus)
    query_vec = vectorizer.transform([query])
    corpus_vec = vectorizer.transform(corpus)
    similarities = cosine_similarity(query_vec, corpus_vec)[0]
    matches = [(i, sim) for i, sim in enumerate(similarities) if sim >= threshold]
    return sorted(matches, key=lambda x: x[1], reverse=True)

def get_embedding(text, model="text-embedding-3-large"):
    text = text.replace("\n", " ")
    start_time = time.time()
    logger.info(f"Generating embedding for text: {text[:50]}...")
    response = client.embeddings.create(input=[text], model=model)
    embedding = response.data[0].embedding
    logger.info(f"Embedding generated in {time.time() - start_time:.2f} seconds")
    return embedding

def cache_corpus_embeddings(corpus, cache_file='corpus_embeddings.pkl'):
    logger.info(f"Generating embeddings for {len(corpus)} items")
    start_time = time.time()
    embeddings = [get_embedding(text) for text in corpus]
    logger.info(f"All embeddings generated in {time.time() - start_time:.2f} seconds")
    with open(cache_file, 'wb') as f:
        pickle.dump(embeddings, f)
    logger.info(f"Embeddings saved to {cache_file}")
    return embeddings
def build_faiss_index(embeddings):
    dimension = len(embeddings[0])
    index = faiss.IndexFlatIP(dimension) 
    print("THIS IS INDEX ",index) # Inner product is equivalent to cosine similarity for normalized vectors
    embeddings_array = np.array(embeddings).astype('float32')
    print(embeddings_array.dtype)
    faiss.normalize_L2(embeddings_array)  # Normalize the vectors
    index.add(embeddings_array)
    return index

# def l2_normalize(v):
#     norm = np.linalg.norm(v, axis=1, keepdims=True)
#     return v / norm

# def build_faiss_index(embeddings):
#     print("Incoming embeddings type:", type(embeddings))
#     print("Number of embeddings:", len(embeddings))
    
#     try:
#         # Convert list of lists to numpy array
#         embeddings_array = np.array(embeddings, dtype=np.float32)
        
#         print("After conversion:")
#         print("Type:", type(embeddings_array))
#         print("Shape:", embeddings_array.shape)
#         print("Data type:", embeddings_array.dtype)

#         dimension = embeddings_array.shape[1]
#         index = faiss.IndexFlatIP(dimension)
        
#         print("THIS IS INDEX ", index)
        
#         print("Before normalization:")
#         print("Embeddings array type:", type(embeddings_array))
#         print("Embeddings array shape:", embeddings_array.shape)
        
#         # Normalize the vectors using our custom function
#         try:
#             embeddings_array = l2_normalize(embeddings_array)
#             print("Normalization successful")
#         except Exception as e:
#             print(f"Error during normalization: {e}")
#             raise
        
#         print("Before adding to index:")
#         print("Embeddings array type:", type(embeddings_array))
#         print("Embeddings array shape:", embeddings_array.shape)
#         print("Embeddings array dtype:", embeddings_array.dtype)
#         print("Is array contiguous:", embeddings_array.flags['C_CONTIGUOUS'])
        
#         # Ensure array is contiguous and float32
#         embeddings_array = np.ascontiguousarray(embeddings_array, dtype=np.float32)
        
#         print("After ensuring contiguous array:")
#         print("Embeddings array type:", type(embeddings_array))
#         print("Embeddings array shape:", embeddings_array.shape)
#         print("Embeddings array dtype:", embeddings_array.dtype)
#         print("Is array contiguous:", embeddings_array.flags['C_CONTIGUOUS'])
        
#         # Add to the index
#         try:
#             index.add(embeddings_array)
#             print("Successfully added to index")
#         except Exception as e:
#             print(f"Error adding to index: {e}")
#             print("Index type:", type(index))
#             print("Index dimension:", index.d)
#             raise
        
#         print("Index created successfully.")
#         print("Number of vectors in index:", index.ntotal)
        
#         return index
#     except Exception as e:
#         print(f"Error in build_faiss_index: {e}")
#         import traceback
#         traceback.print_exc()
#         raise

def load_cached_embeddings(cache_file='corpus_embeddings.pkl'):
    logger.info(f"Loading cached embeddings from {cache_file}")
    start_time = time.time()
    with open(cache_file, 'rb') as f:
        embeddings = pickle.load(f)
    logger.info(f"Loaded {len(embeddings)} embeddings in {time.time() - start_time:.2f} seconds")
    print("help me")
    print(len(embeddings))
    print("this is it =========>")
    print(type(embeddings))
    return embeddings
# def load_cached_embeddings(cache_file='corpus_embeddings.pkl'):
#     logger.info(f"Loading cached embeddings from {cache_file}")
#     start_time = time.time()
    
#     try:
#         with open(cache_file, 'rb') as f:
#             embeddings = pickle.load(f)
        
#         load_time = time.time() - start_time
#         logger.info(f"Loaded {len(embeddings)} embeddings in {load_time:.2f} seconds")
        
#         print("Embeddings loaded successfully")
#         print(f"Number of embeddings: {len(embeddings)}")
#         print(f"Type of embeddings: {type(embeddings)}")
        
#         if isinstance(embeddings, list):
#             print(f"Type of first embedding: {type(embeddings[0])}")
#             if embeddings and isinstance(embeddings[0], list):
#                 print(f"Length of first embedding: {len(embeddings[0])}")
#         elif isinstance(embeddings, np.ndarray):
#             print(f"Shape of embeddings array: {embeddings.shape}")
        
#         return embeddings
    
#     except FileNotFoundError:
#         logger.error(f"Cache file {cache_file} not found")
#         raise
#     except pickle.UnpicklingError:
#         logger.error(f"Error unpickling {cache_file}. The file may be corrupted.")
#         raise
#     except Exception as e:
#         logger.error(f"An unexpected error occurred while loading embeddings: {str(e)}")
#         raise

def faiss_search(query_embedding, index, k=5):
    query_embedding = np.array([query_embedding]).astype('float32')
    faiss.normalize_L2(query_embedding)
    distances, indices = index.search(query_embedding, k)
    return [(int(i), float(distances[0][idx])) for idx, i in enumerate(indices[0])]

def hybrid_search(query, corpus, corpus_embeddings, faiss_index, bm25, k=5):
    logger.info(f"Performing hybrid search for query: {query[:50]}...")
    start_time = time.time()

    # Dense retrieval
    query_embedding = get_embedding(query)
    dense_matches = faiss_search(query_embedding, faiss_index, k=k*2)
    
    # Sparse retrieval (BM25)
    sparse_matches = [(idx, score) for idx, score in enumerate(bm25.get_scores(query.split()))]
    sparse_matches = sorted(sparse_matches, key=lambda x: x[1], reverse=True)[:k*2]
    
    # Combine and re-rank results
    combined_matches = set(dense_matches + sparse_matches)
    reranked_matches = sorted(combined_matches, key=lambda x: x[1], reverse=True)[:k]
    
    logger.info(f"Hybrid search completed in {time.time() - start_time:.2f} seconds. Found {len(reranked_matches)} matches.")
    return reranked_matches

def gpt_categorize(transaction, potential_matches, api_key):
    prompt = f"""
    Transaction to categorize:
    Name: {transaction.name}
    Memo: {transaction.memo}
    Amount: {transaction.amount}
    Type: {transaction.transaction_type}
    Source: {transaction.source}
    NOTE:
    1. THE CATEGORY CANNOT BE THE SAME AS THE SOURCE.
    Potential matches:
    {json.dumps(potential_matches, indent=2)}, NULL
    Respond in the following JSON format:
    {{
        "category": "category_name_or_null",
        "explanation": "brief explanation of your decision, including how you considered the source and amount direction"
    }}
    """
    response = client.chat.completions.create(
        model="gpt-4o-mini",
        response_format={"type": "json_object"},
        messages=[
            {"role": "system", "content": "You are an experienced accountant with 20+ years of experience, you will be provided with transactionsand their potential closing accounts that, Only reply with the most plausible category , reply with NULL if not sure"},
            {"role": "user", "content": prompt}
        ]
    )
    return json.loads(response.choices[0].message.content)

# Function to choose between TF-IDF and OpenAI embeddings for semantic search
def semantic_search(query, corpus, corpus_embeddings, faiss_index, bm25, use_hybrid=True, threshold=0.5):
    if use_hybrid:
        return hybrid_search(query, corpus, corpus_embeddings, faiss_index, bm25, k=5)
    else:
        # Fallback to TF-IDF if not using hybrid
        vectorizer = TfidfVectorizer().fit(corpus)
        query_vec = vectorizer.transform([query])
        corpus_vec = vectorizer.transform(corpus)
        similarities = cosine_similarity(query_vec, corpus_vec)[0]
        matches = [(i, sim) for i, sim in enumerate(similarities) if sim >= threshold]
        return sorted(matches, key=lambda x: x[1], reverse=True)