naam.py

import spacy
from transformers import pipeline
from openai import OpenAI
import warnings

# Ignore warnings
warnings.filterwarnings("ignore")

# Initialize clients
openai_client = OpenAI(api_key="sk-proj-631uHpaVGU6vTjZeC9tCT3BlbkFJI7BVzFMOHFazRoW1Q7y0")

# Load spaCy model
nlp = spacy.load("en_core_web_trf")

# Load Hugging Face NER pipeline
ner_pipeline = pipeline("ner", model="dslim/bert-base-NER")

def preprocess_text(text):
    return ' '.join(str(text).split())

def detect_names_spacy(text):
    doc = nlp(text)
    return [ent.text for ent in doc.ents if ent.label_ == "PERSON"]

def detect_names_bert(text):
    ner_results = ner_pipeline(text)
    names = []
    current_name = ""
    for entity in ner_results:
        if entity['entity'] in ['B-PER', 'I-PER']:
            word = entity['word']
            if word.startswith("##"):
                current_name += word[2:]
            else:
                if current_name:
                    names.append(current_name.strip())
                current_name = word
        else:
            if current_name:
                names.append(current_name.strip())
                current_name = ""
    if current_name:
        names.append(current_name.strip())
    return names

def detect_names_llm(text):
    prompt = f"""
    Analyze the following text and extract any PERSON/HUMAN names mentioned, including names from various cultures:
    "{text}"
    Provide the names in a comma-separated list, or respond with "No names found" if no names are present.
    """
    try:
        completion = openai_client.chat.completions.create(
            model="gpt-4o-mini",
            messages=[
                {"role": "system", "content": "You are a helpful assistant that extracts person names from text, including names from diverse cultural backgrounds."},
                {"role": "user", "content": prompt}
            ]
        )
        names = completion.choices[0].message.content.strip()
        if names.lower() == "no names found":
            return []
        return [name.strip() for name in names.split(',')]
    except Exception as e:
        print(f"Error in OpenAI API: {str(e)}")
        return []

def is_valid_name(name, text):
    if len(name.split()) < 2:
        return False
    if name.lower() in {'mr', 'mrs', 'ms', 'dr', 'prof'}:
        return False
    return True

def detect_names(text):
    preprocessed_text = preprocess_text(text)
    
    spacy_names = detect_names_spacy(preprocessed_text)
    bert_names = detect_names_bert(preprocessed_text)
    llm_names = detect_names_llm(preprocessed_text)
    
    all_names = set(spacy_names + bert_names + llm_names)
    
    validated_names = []
    for name in all_names:
        if is_valid_name(name, preprocessed_text):
            count = (name in spacy_names) + (name in bert_names) + (name in llm_names)
            if count >= 2:
                validated_names.append(name)
    
    return len(validated_names) > 0

def filter_transactions_without_names(transactions):
    """
    Filter out transactions that contain human names in their memo field.
    
    :param transactions: List of Transaction objects
    :return: Tuple of (transactions_without_names, transactions_with_names)
    """
    transactions_without_names = []
    transactions_with_names = []
    
    for transaction in transactions:
        if detect_names(transaction.memo):
            transactions_with_names.append(transaction)
        else:
            transactions_without_names.append(transaction)
    
    return transactions_without_names, transactions_with_names