AI chat: Improved algorithm for source references

The new algorithm also finds the reference in the original document if the AI cites text with gaps inside.
It is based on a pattern matching algorithm normally used in biology to find partial overlaps in DNA strings. Therefore, the "Bio" package has been added as a new requirement (instead of "fuzzysearch", which was used before).

Author: kaixxx

requirements.txt

@@ -21,7 +21,7 @@ langchain-openai>=0.1.16
 langchain-text-splitters>=0.2.2 
 faiss-cpu
 sentence-transformers 
-fuzzysearch 
+Bio
 PyYAML 
 json_repair
 pyinstaller<=6.4.0 

src/qualcoder/ai_chat.py

@@ -40,14 +40,13 @@
 import sqlite3
 import webbrowser
 import re
-import fuzzysearch
 
 from .ai_search_dialog import DialogAiSearch
 from .GUI.ui_ai_chat import Ui_Dialog_ai_chat
 from .helpers import Message
 from .confirm_delete import DialogConfirmDelete
 from .ai_prompts import PromptItem
-from .ai_llm import extract_ai_memo
+from .ai_llm import extract_ai_memo, ai_quote_search
 from .error_dlg import qt_exception_hook
 from .html_parser import html_to_text
 
@@ -691,8 +690,8 @@ def new_text_chat(self, doc_id, doc_name, text, start_pos, prompt: PromptItem):
             f'Be sure to include references to the original data, using this format '
             'definition: `[REF: "{The text from the original data that you want to reference. '
             'I have to match this against the original, so it is very important that you don\'t '
-            'change the quoted text in any way. Do not translate or correct errors, do not '
-            'leave parts of the text out. Create a new reference for every single quote.}"]`. \n'
+            'change the quoted text in any way. Do not translate or correct errors. Create a '
+            'new reference for every single quote.}"]`. \n'
             'These references are invisible text. If you want a direct quote to be '
             'visible to the user, include it in the normal text and add an additional reference '
             'in the above format.\n'
@@ -856,6 +855,7 @@ def update_chat_window(self, scroll_to_bottom=True):
             finally:
                 if scroll_to_bottom:
                     self.ai_output_scroll_to_bottom()
+                    self.ui.plainTextEdit_question.setFocus()
                 else:
                     self.ui.scrollArea_ai_output.verticalScrollBar().setValue(0)
                 self.is_updating_chat_window = False
@@ -869,21 +869,21 @@ def replace_references(self, text, streaming=False):
             # we are not in text analysis chat
             return text
 
-        pattern = r'\[REF: ([\"\'“”„‘’«»])(.+?)([\"\'“”„‘’«»])\]'      
+        pattern = r'\[REF: ([\"\'“”„‘’«»])(.+?)([\"\'“”„‘’«»])\]'  
+        fulltext = self.app.get_text_fulltext(self.ai_text_doc_id)    
 
         # Replacement function
         def replace_match(match):
             if streaming:
                 return f'({self.ai_text_doc_name})'
             quote = match.group(2)
-            # search quote with not more than 10% mismatch (Levenshtein Distance). This is done because the AI sometimes alters the text a little bit.
-            quote_found = fuzzysearch.find_near_matches(quote, self.ai_text_text, 
-                             max_l_dist=round(len(quote) * 0.1))  # result: list [Match(start=x, end=x, dist=x, matched='txt')]
-            if len(quote_found) > 0:
-                quote_start = quote_found[0].start + self.ai_text_start_pos
-                quote = quote_found[0].matched
-                fulltext = self.app.get_text_fulltext(self.ai_text_doc_id)
-                line_start, line_end = self.app.get_line_numbers(fulltext, quote_start, quote_start + len(quote))
+            
+            quote_start, quote_end = ai_quote_search(quote, self.ai_text_text)
+            if quote_start > -1 < quote_end:
+                quote = self.ai_text_text[quote_start:quote_end]
+                quote_start += self.ai_text_start_pos
+                quote_end += self.ai_text_start_pos
+                line_start, line_end = self.app.get_line_numbers(fulltext, quote_start, quote_end)
                 if line_start + line_end > 0:
                     if line_start == line_end:  # one line
                         a = f'(<a href="quote:{self.ai_text_doc_id}_{quote_start}_{len(quote)}">{self.ai_text_doc_name}: {line_start}</a>)'

src/qualcoder/ai_llm.py

@@ -42,10 +42,10 @@
 from .helpers import Message
 from .error_dlg import qt_exception_hook
 from .html_parser import html_to_text
-import fuzzysearch
 import json_repair
 import asyncio
 import configparser
+from Bio.Align import PairwiseAligner
 
 max_memo_length = 1500  # Maximum length of the memo send to the AI
 
@@ -236,6 +236,42 @@ def update_ai_models(current_models: list, current_model_index: int) -> tuple[li
                 current_models.append(model)
     return current_models, current_model_index
 
+def ai_quote_search(quote: str, original: str) -> tuple[int, int]:
+    """Searches the quote in the original text using the Smith-Waterman algorithm.
+    This also tolerates gaps up to complete sentences in the cited text or other 
+    minor differences in the exact wording.
+    The "PairwiseAligner" is normally used to find partial overlaps in DNA-strings.
+    Returns -1, -1 if no match is found.
+    """
+    
+    aligner = PairwiseAligner()
+    aligner.mode = 'local'
+    aligner.match_score = 2         # score for each matched char
+    aligner.mismatch_score = -1     # penalty for mismatched chars (errors)
+    aligner.open_gap_score = -0.5   # penalty for opening a gap (left out chars)
+    aligner.extend_gap_score = -0.1 # penalty for gap continuation
+
+    alignments = aligner.align(original.lower(), quote.lower())
+    if not len(alignments): # nothing found
+        return -1, -1
+
+    best = alignments[0]
+    orig_spans = best.aligned[0]
+    if not len(orig_spans):
+        return -1, -1
+
+    # combine all matched blocks from the first start to the last end:
+    start_idx = orig_spans[0][0]
+    end_idx = orig_spans[-1][1]
+        
+    # only accept a match if it reaches 80% of the max score -> prevents false positives
+    max_score = len(quote) * aligner.match_score
+    score_fraction = best.score / max_score
+    if score_fraction > 0.8:
+        return start_idx, end_idx
+    else:
+        return -1, -1
+
 class AiLLM():
     """ This manages the communication between qualcoder, the vectorstore 
     and the LLM (large language model, e.g. GPT-4)."""
@@ -813,19 +849,14 @@ def _search_analyze_chunk(self, chunk, code_name, code_memo, search_prompt: Prom
            'quote' in res_json and res_json['quote'] != '':  # found something
             # Adjust quote_start
             doc = {}
-            doc['metadata'] = chunk.metadata
-                       
-            # search quote with not more than 30% mismatch (Levenshtein Distance).
-            # This is done because the AI sometimes alters the text a little bit.
-            quote_found = fuzzysearch.find_near_matches(res_json['quote'], chunk.page_content, 
-                             max_l_dist=round(len(res_json['quote']) * 0.3))  # result: list [Match(start=x, end=x, dist=x, matched='txt')]
-            if len(quote_found) > 0:
-                doc['quote_start'] = quote_found[0].start + doc['metadata']['start_index']
-                doc['quote'] = quote_found[0].matched
+            doc['metadata'] = chunk.metadata          
+            quote_start, quote_end = ai_quote_search(res_json['quote'], chunk.page_content)
+            if quote_start > -1 < quote_end:
+                doc['quote_start'] = quote_start + doc['metadata']['start_index']
+                doc['quote'] = chunk.page_content[quote_start:quote_end]
             else:  # quote not found, make the whole chunk the quote
                 doc['quote_start'] = doc['metadata']['start_index']
-                doc['quote'] = chunk.page_content
-        
+                doc['quote'] = chunk.page_content        
             doc['interpretation'] = res_json['interpretation']
         else:  # No quote means the AI discarded this chunk as not relevant
             doc = None