fix: use output for python faithfulness statements (#201)

## Summary
- Fix a bug in Python `Faithfulness` where statements were extracted
from `expected` instead of `output`, causing the scorer to evaluate the
wrong text.
- Update both sync and async `Faithfulness` paths in
`py/autoevals/ragas.py` to route statement extraction through `output`.
- Add a regression test in `py/autoevals/test_ragas.py` that uses
mismatched `output`/`expected` and input-driven mocks so the final score
depends on correct routing.

## Bug
`Faithfulness` should score whether claims in the **generated answer**
are supported by context.
In Python, it incorrectly passed `expected` to statement extraction, so
claims were taken from ground truth rather than model output.

## Fix
- In `Faithfulness._run_eval_async(...)`: change `answer=expected` ->
`answer=output`.
- In `Faithfulness._run_eval_sync(...)`: change `answer=expected` ->
`answer=output`.
- Align sync required-field validation with async by requiring `output`
as well.

## Test
Added `test_faithfulness_extracts_statements_from_output`:
- Uses different `output` and `expected`.
- Mocks `extract_statements` to derive statements from passed `answer`.
- Mocks `extract_faithfulness` to derive verdicts from context
containment.
- Ensures score behavior reflects correct routing (would fail under old
bug).

## Validation
- `uv run --extra dev --extra scipy pytest py/autoevals/test_ragas.py -k
faithfulness_extracts_statements_from_output` passed.
- Pre-commit hooks pass on commit.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: invocan-jsathe

py/autoevals/ragas.py

@@ -983,7 +983,7 @@ async def _run_eval_async(self, output, expected=None, input=None, context=None,
 
         statements = (
             await aextract_statements(
-                client=self.client, question=input, answer=expected, model=self.model, **self.extra_args
+                client=self.client, question=input, answer=output, model=self.model, **self.extra_args
             )
         )["statements"]
 
@@ -1003,12 +1003,10 @@ async def _run_eval_async(self, output, expected=None, input=None, context=None,
         )
 
     def _run_eval_sync(self, output, expected=None, input=None, context=None, **kwargs):
-        check_required("Faithfulness", input=input, context=context)
+        check_required("Faithfulness", input=input, output=output, context=context)
 
         statements = (
-            extract_statements(
-                client=self.client, question=input, answer=expected, model=self.model, **self.extra_args
-            )
+            extract_statements(client=self.client, question=input, answer=output, model=self.model, **self.extra_args)
         )["statements"]
 
         faithfulness = (

py/autoevals/test_ragas.py

@@ -6,6 +6,7 @@
 from httpx import Response
 from openai import OpenAI
 
+import autoevals.ragas as ragas_module
 from autoevals import init
 from autoevals.ragas import *
 
@@ -125,6 +126,51 @@ def test_context_relevancy_score_normal_case():
     assert result.score >= 0.0
 
 
+def test_faithfulness_extracts_statements_from_output(monkeypatch):
+    """Regression test for Faithfulness answer routing.
+
+    This verifies that Faithfulness extracts statements from ``output`` (the model
+    answer being evaluated), not from ``expected`` (ground truth). The test uses
+    mismatched ``output``/``expected`` values and mocked helpers that derive
+    statements/verdicts from their inputs so the final score depends on which
+    field was routed into statement extraction.
+    """
+    captured_answer = None
+
+    def fake_extract_statements(question, answer, client=None, **extra_args):
+        nonlocal captured_answer
+        captured_answer = answer
+        statement = answer.strip().rstrip(".")
+        return {"statements": [statement]}
+
+    def fake_extract_faithfulness(context, statements, client=None, **extra_args):
+        faithfulness = []
+        for statement in statements:
+            verdict = int(statement in context)
+            faithfulness.append(
+                {
+                    "statement": statement,
+                    "verdict": verdict,
+                    "reason": "Supported by context" if verdict else "Not found in context",
+                }
+            )
+        return {"faithfulness": faithfulness}
+
+    monkeypatch.setattr(ragas_module, "extract_statements", fake_extract_statements)
+    monkeypatch.setattr(ragas_module, "extract_faithfulness", fake_extract_faithfulness)
+
+    scorer = Faithfulness()
+    score = scorer.eval(
+        input="What is the capital of France?",
+        output="Paris is the capital of France.",
+        expected="Lyon is the capital of France.",
+        context="Paris is the capital of France.",
+    )
+
+    assert score.score == 1
+    assert captured_answer == "Paris is the capital of France."
+
+
 @respx.mock
 def test_answer_correctness_uses_custom_embedding_model():
     """Test that AnswerCorrectness passes embedding_model parameter through to embeddings API."""