Lighteval TriviaQA evaluation not giving results as per smolLM-360M reported numbers

[Kun-AI-san]

Hi I have been trying to evaluation smollm-360M on triviaQA to reproduce the reported results, But I am just not able to.

import re
import string

import numpy as np

from lighteval.tasks.lighteval_task import LightevalTaskConfig
from lighteval.tasks.requests import Doc, SamplingMethod
from lighteval.metrics.metrics import Metrics
from lighteval.metrics.metrics_sample import LoglikelihoodAcc, ExactMatches
from lighteval.metrics.normalizations import (
    LogProbCharNorm,
    harness_triviaqa_normalizer,
    gsm8k_normalizer,
)
from lighteval.metrics.utils.metric_utils import SampleLevelMetric
from math_utils import parse_math_answer


# ---------------------------------------------------------------------------
# Metrics
# ---------------------------------------------------------------------------
# lighteval 0.13.0 dropped the `Metrics.loglikelihood_acc_norm_nospace`,
# `Metrics.quasi_exact_match_triviaqa` and `Metrics.quasi_exact_match_gsm8k`
# shortcuts (along with `MetricCategory` / `MetricUseCase` and the
# `SampleLevelMetric(..., category=..., use_case=...)` signature). They are
# rebuilt here from the same underlying primitives, which are still
# available, using the new `category=SamplingMethod...` signature and
# passing the sample-level callable object directly (no more `.compute`).

loglikelihood_acc_norm_nospace = SampleLevelMetric(
    metric_name="acc_norm",
    sample_level_fn=LoglikelihoodAcc(logprob_normalization=LogProbCharNorm(ignore_first_space=True)),
    category=SamplingMethod.LOGPROBS,
    corpus_level_fn=np.mean,
    higher_is_better=True,
)

quasi_exact_match_triviaqa = SampleLevelMetric(
    metric_name="qem",
    sample_level_fn=ExactMatches(strip_strings=True, normalize_pred=harness_triviaqa_normalizer),
    category=SamplingMethod.GENERATIVE,
    corpus_level_fn=np.mean,
    higher_is_better=True,
)

quasi_exact_match_gsm8k = SampleLevelMetric(
    metric_name="qem",
    sample_level_fn=ExactMatches(
        strip_strings=True, normalize_pred=gsm8k_normalizer, normalize_gold=gsm8k_normalizer
    ),
    category=SamplingMethod.GENERATIVE,
    corpus_level_fn=np.mean,
    higher_is_better=True,
)

quasi_exact_match_math = SampleLevelMetric(
    metric_name="qem",
    sample_level_fn=ExactMatches(
        strip_strings=True,
        normalize_pred=lambda text: parse_math_answer(text, "math"),
        normalize_gold=lambda text: parse_math_answer(text, "math"),
    ),
    category=SamplingMethod.GENERATIVE,
    corpus_level_fn=np.mean,
    higher_is_better=True,
)


# ---------------------------------------------------------------------------
# Prompt functions
# ---------------------------------------------------------------------------
# `lighteval.tasks.default_prompts` no longer exists in 0.13.0 (prompt
# functions now live next to each built-in task under
# `lighteval/tasks/tasks/<name>.py`). The functions below reproduce the
# behaviour of the old `prompt.arc`, `prompt.openbookqa`, `prompt.winogrande`,
# `prompt.piqa_harness`, `prompt.triviaqa` and `prompt.gsm8k` helpers that
# this file used to import.

def arc_prompt(line, task_name: str = None):
    return Doc(
        task_name=task_name,
        query=f"Question: {line['question']}\nAnswer:",
        choices=[f" {c}" for c in line["choices"]["text"]],
        gold_index=line["choices"]["label"].index(line["answerKey"]),
    )


def openbookqa_prompt(line, task_name: str = None):
    return Doc(
        task_name=task_name,
        query=f"{line['question_stem']}",
        choices=[f" {c}" for c in line["choices"]["text"]],
        gold_index=["A", "B", "C", "D", "E"].index(line["answerKey"].strip()),
    )


def winogrande_prompt(line, task_name: str = None):
    query, end_of_target = line["sentence"].split("_")
    end_of_target = end_of_target.strip()
    return Doc(
        task_name=task_name,
        query=query,
        choices=[f"{line['option1']} {end_of_target}", f"{line['option2']} {end_of_target}"],
        gold_index=int(line["answer"]) - 1 if line["answer"] != "" else -1,  # managing unk test index
    )


def piqa_harness_prompt(line, task_name: str = None):
    return Doc(
        task_name=task_name,
        query=f"Question: {line['goal']}\nAnswer:",
        choices=[f" {line['sol1']}", f" {line['sol2']}"],
        gold_index=int(line["label"]),
    )


def triviaqa_prompt(line, task_name: str = None):
    def _remove_prefixes(aliases):
        # Optimization: Remove any alias that has a strict prefix elsewhere in the list
        aliases.sort()
        ret = [aliases[0]]
        for alias in aliases[1:]:
            if not alias.startswith(ret[-1]):
                ret.append(alias)
        return ret

    list_of_candidates = [
        alias.lower().translate(str.maketrans("", "", string.punctuation))
        for alias in _remove_prefixes(line["answer"]["aliases"])
    ]

    return Doc(
        task_name=task_name,
        query=f"Question: {line['question']}\nAnswer:",
        gold_index=0,
        choices=[list_of_candidates],
    )


def gsm8k_prompt(line, task_name: str = None):
    # Has special analysis in metric for number decomposition
    return Doc(
        task_name=task_name,
        query=f"Question: {line['question']}\nAnswer:",
        choices=[f" {line['answer']}"],
        gold_index=0,
    )


def prompt_hellaswag(line, task_name: str = None):
    def preprocess(text):
        """Comes from AiHarness"""
        # text = text.strip()
        # NOTE: Brackets are artifacts of the WikiHow dataset portion of HellaSwag.
        text = text.replace(" [title]", ". ")
        text = re.sub("\\[.*?\\]", "", text)
        text = text.replace("  ", " ")
        return text

    ctx = f"{line['ctx_a']} {line['ctx_b'].capitalize()} "
    return Doc(
        task_name=task_name,
        query=preprocess(line["activity_label"] + ": " + ctx),
        choices=[" " + preprocess(ending) for ending in line["endings"]],
        gold_index=int(line["label"]) if line["label"] != "" else -1,  # -1 for test
    )

def prompt_commonsense_qa(line, task_name: str = None):
    return Doc(
        task_name=task_name,
        query=line["question"],
        choices=[f" {c}" for c in line["choices"]["text"]],
        gold_index=line["choices"]["label"].index(line["answerKey"].strip()),
        instruction="",
    )

def mmlu_pro_mc_prompt(line, task_name: str = None):
    options = line["options"]
    letters = [chr(ord("A") + i) for i in range(len(options))]
    topic = line["category"].replace('_', ' ')
    query = f"The following are multiple choice questions (with answers) about {topic}.\n\n"
    query += line["question"] + "\n"
    query += "".join([f"{letter}. {choice}\n" for letter, choice in zip(letters, options)])
    query += "Answer:"

    return Doc(
        task_name=task_name,
        query=query,
        choices=letters,
        gold_index=line["answer_index"],
        instruction=f"The following are multiple choice questions (with answers) about {topic}.\n\n",
    )

def mmlu_cloze_prompt(line, task_name: str = None):
    """MMLU prompt without choices"""
    topic = line["subject"]
    prompt = f"The following are questions about {topic.replace('_', ' ')}.\nQuestion: "
    prompt += line["question"] + "\nAnswer:"

    return Doc(
        task_name=task_name,
        query=prompt,
        choices=[f" {c}" for c in line["choices"]],
        gold_index=int(line["answer"]),
        instruction=f"The following are questions about {topic.replace('_', ' ')}.\n",
    )

def bbh_prompt(line, task_name: str = None):
    return Doc(
        task_name=task_name,
        query="Question: " + line["input"] + "\nAnswer: ",
        choices=[line["target"]],
        gold_index=0,
    )

def prompt_math(line, task_name: str = None):
    return Doc(
        task_name=task_name,
        query=f"{line['problem']}\nPlease reason step by step, and put your final answer within \\boxed{{}}.\n\n",
        gold_index=0,
        choices=[f"{line['solution']}\n\n"],
    )


TASKS_TABLE = [
    LightevalTaskConfig(
        name="custom_arc:easy",
        prompt_function=arc_prompt,
        hf_repo="ai2_arc",
        hf_revision="210d026faf9955653af8916fad021475a3f00453",
        hf_subset="ARC-Easy",
        evaluation_splits=["test"],
        metrics=[loglikelihood_acc_norm_nospace],
    ),
    LightevalTaskConfig(
        name="custom_arc:challenge",
        prompt_function=arc_prompt,
        hf_repo="ai2_arc",
        hf_revision="210d026faf9955653af8916fad021475a3f00453",
        hf_subset="ARC-Challenge",
        evaluation_splits=["test"],
        metrics=[loglikelihood_acc_norm_nospace],
    ),
    LightevalTaskConfig(
        name="openbook_qa",
        prompt_function=openbookqa_prompt,
        hf_repo="allenai/openbookqa",
        hf_subset="main",
        hf_revision="388097ea7776314e93a529163e0fea805b8a6454",
        metrics=[loglikelihood_acc_norm_nospace],
    ),
    LightevalTaskConfig(
        name="custom_hellaswag",
        prompt_function=prompt_hellaswag,
        hf_repo="Rowan/hellaswag",
        hf_subset="default",
        hf_revision="6002345709e0801764318f06bf06ce1e7d1a1fe3",
        metrics=[loglikelihood_acc_norm_nospace],
    ),
    LightevalTaskConfig(
        name="commonsense_qa",
        prompt_function=prompt_commonsense_qa,
        hf_repo="tau/commonsense_qa",
        hf_subset="default",
        hf_revision="94630fe30dad47192a8546eb75f094926d47e155",
        metrics=[loglikelihood_acc_norm_nospace],
    ),
    LightevalTaskConfig(
        name="custom_winogrande",
        prompt_function=winogrande_prompt,
        hf_repo="allenai/winogrande",
        hf_subset="winogrande_xl",
        hf_revision="85ac5b5a3b7a930e22d590176e39460400d19e41",
        metrics=[loglikelihood_acc_norm_nospace],
    ),
    LightevalTaskConfig(
        name="custom_piqa",
        prompt_function=piqa_harness_prompt,
        hf_repo="ybisk/piqa",
        hf_subset="plain_text",
        hf_revision="2e8ac2dffd59bac8c3c6714948f4c551a0848bb0",
        metrics=[loglikelihood_acc_norm_nospace],
    ),
    LightevalTaskConfig(
        name="trivia_qa",
        prompt_function=triviaqa_prompt,
        hf_repo="mandarjoshi/trivia_qa",
        hf_subset="rc.nocontext",
        hf_revision="0f7faf33a3908546c6fd5b73a660e0f8ff173c2f",
        hf_avail_splits=["train", "validation"],
        evaluation_splits=["validation"],
        metrics=[quasi_exact_match_triviaqa],
        generation_size=20,
        stop_sequence=["Question:", "Question"],
        few_shots_select="random_sampling_from_train",
    ),
    LightevalTaskConfig(
        name="custom_mmlu_pro",
        prompt_function=mmlu_pro_mc_prompt,
        hf_repo="TIGER-Lab/MMLU-Pro",
        hf_subset="default",
        hf_revision="3373e0b32277875b8db2aa555a333b78a08477ea",
        metrics=[loglikelihood_acc_norm_nospace],
        evaluation_splits=["test"],
        few_shots_split="validation",
    ),
    LightevalTaskConfig(
        name="custom_gsm8k",
        prompt_function=gsm8k_prompt,
        hf_repo="openai/gsm8k",
        hf_subset="main",
        hf_revision="e53f048856ff4f594e959d75785d2c2d37b678ee",
        hf_avail_splits=["train", "test"],
        evaluation_splits=["test"],
        metrics=[quasi_exact_match_gsm8k],
        generation_size=256,
        stop_sequence=["Question:", "Question"],
        few_shots_select="random_sampling_from_train",
    ),
    LightevalTaskConfig(
        name="mmlu_stem",
        prompt_function=mmlu_cloze_prompt,
        hf_repo="TIGER-Lab/MMLU-STEM",
        hf_subset="default",
        hf_revision="78a4b40757f31688d00426d1372dbbc6070d33a8",
        hf_avail_splits=["test"],
        evaluation_splits=["test"],
        metrics=[loglikelihood_acc_norm_nospace],
        generation_size=-1,
    ),
    LightevalTaskConfig(
        name="mmlu",
        prompt_function=mmlu_cloze_prompt,
        hf_repo="cais/mmlu",
        hf_subset="all",
        hf_revision="c30699e8356da336a370243923dbaf21066bb9fe",
        hf_avail_splits=["test"],
        evaluation_splits=["test"],
        metrics=[loglikelihood_acc_norm_nospace],
        generation_size=-1,
    ),
]

BBH_TASKS = [
    LightevalTaskConfig(
        name=f"bbh:{subset}",
        prompt_function=bbh_prompt,
        hf_repo="lighteval/big_bench_hard",
        hf_subset=subset,
        hf_revision="80610173426f05e6f1448f047e2db4840a7dd899",
        metrics=[Metrics.exact_match],
        hf_avail_splits=["train"],
        # this is the only split available, obviously not used in training
        evaluation_splits=["train"],
        few_shots_split="train",
        stop_sequence=["Question:", "Question"],
    )
    for subset in [
        "boolean_expressions",
        "causal_judgement",
        "date_understanding",
        "disambiguation_qa",
        "dyck_languages",
        "formal_fallacies",
        "geometric_shapes",
        "hyperbaton",
        "logical_deduction_five_objects",
        "logical_deduction_seven_objects",
        "logical_deduction_three_objects",
        "movie_recommendation",
        "multistep_arithmetic_two",
        "navigate",
        "object_counting",
        "penguins_in_a_table",
        "reasoning_about_colored_objects",
        "ruin_names",
        "salient_translation_error_detection",
        "snarks",
        "sports_understanding",
        "temporal_sequences",
        "tracking_shuffled_objects_five_objects",
        "tracking_shuffled_objects_seven_objects",
        "tracking_shuffled_objects_three_objects",
        "web_of_lies",
        "word_sorting",
    ]
]

TASKS_TABLE.extend(BBH_TASKS)

MATH_TASKS = [
    LightevalTaskConfig(
        name="math",
        prompt_function=prompt_math,
        hf_repo="HuggingFaceTB/math_tasks",
        hf_subset="math",
        hf_revision="3d34f1076f279000b9315583dcdacfd288898283",
        hf_avail_splits=["train", "test", "demo"],
        evaluation_splits=["test"],
        metrics=[quasi_exact_match_math],
        generation_size=1024,
        stop_sequence=["\n\n"],
        few_shots_split="demo",
        few_shots_select="sequential",
    )
]

TASKS_TABLE.extend(MATH_TASKS)

## MMLU ##
class CustomMMLUEvaluationTask(LightevalTaskConfig):
    def __init__(
        self,
        name,
        prompt_function=None,
        hf_repo="lighteval/mmlu",
        hf_subset=None,
        metrics=[Metrics.loglikelihood_acc, loglikelihood_acc_norm_nospace],
        hf_avail_splits=None,
        evaluation_splits=["test"],
        few_shots_split="dev",
        few_shots_select=None,
        generation_size=-1,
        stop_sequence=None,
        output_regex=None,
        frozen=False,
    ):
        super().__init__(
            name=name,
            prompt_function=prompt_function,
            hf_repo=hf_repo,
            hf_subset=hf_subset,
            metrics=metrics,
            hf_avail_splits=hf_avail_splits,
            evaluation_splits=evaluation_splits,
            few_shots_split=few_shots_split,
            few_shots_select=few_shots_select,
            generation_size=generation_size,
            stop_sequence=stop_sequence,
        )


if __name__ == "__main__":
    print([t.name for t in TASKS_TABLE])
    print(len(TASKS_TABLE))

This is the code I have been using with lighteval accelerate.
The result I am getting is as follows:

Its farcry from the reported score of 9.1.

Any help would be welcome.

[seva9523]

Hi — this looks like a case where separating input or prompt drift from metric and model differences could narrow the investigation.

I maintain [EvalRepro](https://github.com/seva9523/EvalRepro), an Apache-2.0 tool and GitHub Action for comparing materialized AI-evaluation inputs and task contracts. I can volunteer a bounded, fork-side TriviaQA pilot using the revision and split already shown here: materialize the baseline and current Lighteval task documents, then compare sample membership and order, query text, choices, gold targets, and generation settings through privacy-conscious hash manifests.

This would not assume that the score gap is caused by dataset drift. The purpose is to rule input-contract changes in or out before investigating metrics, model revisions, or runtime behaviour. The comparison method and its limitations are illustrated in [this 22,773-record case study](seva9523/EvalRepro#27).

If useful, could you confirm the Lighteval tag or commit and model revision associated with the reported 9.1 baseline? I can publish the exact commands and manifest comparison back on this issue, keeping the result fork-validated unless maintainers review it.