fix(agent): finish the live query in evidence gather instead of stopping at discovery (#4976)

Author: YauhenBichel

.github/workflows/release.yml

@@ -372,6 +372,31 @@ jobs:
             exit 1
           fi
 
+      - name: Re-sign macOS onedir after post-build mutations
+        if: runner.os == 'macOS' && matrix.pyinstaller_mode == 'onedir'
+        shell: bash
+        run: |
+          set -euo pipefail
+          # Replacing/rewriting bundled dylibs (libssl align) invalidates the
+          # adhoc signature PyInstaller applied. Consumer Macs then SIGKILL the
+          # binary on first exec (exit 137, CODESIGNING / Invalid Page) even when
+          # CI runners still let --version through. Sign nested libs first, then
+          # the main binary — ``codesign --deep`` on the directory fails with
+          # "invalid resource directory" for this PyInstaller layout.
+          APP_DIR="./dist/opensre"
+          APP_BIN="${APP_DIR}/opensre"
+          if [ ! -f "$APP_BIN" ]; then
+            echo "No onedir binary at ${APP_BIN}; skip re-sign"
+            exit 0
+          fi
+          find "$APP_DIR" -type f \( -name '*.dylib' -o -name '*.so' -o -name '*.so.*' \) -print0 \
+            | while IFS= read -r -d '' lib; do
+                codesign --force --sign - "$lib"
+              done
+          codesign --force --sign - "$APP_BIN"
+          codesign --verify --strict "$APP_BIN"
+          echo "Re-signed onedir libs + ${APP_BIN} (adhoc) after post-build mutations"
+
       - name: Smoke test binary (Unix)
         if: runner.os != 'Windows'
         shell: bash

AGENTS.md

@@ -311,6 +311,11 @@ Steps:
   when appending to an existing file: **use the import style the file already
   established** (an existing `import core.context_budget as budget` means new
   code calls `budget.name`, not `from core.context_budget import name`).
+- Except block handles `BaseException` (code-quality): catch `Exception`, not
+  `BaseException`. Collecting request/transport failures in a test thread still
+  works — `requests.exceptions.ConnectTimeout` subclasses `Exception`. Catching
+  `BaseException` also swallows `KeyboardInterrupt` / `SystemExit`. Do not keep
+  `noqa: BLE001` to silence it.
 - Unused global variable (CodeQL / code-quality "Unused global variable"):
   CodeQL often **does not credit cross-module imports** as a use of a module-
   level constant. A `FOO = "..."` in `text.py` that is only read via

core/agent_harness/AGENTS.md

@@ -63,8 +63,9 @@ After an L1 gather, `reclassify_evidence_need_after_gather` may flip to
 Prefer `GatheredEvidence.tool_results` from the gather loop; fall back to
 `split`/`partition` on the rendered `Tool:`/`Result:` observation (see
 `gather_observation.iter_tool_result_blocks`) — never regex or phrase lists.
-HogQL / empty-result failures and ordinary values that mention auth words stay
-L1 (no CTA).
+HogQL / empty-result failures stay L1 (no L0 CTA). A metric gather that never
+ran a live query still gets a draft HogQL/PromQL block and one
+`/integrations setup …` line, then stops (`turns/metric_query_floor.py`).
 
 **No keyword intent routing around the action agent.** Do not scan user text
 with regex/keywords to skip gather, attach goals, or bypass `execute_actions`.

core/agent_harness/prompts/assistant/observation.py

@@ -9,6 +9,7 @@
 _HANDOFF_GUIDANCE_PREFIXES: tuple[str, ...] = (
     "evidence_tier:L0_degraded:config",
     "evidence_tier:L0_degraded",
+    "evidence_tier:metric_unformed",
     "session_goal:",
     "database_query:",
     "incident_description:",

core/agent_harness/prompts/assistant/text.py

@@ -96,6 +96,20 @@
         "and do not open an onboarding wizard unprompted. A bare user yes after "
         "that CTA will run the connect slash; do not invent a second Want-me-to.\n\n"
     ),
+    "evidence_tier:metric_unformed": (
+        "Gather ran but did not execute a live metric query (schema/list probes "
+        "only, unknown event, or the query could not be formed). Do not invent "
+        "a count.\n"
+        "Structure the reply like this:\n"
+        "1. One plain sentence: the live query could not be formed and why.\n"
+        "2. A short draft query in a fenced code block, labeled as a draft — "
+        "HogQL for PostHog / product analytics, PromQL for Grafana / metrics. "
+        "Never invent metric numbers as fact.\n"
+        "3. Exactly one setup line using a valid `/integrations setup <id>` "
+        "(preferred source id from this session). Do not invent a vendor.\n"
+        "Do NOT offer a full incident investigation. Do NOT close with "
+        "**Want me to:**. Stop after this reply.\n\n"
+    ),
     # Connected preferred source failed auth/config after gather.
     # Prefix: ``evidence_tier:L0_degraded:config:<ids>`` (matched before plain L0).
     "evidence_tier:L0_degraded:config": (

core/agent_harness/prompts/assistant/turn.py

@@ -103,9 +103,12 @@ def _build_integration_guard(ctx: TurnSnapshot) -> str:
     already reaches the gather prompt; the answer path was told only when the
     set was empty, so a reply could not say what it had looked at.
 
-    Also pins valid ``/integrations setup`` ids and preferred product-analytics
-    coverage so the model cannot invent vendors (parity S5: Mixpanel CTA while
-    PostHog was already ready).
+    When something is connected, the valid ``/integrations setup`` ids and
+    preferred product-analytics coverage are named too, so a reply cannot
+    recommend an invented vendor over a ready one (parity S5: Mixpanel CTA
+    while PostHog was already connected). An empty session gets neither: the
+    id roster is several hundred characters of vendor names, and on a session
+    with nothing connected it crowds out the user's actual question.
     """
     from platform.harness_ports import (
         preferred_evidence_sources_for,
@@ -132,7 +135,7 @@ def _build_integration_guard(ctx: TurnSnapshot) -> str:
             "integrations, answer with guidance only."
         )
 
-    setupable = tuple(setupable_integration_services())
+    setupable = tuple(setupable_integration_services()) if connected else ()
     if setupable:
         parts.append(
             "Only these service ids are valid in `/integrations setup <id>`: "

core/agent_harness/session_goal/evaluate.py

@@ -21,7 +21,9 @@
   gather) and the reply is non-empty → achieve (same-turn answer). Waiting for
   a scrubbed/forgotten tag forced a redundant outer turn that repeated the
   live answer (parity S1 R7). Gather successes count: metric handoffs often
-  leave action ``executed_success_count`` at 0.
+  leave action ``executed_success_count`` at 0. Final-route identity alone
+  (``cli_agent_fallback`` / summarize) is not evidence — unsupported fallbacks
+  must not close the goal.
 * ``achieved`` with no checklist on a handoff goal → require tool evidence, or
   stay active.
 * Hosts may wrap :func:`evaluate_session_goal` with an LLM confirm for the
@@ -255,6 +257,8 @@ def evaluate_session_goal(
             # wait for session_goal:achieved — that tag is scrubbed from the
             # user-visible reply and models often omit it, which previously
             # forced a second outer turn that repeated the same metric answer.
+            # Require real action/gather successes — route identity alone would
+            # let unsupported fallbacks close the goal without evidence.
             verdict = SessionGoalVerdict(
                 status=SessionGoalStatus.ACHIEVED,
                 reason=SessionGoalReason.ACHIEVED_TOOL_EVIDENCE,

core/agent_harness/session_goal/run_until.py

@@ -245,6 +245,7 @@ def run_until_session_goal(
 
     if active.turns_used == 0:
         active = active.record_turn()
+        attach_session_goal(session, active)
 
     active, last, stop = _finish_outer_turn(
         session,
@@ -272,6 +273,7 @@ def run_until_session_goal(
         _announce_working(session, active, on_progress)
         last = chat(continuation_nudge(active))
         active = active.record_turn()
+        attach_session_goal(session, active)
         active, last, stop = _finish_outer_turn(
             session,
             active,

core/agent_harness/turns/answer_finalize.py

@@ -35,6 +35,7 @@
     is_non_investigation_handoff,
     is_prior_investigation_follow_up_handoff,
 )
+from core.agent_harness.turns.metric_query_floor import apply_unformed_metric_floor
 from core.agent_harness.turns.turn_route import RouteIntent
 from platform.harness_ports import integration_setup_command
 
@@ -148,6 +149,14 @@ def finalize_routed_answer(
     text = response_text
     if should_suppress_investigation_offer(evidence_need) or should_skip_gather(evidence_need):
         text = append_upgrade_cta(session, text, evidence_need)
+    # Metric gather that never ran a live query still gets a draft HogQL/PromQL
+    # block and one setup slash (parity S2). No-op when a query already ran.
+    text = apply_unformed_metric_floor(
+        text,
+        evidence_need,
+        observation=evidence_for_offer,
+        setup_command_for=integration_setup_command,
+    )
     # Bookkeeping only — never feed this into route selection.
     text_changed_after_streaming = text != streamed_text
 

core/agent_harness/turns/evidence_need.py

@@ -342,8 +342,10 @@ def reclassify_evidence_need_after_gather(
     (``available: False`` + ``error``). Prefer ``GatheredEvidence.tool_results``
     (or the ``tool_results`` kwarg); fall back to ``split``/``partition`` on the
     observation string — never phrase lists over prose.
-    HogQL / empty-result failures stay L1 (honest answer, no UpgradeCTA).
-    Missing-source L0 is decided before gather and is left unchanged.
+    HogQL / empty-result failures stay L1 (honest answer). A metric gather that
+    never executed a live query still gets a draft HogQL/PromQL block and one
+    setup slash without flipping this tier. Missing-source L0 is decided before
+    gather and is left unchanged.
     """
     if need.tier != EvidenceTier.L1 or not need.required_for_authoritative or not need.connected:
         return need