Skip to content

Commit 459de16

Browse files
committed
fix: profile UX for first-time users — archetype descriptions, readable error hints, efficiency benchmarks
Three changes that improve what a stranger sees on first run: - Archetype names now include one-line descriptions (e.g., "short directives + corrections + affirmation") - Error hints truncate at word boundaries with "..." instead of mid-word - Efficiency metric includes context label (tight/typical/high rework) - 7 new profile UX regression tests (563 total)
1 parent b28269d commit 459de16

4 files changed

Lines changed: 179 additions & 5 deletions

File tree

pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
44

55
[project]
66
name = "agentsesh"
7-
version = "0.12.0"
7+
version = "0.12.1"
88
description = "Agent session intelligence — behavioral analysis, grading, remediation, replay, outcome testing, and prompt debugging for AI agent sessions"
99
readme = "README.md"
1010
license = {text = "MIT"}

sesh/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
"""sesh — Agent Session Intelligence CLI."""
22

3-
__version__ = "0.12.0"
3+
__version__ = "0.12.1"

sesh/analyzers/profile.py

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -256,8 +256,18 @@ def build_profile(
256256
else:
257257
pos_bias = "late"
258258

259-
# Most common hint
260-
hint_counter = Counter(e.hint[:40] for e in events)
259+
# Most common hint — truncate at word boundary
260+
def _truncate_hint(h: str, max_len: int = 60) -> str:
261+
h = h.replace("\n", " ").strip()
262+
if len(h) <= max_len:
263+
return h
264+
# Find last space before max_len
265+
cut = h[:max_len].rfind(" ")
266+
if cut < 20:
267+
cut = max_len # no good break point, hard cut
268+
return h[:cut] + "..."
269+
270+
hint_counter = Counter(_truncate_hint(e.hint) for e in events)
261271
common_hint = hint_counter.most_common(1)[0][0]
262272

263273
profile.stuck_patterns.append(StuckPattern(
@@ -424,7 +434,9 @@ def format_profile(profile: BehavioralProfile) -> str:
424434
lines.append("")
425435
lines.append("Efficiency")
426436
lines.append("\u2500" * 10)
427-
lines.append(f" Avg edits/commit: {profile.avg_edits_per_commit}")
437+
avg = profile.avg_edits_per_commit
438+
context = "tight" if avg <= 5 else "typical" if avg <= 15 else "high rework"
439+
lines.append(f" Avg edits/commit: {avg} ({context})")
428440
lines.append(f" Median edits/commit: {profile.median_edits_per_commit}")
429441

430442
# Stuck patterns
@@ -501,6 +513,13 @@ def format_profile(profile: BehavioralProfile) -> str:
501513

502514
# Archetype distribution
503515
if profile.archetype_distribution:
516+
_archetype_desc = {
517+
"The Partnership": "short directives + corrections + affirmation",
518+
"The Spec Dump": "detailed spec upfront, human disengages",
519+
"The Autopilot": "direction given, no feedback loop",
520+
"The Struggle": "correction-heavy, human stays engaged",
521+
"The Micromanager": "checking every few tool calls",
522+
}
504523
lines.append("")
505524
total_arch = sum(profile.archetype_distribution.values())
506525
for archetype, count in sorted(
@@ -513,6 +532,9 @@ def format_profile(profile: BehavioralProfile) -> str:
513532
lines.append(
514533
f" {archetype:22s} {count:3d} ({pct:.0f}%){marker}"
515534
)
535+
desc = _archetype_desc.get(archetype)
536+
if desc:
537+
lines.append(f" {desc}")
516538

517539
# Collaboration metrics
518540
if profile.avg_correction_rate > 0 or profile.avg_affirmation_rate > 0:

tests/test_ux.py

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -380,3 +380,155 @@ def test_verbose_adds_detail(self):
380380
verbose_lines = len(format_analysis(result, verbose=True).split("\n"))
381381
# Verbose should add content, not be identical
382382
assert verbose_lines >= default_lines
383+
384+
385+
# === Profile UX Tests ===
386+
# What a stranger sees when they run `sesh analyze --profile`.
387+
388+
389+
from sesh.analyzers.profile import BehavioralProfile, StuckPattern, format_profile
390+
391+
392+
def _build_profile(
393+
archetype_dist=None,
394+
dominant="The Partnership",
395+
stuck_patterns=None,
396+
avg_edits=10.6,
397+
) -> BehavioralProfile:
398+
"""Create a realistic behavioral profile for UX testing."""
399+
return BehavioralProfile(
400+
total_sessions=100,
401+
sessions_analyzed=95,
402+
type_distribution={
403+
"BUILD_UNCOMMITTED": 45,
404+
"BUILD_TESTED": 20,
405+
"BUILD_UNTESTED": 15,
406+
"RESEARCH": 10,
407+
"WORKSPACE": 5,
408+
},
409+
sessions_with_commits=35,
410+
total_commits=110,
411+
avg_commits_per_build=1.4,
412+
sessions_with_tests=25,
413+
test_resolution_rate=0.96,
414+
stuck_patterns=stuck_patterns or [],
415+
sessions_with_stuck=8,
416+
thrashed_files=[],
417+
avg_edits_per_commit=avg_edits,
418+
median_edits_per_commit=6.0,
419+
outcome_grades={"A": 15, "B": 12, "C": 18, "D": 30, "F": 10},
420+
avg_outcome_score=58.0,
421+
early_avg_score=55.0,
422+
recent_avg_score=70.0,
423+
trend="improving",
424+
avg_collab_score=92.0,
425+
collab_grade_distribution={"A": 50, "B": 20, "C": 15, "D": 10},
426+
archetype_distribution=archetype_dist or {
427+
"The Partnership": 60,
428+
"The Struggle": 15,
429+
"The Micromanager": 12,
430+
"The Spec Dump": 3,
431+
"The Autopilot": 5,
432+
},
433+
dominant_archetype=dominant,
434+
collab_trend="stable",
435+
early_collab_score=90.0,
436+
recent_collab_score=93.0,
437+
avg_correction_rate=0.45,
438+
avg_affirmation_rate=0.35,
439+
avg_words_per_turn=200.0,
440+
)
441+
442+
443+
class TestProfileArchetypeExplanations:
444+
"""Archetype names alone mean nothing to a stranger — descriptions required."""
445+
446+
def test_each_archetype_has_description(self):
447+
profile = _build_profile()
448+
output = format_profile(profile)
449+
for archetype in ["Partnership", "Struggle", "Micromanager", "Spec Dump", "Autopilot"]:
450+
assert archetype in output, f"Missing archetype: {archetype}"
451+
# Each archetype line should be followed by a description line
452+
lines = output.split("\n")
453+
for i, line in enumerate(lines):
454+
if "The Partnership" in line and "%" in line:
455+
assert i + 1 < len(lines), "No description after The Partnership"
456+
assert "directive" in lines[i + 1].lower() or "correction" in lines[i + 1].lower(), (
457+
f"Description after Partnership should explain the style, got: {lines[i + 1]}"
458+
)
459+
460+
def test_spec_dump_description_warns(self):
461+
"""The Spec Dump description should signal this style is problematic."""
462+
profile = _build_profile()
463+
output = format_profile(profile)
464+
lines = output.split("\n")
465+
for i, line in enumerate(lines):
466+
if "The Spec Dump" in line and "%" in line:
467+
assert "disengage" in lines[i + 1].lower(), (
468+
"Spec Dump description should mention disengagement"
469+
)
470+
471+
472+
class TestProfileStuckHints:
473+
"""Error hints must be readable — no mid-word truncation."""
474+
475+
def test_no_mid_word_truncation(self):
476+
"""Hints should end at word boundaries or with '...'."""
477+
profile = _build_profile(
478+
stuck_patterns=[
479+
StuckPattern(
480+
tool="Write",
481+
hint="<tool_use_error>File has not been read yet. Read it first before editing.",
482+
count=4,
483+
avg_length=3.0,
484+
position_bias="late",
485+
),
486+
]
487+
)
488+
output = format_profile(profile)
489+
# Should NOT end mid-word like "read y" or "been read y"
490+
assert "read y\"" not in output, "Hint truncated mid-word"
491+
# Should either show full message or end with "..."
492+
for line in output.split("\n"):
493+
if '"<tool_use_error>' in line:
494+
assert line.rstrip().endswith('..."') or "Read it first" in line, (
495+
f"Hint should end cleanly: {line}"
496+
)
497+
498+
def test_short_hints_not_truncated(self):
499+
"""Short error messages should appear in full."""
500+
profile = _build_profile(
501+
stuck_patterns=[
502+
StuckPattern(
503+
tool="Bash",
504+
hint="Exit code 1",
505+
count=3,
506+
avg_length=3.5,
507+
position_bias="mid",
508+
),
509+
]
510+
)
511+
output = format_profile(profile)
512+
assert "Exit code 1" in output
513+
assert "..." not in output.split("Exit code 1")[1].split("\n")[0]
514+
515+
516+
class TestProfileEfficiencyContext:
517+
"""Metrics without benchmarks are meaningless to a stranger."""
518+
519+
def test_edits_per_commit_has_context(self):
520+
profile = _build_profile(avg_edits=10.6)
521+
output = format_profile(profile)
522+
assert "typical" in output or "tight" in output or "high rework" in output, (
523+
"Edits/commit should include a benchmark label"
524+
)
525+
526+
def test_tight_efficiency_labeled(self):
527+
profile = _build_profile(avg_edits=3.0)
528+
output = format_profile(profile)
529+
assert "tight" in output
530+
531+
def test_high_rework_labeled(self):
532+
profile = _build_profile(avg_edits=25.0)
533+
output = format_profile(profile)
534+
assert "high rework" in output

0 commit comments

Comments
 (0)