feat: add failure signatures and replace error clusters

src/beats/beats.api.js

@@ -71,6 +71,20 @@ class BeatsApi {
     });
   }
 
+
+  /**
+   * @param {string} run_id
+   * @returns {import('./beats.types').IFailureSignature[]}
+   */
+  getFailureSignatures(run_id) {
+    return request.get({
+      url: `${this.getBaseUrl()}/api/core/v1/automation/test-run-executions/${run_id}/failure-signatures`,
+      headers: {
+        'x-api-key': this.config.api_key
+      }
+    });
+  }
+
   /**
    *
    * @param {string} run_id

src/beats/beats.js

@@ -1,7 +1,7 @@
 const { getCIInformation } = require('../helpers/ci');
 const logger = require('../utils/logger');
 const { BeatsApi } = require('./beats.api');
-const { HOOK, PROCESS_STATUS } = require('../helpers/constants');
+const { HOOK, PROCESS_STATUS, EXTENSION } = require('../helpers/constants');
 const TestResult = require('test-results-parser/src/models/TestResult');
 const { BeatsAttachments } = require('./beats.attachments');
 
@@ -99,6 +99,7 @@ class Beats {
     await this.#attachFailureSummary();
     await this.#attachFailureAnalysis();
     await this.#attachSmartAnalysis();
+    await this.#attachFailureSignatures();
     await this.#attachErrorClusters();
   }
 
@@ -169,6 +170,30 @@ class Beats {
     }
   }
 
+  async #attachFailureSignatures() {
+    if (this.result.status !== 'FAIL') {
+      return;
+    }
+    if (this.config.show_failure_signatures === false) {
+      return;
+    }
+    try {
+      logger.info('🔍 Fetching Failure Signatures...');
+      const signatures = await this.api.getFailureSignatures(this.test_run_id);
+      this.config.extensions.push({
+        name: EXTENSION.FAILURE_SIGNATURES,
+        hook: HOOK.AFTER_SUMMARY,
+        order: 400,
+        inputs: {
+          data: signatures
+        }
+      });
+    }
+    catch (error) {
+      logger.error(`❌ Unable to attach failure signatures: ${error.message}`, error);
+    }
+  }
+
   #getDelay() {
     if (process.env.TEST_BEATS_DELAY) {
       return parseInt(process.env.TEST_BEATS_DELAY);
@@ -203,10 +228,11 @@ class Beats {
   }
 
   async #attachErrorClusters() {
-    if (this.result.status !== 'FAIL') {
+    // Legacy feature: only attach if explicitly enabled
+    if (this.config.show_error_clusters !== true) {
       return;
     }
-    if (this.config.show_error_clusters === false) {
+    if (this.result.status !== 'FAIL') {
       return;
     }
     try {
@@ -215,7 +241,7 @@ class Beats {
       this.config.extensions.push({
         name: 'error-clusters',
         hook: HOOK.AFTER_SUMMARY,
-        order: 400,
+        order: 401,
         inputs: {
           data: res.values
         }

src/beats/beats.types.d.ts

@@ -38,3 +38,11 @@ export type IFailureAnalysisMetric = {
   name: string
   count: number
 }
+
+
+export type IFailureSignature = {
+  id: string
+  signature: string
+  failure_type: string
+  count: number
+}

src/commands/generate-config.command.js

@@ -432,7 +432,8 @@ class GenerateConfigCommand {
             { title: 'Metadata', value: 'metadata' },
             { title: 'AI Failure Summary', value: 'ai-failure-summary' },
             { title: 'Smart Analysis', value: 'smart-analysis' },
-            { title: 'Error Clusters', value: 'error-clusters' }
+            { title: 'Failure Signatures', value: 'failure-signatures' },
+            { title: 'Error Clusters (Legacy)', value: 'error-clusters' }
         ];
     }
 }

src/extensions/failure-signatures.extension.js

@@ -0,0 +1,43 @@
+const { BaseExtension } = require('./base.extension');
+const { STATUS, HOOK } = require("../helpers/constants");
+const { truncate } = require('../helpers/helper');
+
+class FailureSignaturesExtension extends BaseExtension {
+
+  constructor(target, extension, result, payload, root_payload) {
+    super(target, extension, result, payload, root_payload);
+    this.#setDefaultOptions();
+    this.#setDefaultInputs();
+    this.updateExtensionInputs();
+  }
+
+  run() {
+    this.#setText();
+    this.attach();
+  }
+
+  #setDefaultOptions() {
+    this.default_options.hook = HOOK.AFTER_SUMMARY,
+    this.default_options.condition = STATUS.PASS_OR_FAIL;
+  }
+
+  #setDefaultInputs() {
+    this.default_inputs.title = 'Top Failures';
+    this.default_inputs.title_link = '';
+  }
+
+  #setText() {
+    const signatures = this.extension.inputs.data;
+    if (!signatures || !signatures.length) {
+      return;
+    }
+
+    const texts = [];
+    for (const signature of signatures) {
+      texts.push(`${truncate(signature.signature, 150)} - ${this.bold(`(x${signature.count})`)}`);
+    }
+    this.text = this.platform.bullets(texts);
+  }
+}
+
+module.exports =  { FailureSignaturesExtension }

src/extensions/index.js

@@ -13,6 +13,7 @@ const { EXTENSION } = require('../helpers/constants');
 const { checkCondition } = require('../helpers/helper');
 const logger = require('../utils/logger');
 const { ErrorClustersExtension } = require('./error-clusters.extension');
+const { FailureSignaturesExtension } = require('./failure-signatures.extension');
 const { FailureAnalysisExtension } = require('./failure-analysis.extension');
 const { BrowserstackExtension } = require('./browserstack.extension');
 
@@ -75,6 +76,8 @@ function getExtensionRunner(extension, options) {
       return new ErrorClustersExtension(options.target, extension, options.result, options.payload, options.root_payload);
     case EXTENSION.BROWSERSTACK:
       return new BrowserstackExtension(options.target, extension, options.result, options.payload, options.root_payload);
+    case EXTENSION.FAILURE_SIGNATURES:
+      return new FailureSignaturesExtension(options.target, extension, options.result, options.payload, options.root_payload);
     default:
       return require(extension.name);
   }

src/helpers/constants.js

@@ -27,6 +27,7 @@ const EXTENSION = Object.freeze({
   FAILURE_ANALYSIS: 'failure-analysis',
   SMART_ANALYSIS: 'smart-analysis',
   ERROR_CLUSTERS: 'error-clusters',
+  FAILURE_SIGNATURES: 'failure-signatures',
   BROWSERSTACK: 'browserstack',
   HYPERLINKS: 'hyperlinks',
   MENTIONS: 'mentions',

src/index.d.ts

@@ -20,10 +20,10 @@ export interface IExtension {
   condition?: Condition;
   hook?: Hook;
   order?: number;
-  inputs?: ReportPortalAnalysisInputs | ReportPortalHistoryInputs | HyperlinkInputs | MentionInputs | QuickChartTestSummaryInputs | PercyAnalysisInputs | CustomExtensionInputs | MetadataInputs | CIInfoInputs | AIFailureSummaryInputs | BrowserstackInputs;
+  inputs?: ReportPortalAnalysisInputs | ReportPortalHistoryInputs | HyperlinkInputs | MentionInputs | QuickChartTestSummaryInputs | PercyAnalysisInputs | CustomExtensionInputs | MetadataInputs | CIInfoInputs | AIFailureSummaryInputs | BrowserstackInputs | FailureSignaturesInputs | ErrorClustersInputs;
 }
 
-export type ExtensionName = 'report-portal-analysis' | 'hyperlinks' | 'mentions' | 'report-portal-history' | 'quick-chart-test-summary' | 'metadata' | 'ci-info' | 'custom' | 'ai-failure-summary';
+export type ExtensionName = 'report-portal-analysis' | 'hyperlinks' | 'mentions' | 'report-portal-history' | 'quick-chart-test-summary' | 'metadata' | 'ci-info' | 'custom' | 'ai-failure-summary' | 'failure-signatures' | 'error-clusters';
 export type Hook = 'start' | 'end' | 'after-summary';
 export type TargetName = 'slack' | 'teams' | 'chat' | 'github' | 'github-output' | 'custom' | 'delay';
 export type PublishReportType = 'test-summary' | 'test-summary-slim' | 'failure-details';
@@ -86,6 +86,23 @@ export interface AIFailureSummaryInputs extends ExtensionInputs {
   failure_summary: string;
 }
 
+export interface FailureSignaturesInputs extends ExtensionInputs {
+  data?: Array<{
+    id: string;
+    signature: string;
+    failure_type: string;
+    count: number;
+  }>;
+}
+
+export interface ErrorClustersInputs extends ExtensionInputs {
+  data?: Array<{
+    test_failure_id: string;
+    failure: string;
+    count: number;
+  }>;
+}
+
 export interface BrowserStackAutomationBuild {
   name: string;
   hashed_id: string;
@@ -309,7 +326,9 @@ export interface PublishReport {
   show_failure_summary?: boolean;
   show_failure_analysis?: boolean;
   show_smart_analysis?: boolean;
+  /** @deprecated Use show_failure_signatures instead. Legacy feature - only enabled when explicitly set to true. */
   show_error_clusters?: boolean;
+  show_failure_signatures?: boolean;
   targets?: ITarget[];
   extensions?: IExtension[];
   results?: ParseOptions[] | PerformanceParseOptions[] | CustomResultOptions[];

test/beats.spec.js

@@ -47,7 +47,7 @@ describe('TestBeats', () => {
   it('should send results with failures to beats', async () => {
     const id1 = mock.addInteraction('post test results to beats');
     const id2 = mock.addInteraction('get test results from beats');
-    const id3 = mock.addInteraction('get empty error clusters from beats');
+    const id3 = mock.addInteraction('get empty failure signatures from beats');
     const id4 = mock.addInteraction('post test-summary with beats to teams with ai failure summary');
     await publish({
       config: {
@@ -82,7 +82,7 @@ describe('TestBeats', () => {
     const id1 = mock.addInteraction('post test results to beats');
     const id2 = mock.addInteraction('get test results from beats');
     const id3 = mock.addInteraction('upload attachments');
-    const id4 = mock.addInteraction('get empty error clusters from beats');
+    const id4 = mock.addInteraction('get empty failure signatures from beats');
     const id5 = mock.addInteraction('post test-summary to teams with strict as false');
     await publish({
       config: {
@@ -118,7 +118,7 @@ describe('TestBeats', () => {
     const id1 = mock.addInteraction('post test results to beats');
     const id2 = mock.addInteraction('get test results from beats');
     const id3 = mock.addInteraction('upload attachments');
-    const id4 = mock.addInteraction('get empty error clusters from beats');
+    const id4 = mock.addInteraction('get empty failure signatures from beats');
     await publish({
       config: {
         api_key: 'api-key',
@@ -165,7 +165,7 @@ describe('TestBeats', () => {
   it('should send results with smart analysis to beats', async () => {
     const id1 = mock.addInteraction('post test results to beats');
     const id2 = mock.addInteraction('get test results with smart analysis from beats');
-    const id3 = mock.addInteraction('get empty error clusters from beats');
+    const id3 = mock.addInteraction('get empty failure signatures from beats');
     const id4 = mock.addInteraction('post test-summary with beats to teams with ai failure summary and smart analysis');
     await publish({
       config: {
@@ -196,14 +196,79 @@ describe('TestBeats', () => {
     assert.equal(mock.getInteraction(id4).exercised, true);
   });
 
-  it('should send results with error clusters to beats', async () => {
+  it('should send results with error clusters to beats (legacy opt-in)', async () => {
     const id1 = mock.addInteraction('post test results to beats');
     const id2 = mock.addInteraction('get test results from beats');
     const id3 = mock.addInteraction('get error clusters from beats');
-    const id4 = mock.addInteraction('post test-summary with beats to teams with error clusters');
-    const id5 = mock.addInteraction('post test-summary with beats to slack with error clusters');
-    const id6 = mock.addInteraction('post test-summary with beats to chat with error clusters');
-    const id7 = mock.addInteraction('post test-summary with beats to github with error clusters');
+    const id4 = mock.addInteraction('get empty failure signatures from beats');
+    const id5 = mock.addInteraction('post test-summary with beats to teams with error clusters');
+    const id6 = mock.addInteraction('post test-summary with beats to slack with error clusters');
+    const id7 = mock.addInteraction('post test-summary with beats to chat with error clusters');
+    const id8 = mock.addInteraction('post test-summary with beats to github with error clusters');
+    await publish({
+      config: {
+        api_key: 'api-key',
+        project: 'project-name',
+        run: 'build-name',
+        show_error_clusters: true,
+        targets: [
+          {
+            name: 'teams',
+            inputs: {
+              url: 'http://localhost:9393/message'
+            }
+          },
+          {
+            name: 'slack',
+            inputs: {
+              url: 'http://localhost:9393/message'
+            }
+          },
+          {
+            name: 'chat',
+            inputs: {
+              url: 'http://localhost:9393/message'
+            }
+          },
+          {
+            name: 'github',
+            inputs: {
+              url: 'http://localhost:9393',
+              owner: 'org',
+              repo: 'repo',
+              pull_number: '123',
+              token: 'test-token'
+            }
+          }
+        ],
+        results: [
+          {
+            type: 'testng',
+            files: [
+              'test/data/testng/single-suite-failures.xml'
+            ]
+          }
+        ]
+      }
+    });
+    assert.equal(mock.getInteraction(id1).exercised, true);
+    assert.equal(mock.getInteraction(id2).exercised, true);
+    assert.equal(mock.getInteraction(id3).exercised, true);
+    assert.equal(mock.getInteraction(id4).exercised, true);
+    assert.equal(mock.getInteraction(id5).exercised, true);
+    assert.equal(mock.getInteraction(id6).exercised, true);
+    assert.equal(mock.getInteraction(id7).exercised, true);
+    assert.equal(mock.getInteraction(id8).exercised, true);
+  });
+
+  it('should send results with failure signatures to beats', async () => {
+    const id1 = mock.addInteraction('post test results to beats');
+    const id2 = mock.addInteraction('get test results from beats');
+    const id3 = mock.addInteraction('get failure signatures from beats');
+    const id4 = mock.addInteraction('post test-summary with beats to teams with failure signatures');
+    const id5 = mock.addInteraction('post test-summary with beats to slack with failure signatures');
+    const id6 = mock.addInteraction('post test-summary with beats to chat with failure signatures');
+    const id7 = mock.addInteraction('post test-summary with beats to github with failure signatures');
     await publish({
       config: {
         api_key: 'api-key',
@@ -261,7 +326,7 @@ describe('TestBeats', () => {
   it('should send results with failure analysis to beats', async () => {
     const id1 = mock.addInteraction('post test results to beats');
     const id2 = mock.addInteraction('get test results with failure analysis from beats');
-    const id3 = mock.addInteraction('get empty error clusters from beats');
+    const id3 = mock.addInteraction('get empty failure signatures from beats');
     const id4 = mock.addInteraction('get failure analysis from beats');
     const id5 = mock.addInteraction('post test-summary with beats to teams with ai failure summary and smart analysis and failure analysis');
     await publish({