codeprism/test-harness-fixed.yaml at main · rustic-ai/codeprism · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
global:
  max_global_concurrency: 2
  global_timeout_seconds: 300
  default_project_path: "test-projects"
  fail_fast: false
  retry:
    max_retries: 2
    retry_delay_ms: 1000
    exponential_backoff: true
    retry_on_patterns:
      - "connection refused"
      - "timeout"
  logging:
    level: "info"
    console: true
    timestamps: true
    json_format: false

server:
  start_command: "cargo"
      args: ["run", "--bin", "codeprism", "--", "--mcp"]
  env: {}
  working_dir: "."
  startup_timeout_seconds: 30
  shutdown_timeout_seconds: 10
  health_check:
    enabled: false
    interval_seconds: 10
    failure_threshold: 3
    timeout_seconds: 5

test_suites:
  - name: "basic-core-tools-fixed"
    description: "Basic test of core MCP tools with correct validation"
    parallel_execution: false

    test_cases:
      - id: "repository_stats_fixed"
        tool_name: "repository_stats"
        enabled: true
        description: "Test basic repository statistics with MCP response format"
        project_path: "test-projects/rust-test-project"
        input_params: {}

        expected:
          patterns:
            # Validate the MCP response structure first
            - key: "content"
              validation:
                type: "ArrayLength"
                min: 1
              required: true

            - key: "content[0].type"
              validation:
                type: "Equals"
                value: "text"
              required: true

            - key: "isError"
              validation:
                type: "Equals"
                value: false
              required: true

            # For now, just validate that we have text content
            # TODO: Parse the JSON within content[0].text
            - key: "content[0].text"
              validation:
                type: "Regex"
                pattern: ".*total_files.*"
              required: true

          custom_scripts: []
          allow_extra_fields: true

        performance:
          max_execution_time_ms: 10000
          max_memory_mb: 100

      - id: "search_symbols_fixed"
        tool_name: "search_symbols"
        enabled: true
        description: "Test basic symbol search with MCP response format"
        project_path: "test-projects/rust-test-project"
        input_params:
          pattern: "main"

        expected:
          patterns:
            # Validate the MCP response structure
            - key: "content"
              validation:
                type: "ArrayLength"
                min: 1
              required: true

            - key: "isError"
              validation:
                type: "Equals"
                value: false
              required: true

          custom_scripts: []
          allow_extra_fields: true

        performance:
          max_execution_time_ms: 10000
          max_memory_mb: 100

reporting:
  output_dir: "test-reports"
  formats: ["html", "json"]
  open_html: false
  include_debug_info: true
  charts:
    enabled: true
    types: ["response_time", "memory_usage", "success_rate"]
    size:
      width: 800
      height: 400
  trend_analysis: false

environment:
  variables: {}
  path_additions: []
  limits:
    max_memory_mb: 1024
    max_cpu_seconds: 300
    max_open_files: 1024
    max_process_time_seconds: 300