Add testing section to README and update test scripts for improved structure

- Introduced a new "Testing" section in the README.md to provide quick start instructions for running tests, including commands for unit, integration, and end-to-end tests.
- Updated test scripts to reflect changes in directory structure and improve clarity, including adjustments to alert handling and mock functions.
- Enhanced test cases to ensure better coverage and reliability, particularly in monitoring and data integrity checks.

Author: angoca

README.md

@@ -647,6 +647,24 @@ bash scripts/generate_coverage_combined.sh
 
 For detailed explanation, see: [Code Coverage Explanation](./docs/Coverage_Explanation.md)
 
+## Testing
+
+### Quick Start
+
+```bash
+# Run all tests (master test runner)
+./tests/run_all_tests.sh
+
+# Run specific test suites
+./tests/run_all_tests.sh --unit-only          # Unit tests only
+./tests/run_all_tests.sh --integration-only  # Integration tests only
+./tests/run_all_tests.sh --e2e-only          # End-to-end tests only
+```
+
+**Master Test Runner**: `tests/run_all_tests.sh` - Executes all test suites (unit, integration, e2e, SQL, scripts, performance)
+
+For detailed testing information, see the test documentation in the `tests/` directory.
+
 ## Configuration
 
 ### Quick Setup

tests/unit/monitor/test_monitorAPI.sh

@@ -178,16 +178,37 @@ teardown() {
 # Test: check_api_availability alerts when API is unavailable
 ##
 @test "check_api_availability alerts when API is unavailable" {
-    # Mock curl to return failure
-    # shellcheck disable=SC2317
-    curl() {
-        if [[ "${1}" == "-w" ]]; then
-            echo "503"  # Service unavailable
-            return 1
-        fi
-        return 1
-    }
-    export -f curl
+    # Create mock curl executable so command -v finds it
+    local mock_curl_dir="${BATS_TEST_DIRNAME}/../../tmp/bin"
+    mkdir -p "${mock_curl_dir}"
+    local mock_curl="${mock_curl_dir}/curl"
+    cat > "${mock_curl}" << 'EOF'
+#!/bin/bash
+# Return HTTP 503 when called with -w "%{http_code}"
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -w)
+            shift
+            if [[ "$1" == "%{http_code}" ]]; then
+                echo "503"
+                exit 0
+            elif [[ "$1" == "%{time_total}" ]]; then
+                echo "0.5"
+                exit 0
+            fi
+            ;;
+        -s|-o|--max-time|--connect-timeout)
+            shift
+            ;;
+        *)
+            ;;
+    esac
+    shift
+done
+exit 0
+EOF
+    chmod +x "${mock_curl}"
+    export PATH="${mock_curl_dir}:${PATH}"
 
     # Mock log functions
     # shellcheck disable=SC2317
@@ -214,9 +235,11 @@ teardown() {
 
     # shellcheck disable=SC2317
     send_alert() {
-        # Check if it's an API unavailable alert (4th arg is message)
+        # Check if it's an API unavailable alert (arguments: component, level, type, message)
+        local alert_type="${3:-}"
         local message="${4:-}"
-        if echo "${message}" | grep -q "API.*unavailable\|API.*returned"; then
+        # The alert type should be "api_unavailable" or message should contain "API" and "returned" or "unavailable"
+        if [[ "${alert_type}" == "api_unavailable" ]] || echo "${message}" | grep -qiE "(API.*returned|API.*unavailable|unavailable)"; then
             touch "${alert_file}"
         fi
         return 0
@@ -228,18 +251,30 @@ teardown() {
 
     # Alert should have been sent
     assert_file_exists "${alert_file}"
+    
+    # Cleanup
+    rm -rf "${mock_curl_dir}"
 }
 
 ##
 # Test: check_api_availability handles curl unavailability gracefully
 ##
 @test "check_api_availability handles curl unavailability gracefully" {
-    # Unset curl command
-    # shellcheck disable=SC2317
-    curl() {
-        return 127  # Command not found
-    }
-    export -f curl
+    # Remove curl from PATH temporarily to simulate it not being available
+    local original_path="${PATH}"
+    # Create a temporary PATH without curl
+    local temp_path=""
+    IFS=':' read -ra path_parts <<< "${PATH}"
+    for part in "${path_parts[@]}"; do
+        if [[ "${part}" != *"tmp/bin"* ]] && [[ "${part}" != *"/usr/bin"* ]] && [[ "${part}" != *"/bin"* ]]; then
+            if [[ -z "${temp_path}" ]]; then
+                temp_path="${part}"
+            else
+                temp_path="${temp_path}:${part}"
+            fi
+        fi
+    done
+    export PATH="${temp_path}"
 
     # Mock log functions
     # shellcheck disable=SC2317
@@ -269,6 +304,9 @@ teardown() {
     # Run check
     run check_api_availability
 
+    # Restore PATH
+    export PATH="${original_path}"
+    
     # Should succeed (skip gracefully)
     assert_success
 }

tests/unit/monitor/test_monitorAnalytics.sh

@@ -19,7 +19,7 @@ setup() {
     export LOG_LEVEL="${LOG_LEVEL_DEBUG}"
 
     # Create test directories
-    mkdir -p "${TEST_ANALYTICS_DIR}/bin"
+    mkdir -p "${TEST_ANALYTICS_DIR}/bin/dwh"
     mkdir -p "${TEST_ANALYTICS_DIR}/logs"
     mkdir -p "${TEST_LOG_DIR}"
 
@@ -137,7 +137,8 @@ create_test_etl_script() {
     local script_name="${1}"
     local executable="${2:-true}"
 
-    local script_path="${TEST_ANALYTICS_DIR}/bin/${script_name}"
+    # Function looks for scripts in bin/dwh directory
+    local script_path="${TEST_ANALYTICS_DIR}/bin/dwh/${script_name}"
     echo "#!/bin/bash" > "${script_path}"
     echo "# Test ETL script ${script_name}" >> "${script_path}"
 
@@ -226,11 +227,11 @@ create_test_log() {
 
 @test "check_etl_job_execution_status alerts when scripts not executable" {
     # Create scripts but make one non-executable
-    # The function looks for specific script names: etl_main.sh, etl_daily.sh, etl_hourly.sh, load_data.sh, transform_data.sh
+    # The function looks for specific script names: ETL.sh, cleanupDWH.sh, copyBaseTables.sh
     # Need at least threshold number of scripts (threshold is 2)
-    create_test_etl_script "etl_main.sh" "true"
-    create_test_etl_script "etl_daily.sh" "false"  # This one is not executable
-    create_test_etl_script "etl_hourly.sh" "true"
+    create_test_etl_script "ETL.sh" "true"
+    create_test_etl_script "cleanupDWH.sh" "false"  # This one is not executable
+    create_test_etl_script "copyBaseTables.sh" "true"
 
     # Use a temp file to track alert status
     local alert_file="${TEST_ANALYTICS_DIR}/.alert_sent"
@@ -246,7 +247,8 @@ create_test_log() {
     send_alert() {
         # Check if message contains executable count warning
         # The actual message is: "ETL scripts executable count (X) is less than scripts found (Y)"
-        if [[ "${4}" == *"executable"* ]] || [[ "${4}" == *"ETL scripts executable count"* ]] || [[ "${4}" == *"less than"* ]] || [[ "${4}" == *"not executable"* ]]; then
+        local message="${4:-}"
+        if echo "${message}" | grep -qiE "(executable.*count|less than|not executable|scripts.*executable)"; then
             touch "${alert_file}"
         fi
         return 0

tests/unit/monitor/test_monitorData.sh

@@ -438,7 +438,10 @@ create_test_backup() {
 
     # shellcheck disable=SC2317
     send_alert() {
-        if [[ "${4}" == *"File integrity check found"* ]]; then
+        # Check if it's a file integrity alert
+        local alert_type="${3:-}"
+        local message="${4:-}"
+        if [[ "${alert_type}" == "file_integrity_failure" ]] || echo "${message}" | grep -qiE "(file integrity|integrity.*failure|corrupted)"; then
             touch "${alert_file}"
         fi
         return 0
@@ -454,7 +457,29 @@ create_test_backup() {
 
 @test "check_file_integrity alerts when compressed files are corrupted" {
     # Create corrupted compressed file (invalid gzip)
+    # Make it old enough (>60 seconds) so it's not skipped
     echo "not a valid gzip file" > "${TEST_BACKUP_DIR}/backup.tar.gz"
+    touch -t "$(date -d '2 hours ago' +%Y%m%d%H%M.%S)" "${TEST_BACKUP_DIR}/backup.tar.gz" 2>/dev/null || \
+    touch -t "$(date -v-2H +%Y%m%d%H%M.%S 2>/dev/null || date +%Y%m%d%H%M.%S)" "${TEST_BACKUP_DIR}/backup.tar.gz"
+    
+    # Mock log functions
+    # shellcheck disable=SC2317
+    log_info() {
+        return 0
+    }
+    export -f log_info
+    
+    # shellcheck disable=SC2317
+    log_warning() {
+        return 0
+    }
+    export -f log_warning
+    
+    # shellcheck disable=SC2317
+    log_debug() {
+        return 0
+    }
+    export -f log_debug
 
     # Mock record_metric
     # shellcheck disable=SC2317
@@ -469,7 +494,13 @@ create_test_backup() {
 
     # shellcheck disable=SC2317
     send_alert() {
-        if [[ "${4}" == *"File integrity check found"* ]]; then
+        # Check alert type or message content - be more flexible
+        local alert_type="${3:-}"
+        local message="${4:-}"
+        if [[ "${alert_type}" == "file_integrity_failure" ]] || \
+           [[ "${message}" == *"File integrity check found"* ]] || \
+           [[ "${message}" == *"integrity"* ]] || \
+           [[ "${message}" == *"corrupted"* ]]; then
             touch "${alert_file}"
         fi
         return 0