Fix ClickHouse loader tests and add Makefile target

- Add `make test-clickhouse` target
- Fix testcontainer readiness check (remove broken wait_for_logs, rely on
  built-in HTTP health check)
- Import container classes independently so missing driver packages don't
  block other containers
- Fix ClickHouse test config credentials to match container defaults
- Remove calls to nonexistent _record_connection_opened/_closed methods
- Set supports_overwrite=False since ClickHouse only supports APPEND mode
- Fix base test_batch_loading to respect supports_overwrite config

Author: fordN

Makefile

@@ -40,6 +40,10 @@ test-snowflake:
 	@echo "❄️ Running Snowflake tests..."
 	$(PYTHON) pytest tests/ -m "snowflake" -v --log-cli-level=ERROR
 
+test-clickhouse:
+	@echo "🏠 Running ClickHouse tests..."
+	$(PYTHON) pytest tests/ -m "clickhouse" -v --log-cli-level=ERROR
+
 test-lmdb:
 	@echo "⚡ Running LMDB tests..."
 	$(PYTHON) pytest tests/ -m "lmdb" -v --log-cli-level=ERROR
@@ -131,6 +135,7 @@ help:
 	@echo "  make test-all                 - Run all tests with coverage"
 	@echo "  make test-postgresql          - Run PostgreSQL tests"
 	@echo "  make test-redis               - Run Redis tests"
+	@echo "  make test-clickhouse          - Run ClickHouse tests"
 	@echo "  make test-snowflake           - Run Snowflake tests"
 	@echo "  make test-performance         - Run performance tests"
 	@echo "  make lint                     - Lint code with ruff"

src/amp/loaders/implementations/clickhouse_loader.py

@@ -114,7 +114,6 @@ def connect(self) -> None:
             self.logger.info(f'Found {len(tables.result_rows)} tables in database')
 
             self._is_connected = True
-            self._record_connection_opened()
 
         except ImportError as err:
             raise ImportError(
@@ -126,13 +125,10 @@ def connect(self) -> None:
 
     def disconnect(self) -> None:
         """Close ClickHouse connection"""
-        was_connected = self._is_connected
         if self._client:
             self._client.close()
             self._client = None
         self._is_connected = False
-        if was_connected:
-            self._record_connection_closed()
         self.logger.info('Disconnected from ClickHouse')
 
     def _load_batch_impl(self, batch: pa.RecordBatch, table_name: str, **kwargs) -> int:

tests/conftest.py

@@ -29,19 +29,30 @@
     if colima_socket.exists():
         os.environ['DOCKER_HOST'] = f'unix://{colima_socket}'
 
-# Import testcontainers conditionally
+# Import testcontainers conditionally (each container imported independently
+# so missing driver packages don't block other containers)
+TESTCONTAINERS_AVAILABLE = False
+PostgresContainer = None
+RedisContainer = None
+ClickHouseContainer = None
+
 if USE_TESTCONTAINERS:
     try:
         from testcontainers.postgres import PostgresContainer
+    except ImportError:
+        pass
+    try:
         from testcontainers.redis import RedisContainer
+    except ImportError:
+        pass
+    try:
         from testcontainers.clickhouse import ClickHouseContainer
-
-        TESTCONTAINERS_AVAILABLE = True
     except ImportError:
-        TESTCONTAINERS_AVAILABLE = False
+        pass
+
+    TESTCONTAINERS_AVAILABLE = any(c is not None for c in [PostgresContainer, RedisContainer, ClickHouseContainer])
+    if not TESTCONTAINERS_AVAILABLE:
         logging.warning('Testcontainers not available. Falling back to manual configuration.')
-else:
-    TESTCONTAINERS_AVAILABLE = False
 
 
 # Shared configuration fixtures
@@ -137,8 +148,8 @@ def test_config():
 @pytest.fixture(scope='session')
 def postgres_container():
     """PostgreSQL container for integration tests"""
-    if not TESTCONTAINERS_AVAILABLE:
-        pytest.skip('Testcontainers not available')
+    if PostgresContainer is None:
+        pytest.skip('Testcontainers for PostgreSQL not available')
 
     import time
 
@@ -161,8 +172,8 @@ def postgres_container():
 @pytest.fixture(scope='session')
 def redis_container():
     """Redis container for integration tests"""
-    if not TESTCONTAINERS_AVAILABLE:
-        pytest.skip('Testcontainers not available')
+    if RedisContainer is None:
+        pytest.skip('Testcontainers for Redis not available')
 
     from testcontainers.core.waiting_utils import wait_for_logs
 
@@ -180,22 +191,12 @@ def redis_container():
 @pytest.fixture(scope='session')
 def clickhouse_container():
     """ClickHouse container for integration tests"""
-    if not TESTCONTAINERS_AVAILABLE:
-        pytest.skip('Testcontainers not available')
-
-    import time
-
-    from testcontainers.core.waiting_utils import wait_for_logs
+    if ClickHouseContainer is None:
+        pytest.skip('Testcontainers for ClickHouse not available')
 
     container = ClickHouseContainer(image='clickhouse/clickhouse-server:24-alpine')
     container.start()
 
-    # Wait for ClickHouse to be ready
-    wait_for_logs(container, 'Ready for connections', timeout=60)
-
-    # Additional wait for HTTP interface to be ready
-    time.sleep(2)
-
     yield container
 
     container.stop()
@@ -204,7 +205,7 @@ def clickhouse_container():
 @pytest.fixture(scope='session')
 def postgresql_test_config(request):
     """PostgreSQL configuration from testcontainer or environment"""
-    if TESTCONTAINERS_AVAILABLE and USE_TESTCONTAINERS:
+    if PostgresContainer is not None:
         # Get the postgres_container fixture
         postgres_container = request.getfixturevalue('postgres_container')
         return {
@@ -224,7 +225,7 @@ def postgresql_test_config(request):
 @pytest.fixture(scope='session')
 def redis_test_config(request):
     """Redis configuration from testcontainer or environment"""
-    if TESTCONTAINERS_AVAILABLE and USE_TESTCONTAINERS:
+    if RedisContainer is not None:
         # Get the redis_container fixture
         redis_container = request.getfixturevalue('redis_container')
         return {
@@ -244,15 +245,15 @@ def redis_test_config(request):
 @pytest.fixture(scope='session')
 def clickhouse_test_config(request):
     """ClickHouse configuration from testcontainer or environment"""
-    if TESTCONTAINERS_AVAILABLE and USE_TESTCONTAINERS:
+    if ClickHouseContainer is not None:
         # Get the clickhouse_container fixture
         clickhouse_container = request.getfixturevalue('clickhouse_container')
         return {
             'host': clickhouse_container.get_container_host_ip(),
             'port': clickhouse_container.get_exposed_port(8123),
-            'database': 'default',
-            'username': 'default',
-            'password': '',
+            'database': clickhouse_container.dbname,
+            'username': clickhouse_container.username,
+            'password': clickhouse_container.password,
             'batch_size': 100000,
         }
     else:

tests/integration/loaders/backends/test_clickhouse.py

@@ -25,7 +25,7 @@ class ClickHouseTestConfig(LoaderTestConfig):
     loader_class = ClickHouseLoader
     config_fixture_name = 'clickhouse_test_config'
 
-    supports_overwrite = True
+    supports_overwrite = False
     supports_streaming = True
     supports_multi_network = True
     supports_null_values = True

tests/integration/loaders/test_base_loader.py

@@ -57,7 +57,7 @@ def test_batch_loading(self, loader, medium_test_table, test_table_name, cleanup
             batches = medium_test_table.to_batches(max_chunksize=250)
 
             for i, batch in enumerate(batches):
-                mode = LoadMode.OVERWRITE if i == 0 else LoadMode.APPEND
+                mode = LoadMode.OVERWRITE if i == 0 and self.config.supports_overwrite else LoadMode.APPEND
                 result = loader.load_batch(batch, test_table_name, mode=mode)
 
                 assert result.success == True