test: move ray and spark tests to component layout (#6050)

Signed-off-by: Nikoloz Turazashvili <turazashvili@gmail.com>

Author: turazashvili

Makefile

@@ -186,7 +186,7 @@ test-python-integration: ## Run Python integration tests (CI)
 		-k "(not snowflake or not test_historical_features_main)" \
 		-m "not rbac_remote_integration_test and not ray_offline_stores_only" \
 		--ignore=sdk/python/tests/integration/registration \
-		--ignore=sdk/python/tests/integration/compute_engines/ray_compute \
+		--ignore=sdk/python/tests/component/ray \
 		--log-cli-level=INFO -s \
 		sdk/python/tests
 
@@ -203,7 +203,7 @@ test-python-integration-local: ## Run Python integration tests (local dev mode)
 	uv run python -m pytest --tb=short -v -n auto --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
 		-k "not test_lambda_materialization and not test_snowflake_materialization" \
 		-m "not rbac_remote_integration_test and not ray_offline_stores_only" \
-		--ignore=sdk/python/tests/integration/compute_engines/ray_compute \
+		--ignore=sdk/python/tests/component/ray \
 		--ignore=sdk/python/tests/integration/registration \
 		--log-cli-level=INFO -s \
 		sdk/python/tests

pixi.lock

@@ -289,7 +289,7 @@ feast = { path = ".", editable = true, extras = ["ray", "grpcio", "test"] }
 
 [tool.pixi.feature.ray-tests.tasks]
 test-offline = { cmd = "python -m pytest -v --integration --ignore=sdk/python/tests/integration/offline_store/test_dqm_validation.py sdk/python/tests/integration/offline_store", env = { PYTHONPATH = ".", FULL_REPO_CONFIGS_MODULE = "sdk.python.tests.universal.feature_repos.ray_repo_configuration", FEAST_IS_LOCAL_TEST = "True" } }
-test-compute = { cmd = "python -m pytest -v --integration sdk/python/tests/integration/compute_engines/ray_compute", env = { PYTHONPATH = ".", FEAST_IS_LOCAL_TEST = "True" } }
+test-compute = { cmd = "python -m pytest -v --integration sdk/python/tests/component/ray", env = { PYTHONPATH = ".", FEAST_IS_LOCAL_TEST = "True" } }
 test = { depends-on = ["test-offline", "test-compute"] }
 
 [tool.pixi.feature.registration-tests.pypi-dependencies]

pyproject.toml

@@ -4,7 +4,7 @@
 auto-discovered by pytest.
 """
 
-from tests.integration.compute_engines.ray_compute.ray_shared_utils import (
+from tests.component.ray.ray_shared_utils import (
     entity_df,
     feature_dataset,
     ray_environment,

…/compute_engines/ray_compute/conftest.py …k/python/tests/component/ray/conftest.pysdk/python/tests/integration/compute_engines/ray_compute/conftest.py renamed to sdk/python/tests/component/ray/conftest.py sdk/python/tests/integration/compute_engines/ray_compute/conftest.py renamed to sdk/python/tests/component/ray/conftest.py

@@ -21,7 +21,7 @@
 )
 from feast.transformation.ray_transformation import RayTransformation
 from feast.types import Float32, Int32, Int64
-from tests.integration.compute_engines.ray_compute.ray_shared_utils import (
+from tests.component.ray.ray_shared_utils import (
     driver,
     now,
 )

…_engines/ray_compute/ray_shared_utils.py …/tests/component/ray/ray_shared_utils.pysdk/python/tests/integration/compute_engines/ray_compute/ray_shared_utils.py renamed to sdk/python/tests/component/ray/ray_shared_utils.py sdk/python/tests/integration/compute_engines/ray_compute/ray_shared_utils.py renamed to sdk/python/tests/component/ray/ray_shared_utils.py

@@ -10,7 +10,7 @@
     MaterializationJobStatus,
 )
 from feast.types import Float32, Int32, Int64
-from tests.integration.compute_engines.ray_compute.ray_shared_utils import (
+from tests.component.ray.ray_shared_utils import (
     create_entity_df,
     create_feature_dataset,
     create_unique_sink_source,

…ngines/ray_compute/repo_configuration.py …ests/component/ray/repo_configuration.pysdk/python/tests/integration/compute_engines/ray_compute/repo_configuration.py renamed to sdk/python/tests/component/ray/repo_configuration.py sdk/python/tests/integration/compute_engines/ray_compute/repo_configuration.py renamed to sdk/python/tests/component/ray/repo_configuration.py

@@ -0,0 +1,23 @@
+import pytest
+from pyspark.sql import SparkSession
+
+
+@pytest.fixture(scope="session")
+def spark_session():
+    spark = (
+        SparkSession.builder.appName("FeastSparkTests")
+        .master("local[*]")
+        .config("spark.sql.shuffle.partitions", "1")
+        .config("spark.driver.host", "127.0.0.1")
+        .config("spark.driver.bindAddress", "127.0.0.1")
+        .getOrCreate()
+    )
+
+    yield spark
+
+    spark.stop()
+
+
+@pytest.fixture
+def spark_fixture(spark_session):
+    yield spark_session