feat: TabPFN implementation

.github/workflows/integration-build.yml

@@ -15,7 +15,7 @@ jobs:
     timeout-minutes: 95
     strategy:
       matrix:
-        python-version: [ 3.8, 3.9, '3.10' ]
+        python-version: [ 3.9, '3.10' ]
 
     steps:
       - name: Checkout branch

.github/workflows/publish_pypi.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ 3.8 ]
+        python-version: [ 3.9 ]
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/unit-build.yml

@@ -17,7 +17,7 @@ jobs:
     timeout-minutes: 15
     strategy:
       matrix:
-        python-version: [ 3.8, 3.9, '3.10' ]
+        python-version: [ 3.9, '3.10' ]
 
     steps:
     - uses: actions/checkout@v2

.readthedocs.yml

@@ -8,7 +8,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.8"
+    python: "3.9"
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

.travis.yml

@@ -1,7 +1,6 @@
 language: python
 
 python:
-  - "3.8"
   - "3.9"
   - "3.10"
 

docs/source/introduction/fedot_features/automation_features.rst

@@ -173,6 +173,10 @@ Apart from that there are other options whose names speak for themselves: ``'sta
    `polyfit`,Polynomial approximation,Forecasting
    `stl_arima`,STL Decomposition with ARIMA,Forecasting
    `ts_naive_average`,Naive Average,Forecasting
+   `tabpfn`,TabPFN classifier,Classification
+   `tabpfnreg`,TabPFN regressor,Regression,
+   `autotabpfn`,AutoTabPFN classifier,Classification
+   `autotabpfnreg`,AutoTabPFN regressor,Regression,
 
 
 .. csv-table:: Available models implementations

fedot/core/operations/evaluation/operation_implementations/models/tabpfn.py

@@ -0,0 +1,100 @@
+import os
+
+import numpy as np
+from tabpfn import TabPFNClassifier, TabPFNRegressor
+from tabpfn_extensions.post_hoc_ensembles.sklearn_interface import AutoTabPFNClassifier, AutoTabPFNRegressor
+from typing import Optional
+from fedot.core.data.data import InputData, OutputData
+from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ModelImplementation
+from fedot.core.operations.operation_parameters import OperationParameters
+from fedot.core.utils import default_fedot_data_dir
+
+
+class FedotTabPFNImplementation(ModelImplementation):
+    __operation_params = [
+        'enable_categorical',
+        'max_samples',
+        'max_features',
+        'model_path'
+    ]
+
+    def __init__(self, params: Optional[OperationParameters] = None):
+        super().__init__(params)
+
+        self.model_params = {
+            k: v for k, v in self.params.to_dict().items() if k not in self.__operation_params
+        }
+
+        model_path = self.params.get('model_path', None)
+        if model_path == "auto":
+            self.model_params['model_path'] = os.path.join(default_fedot_data_dir(), 'tabpfn')
+        elif model_path is not None:
+            self.model_params['model_path'] = model_path
+
+        self.model = None
+        self.classes_ = None
+
+    def fit(self, input_data: InputData):
+        self.model.categorical_features_indices = input_data.categorical_idx
+
+        if self.params.get('enable_categorical'):
+            input_data = input_data.get_not_encoded_data()
+
+        self.model.fit(X=input_data.features, y=input_data.target)
+
+        return self.model
+
+    def predict(self, input_data: InputData) -> OutputData:
+        if self.params.get('enable_categorical'):
+            input_data = input_data.get_not_encoded_data()
+
+        prediction = self.model.predict(input_data.features)
+
+        output_data = self._convert_to_output(
+            input_data=input_data,
+            predict=prediction
+        )
+        return output_data
+
+    def predict_proba(self, input_data: InputData):
+        if self.params.get('enable_categorical'):
+            input_data = input_data.get_not_encoded_data()
+
+        prediction = self.model.predict_proba(input_data.features)
+        output_data = self._convert_to_output(
+            input_data=input_data,
+            predict=prediction
+        )
+        return output_data
+
+
+class FedotTabPFNClassificationImplementation(FedotTabPFNImplementation):
+    def __init__(self, params: Optional[OperationParameters] = None):
+        super().__init__(params)
+        self.model = TabPFNClassifier(**self.model_params)
+
+    def fit(self, input_data: InputData):
+        self.classes_ = np.unique(np.array(input_data.target))
+        return super().fit(input_data=input_data)
+
+
+class FedotTabPFNRegressionImplementation(FedotTabPFNImplementation):
+    def __init__(self, params: Optional[OperationParameters] = None):
+        super().__init__(params)
+        self.model = TabPFNRegressor(**self.model_params)
+
+
+class FedotAutoTabPFNClassificationImplementation(FedotTabPFNImplementation):
+    def __init__(self, params: Optional[OperationParameters] = None):
+        super().__init__(params)
+        self.model = AutoTabPFNClassifier(**self.model_params)
+
+    def fit(self, input_data: InputData):
+        self.classes_ = np.unique(np.array(input_data.target))
+        return super().fit(input_data=input_data)
+
+
+class FedotAutoTabPFNRegressionImplementation(FedotTabPFNImplementation):
+    def __init__(self, params: Optional[OperationParameters] = None):
+        super().__init__(params)
+        self.model = AutoTabPFNRegressor(**self.model_params)

fedot/core/operations/evaluation/tabpfn.py

@@ -0,0 +1,94 @@
+from typing import Optional
+
+from fedot.core.data.data import InputData, OutputData
+from fedot.core.operations.evaluation.evaluation_interfaces import EvaluationStrategy
+from fedot.core.operations.evaluation.operation_implementations.models.tabpfn import \
+    FedotTabPFNClassificationImplementation, FedotTabPFNRegressionImplementation, \
+    FedotAutoTabPFNClassificationImplementation, FedotAutoTabPFNRegressionImplementation
+from fedot.core.operations.operation_parameters import OperationParameters
+from fedot.core.repository.tasks import TaskTypesEnum
+from fedot.utilities.random import ImplementationRandomStateHandler
+
+
+class TabPFNStrategy(EvaluationStrategy):
+    _operations_by_types = {
+        'tabpfn': FedotTabPFNClassificationImplementation,
+        'tabpfnreg': FedotTabPFNRegressionImplementation,
+        'autotabpfn': FedotAutoTabPFNClassificationImplementation,
+        'autotabpfnreg': FedotAutoTabPFNRegressionImplementation,
+    }
+
+    def __init__(self, operation_type: str, params: Optional[OperationParameters] = None):
+        self.operation_impl = self._convert_to_operation(operation_type)
+        super().__init__(operation_type, params)
+        self.device = params.get('device', 'auto')
+        self.max_samples = params.get('max_samples', 1000)
+        self.max_features = params.get('max_features', 500)
+
+    def fit(self, train_data: InputData):
+        check_data_size(
+            data=train_data,
+            device=self.device,
+            max_samples=self.max_samples,
+            max_features=self.max_features,
+        )
+        if train_data.task.task_type == TaskTypesEnum.ts_forecasting:
+            raise ValueError('Time series forecasting not supported for TabPFN')
+
+        operation_implementation = self.operation_impl(self.params_for_fit)
+
+        with ImplementationRandomStateHandler(implementation=operation_implementation):
+            operation_implementation.fit(train_data)
+
+        return operation_implementation
+
+    def predict(self, trained_operation, predict_data: InputData) -> OutputData:
+        raise NotImplementedError()
+
+
+class TabPFNClassificationStrategy(TabPFNStrategy):
+    def __init__(self, operation_type: str, params: Optional[OperationParameters] = None):
+        super().__init__(operation_type, params)
+
+    def predict(self, trained_operation, predict_data: InputData) -> OutputData:
+        if self.output_mode == 'labels':
+            output = trained_operation.predict(predict_data)
+        elif self.output_mode in ['probs', 'full_probs', 'default']:
+            n_classes = len(trained_operation.classes_)
+            output = trained_operation.predict_proba(predict_data)
+            if n_classes < 2:
+                raise ValueError('Data set contain only 1 target class. Please reformat your data.')
+            elif (n_classes == 2 and self.output_mode != 'full_probs'
+                  and len(output.predict.shape) > 1):
+                output.predict = output.predict[:, 1]
+        else:
+            raise ValueError(f'Output model {self.output_mode} is not supported')
+
+        return output
+
+
+class TabPFNRegressionStrategy(TabPFNStrategy):
+    def __init__(self, operation_type: str, params: Optional[OperationParameters] = None):
+        super().__init__(operation_type, params)
+
+    def predict(self, trained_operation, predict_data: InputData) -> OutputData:
+        return trained_operation.predict(predict_data)
+
+
+def check_data_size(
+        data: InputData,
+        device: str = "auto",
+        max_samples: int = 1000,
+        max_features: int = 500,
+) -> bool:
+    if data.features.shape[0] > max_samples:
+        raise ValueError(
+            f"Input data has too many samples ({data.features.shape[0]}), "
+            f"maximum is {max_samples} for device '{device}'"
+        )
+    if data.features.shape[1] > max_features:
+        raise ValueError(
+            f"Input data has too many features ({data.features.shape[1]}), "
+            f"maximum is {max_features}"
+        )
+    return True

fedot/core/pipelines/tuning/search_space.py

@@ -930,6 +930,45 @@ def get_parameters_dict(self):
                     'hyperopt-dist': hp.choice,
                     'sampling-scope': [['euclidean', 'manhattan', 'cosine']],
                     'type': 'categorical'}
+            },
+            'tabpfn': {
+                'n_estimators': {
+                    'hyperopt-dist': hp.uniformint,
+                    'sampling-scope': [1, 10],
+                    'type': 'discrete'
+                },
+                'softmax_temperature': {
+                    'hyperopt-dist': hp.uniform,
+                    'sampling-scope': [0.0, 1.0],
+                    'type': 'continuous'
+                },
+                'balance_probabilities': {
+                    'hyperopt-dist': hp.choice,
+                    'sampling-scope': [[True, False]],
+                    'type': 'categorical'
+                },
+                'average_before_softmax': {
+                    'hyperopt-dist': hp.choice,
+                    'sampling-scope': [[True, False]],
+                    'type': 'categorical'
+                },
+            },
+            'tabpfnreg': {
+                'n_estimators': {
+                    'hyperopt-dist': hp.uniformint,
+                    'sampling-scope': [1, 10],
+                    'type': 'discrete'
+                },
+                'softmax_temperature': {
+                    'hyperopt-dist': hp.uniform,
+                    'sampling-scope': [0.0, 1.0],
+                    'type': 'continuous'
+                },
+                'average_before_softmax': {
+                    'hyperopt-dist': hp.choice,
+                    'sampling-scope': [[True, False]],
+                    'type': 'categorical'
+                },
             }
         }