Merge pull request #99 from amd/alex_ext_pkg

Entry points

Author: alexandraBara

docs/node-scraper-external/README.md

@@ -1,7 +1,12 @@
 # node-scraper external plugins (example)
 
 This directory lives at **`/docs/node-scraper-external`** in the `node-scraper` repo and contains
-an example external plugin package you can install in editable mode.
+an example external plugin package that demonstrates how to create plugins for node-scraper.
+
+## Overview
+
+External plugins are discovered by node-scraper via **Python entry points**. This allows plugins
+to be distributed as separate packages and automatically discovered when installed.
 
 ## Installation
 
@@ -12,44 +17,126 @@ cd ~/node-scraper
 source venv/bin/activate
 pip install -e ./docs/node-scraper-external
 ```
-You should see `ext-nodescraper-plugins` installed in editable mode.
 
+This installs `ext-nodescraper-plugins` in editable mode and registers the plugin entry points.
 
-## Verify the external package is importable
+## Verify Plugin Discovery
+
+Check that node-scraper discovered the external plugin:
 
 ```bash
-python - <<'PY'
-import ext_nodescraper_plugins
-print("ext_nodescraper_plugins loaded from:", ext_nodescraper_plugins.__file__)
-PY
+node-scraper run-plugins -h
 ```
 
-## Run external plugins
+You should see `SamplePlugin` listed alongside built-in plugins.
 
-Confirm the CLI sees your external plugin(s):
+## Run the Example Plugin
 
 ```bash
-node-scraper run-plugins -h
 node-scraper run-plugins SamplePlugin
 ```
 
-## Add your own plugins
+## How It Works
+
+### Entry Points
 
-Add new modules under the **`ext_nodescraper_plugins/`** package. Example layout:
+Plugins are registered in `pyproject.toml` using entry points:
+
+```toml
+[project.entry-points."nodescraper.plugins"]
+SamplePlugin = "ext_nodescraper_plugins.sample.sample_plugin:SamplePlugin"
+```
+
+When you install the package, Python registers these entry points in the package metadata.
+Node-scraper automatically discovers and loads plugins from the `nodescraper.plugins` entry point group.
+
+### Plugin Structure
 
 ```
 /docs/node-scraper-external
-├─ pyproject.toml
-└─ ext_nodescraper_plugins/
-   └─ sample/
+├─ pyproject.toml              # Package metadata + entry points
+└─ ext_nodescraper_plugins/    # Plugin package
+   └─ sample/                   # Plugin module
       ├─ __init__.py
-      └─ sample_plugin.py
+      ├─ sample_plugin.py       # Plugin class
+      ├─ sample_collector.py    # Data collector
+      ├─ sample_analyzer.py     # Data analyzer
+      └─ sample_data.py         # Data model
+```
+
+## Creating Your Own External Plugins
+
+### Step 1: Create Package Structure
+
+```bash
+mkdir my-plugin-package
+cd my-plugin-package
+mkdir -p ext_nodescraper_plugins/my_plugin
 ```
 
+### Step 2: Create pyproject.toml
+
+```toml
+[project]
+name = "my-plugin-package"
+version = "0.1.0"
+requires-python = ">=3.10"
+dependencies = ["amd-node-scraper"]
+
+[project.entry-points."nodescraper.plugins"]
+MyPlugin = "ext_nodescraper_plugins.my_plugin:MyPlugin"
+
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
 ```
 
-Re-install (editable mode picks up code changes automatically, but if you add new files you may
-need to re-run):
+### Step 3: Implement Your Plugin
+
+Create `ext_nodescraper_plugins/my_plugin/__init__.py`:
+
+```python
+from nodescraper.base import InBandDataPlugin, InBandDataCollector
+from pydantic import BaseModel
+
+class MyDataModel(BaseModel):
+    """Your data model"""
+    data: dict
+
+class MyCollector(InBandDataCollector[MyDataModel, None]):
+    """Your data collector"""
+    DATA_MODEL = MyDataModel
+
+    def collect_data(self, args=None):
+        # Collection logic
+        return MyDataModel(data={})
+
+class MyPlugin(InBandDataPlugin[MyDataModel, None, None]):
+    """Your plugin"""
+    DATA_MODEL = MyDataModel
+    COLLECTOR = MyCollector
+```
+
+### Step 4: Install and Test
+
 ```bash
 pip install -e .
+node-scraper run-plugins -h  # Should show MyPlugin
+node-scraper run-plugins MyPlugin
 ```
+
+## Adding More Plugins to This Package
+
+To add additional plugins to this example package:
+
+1. **Create a new module** under `ext_nodescraper_plugins/`
+2. **Register the entry point** in `pyproject.toml`:
+   ```toml
+   [project.entry-points."nodescraper.plugins"]
+   SamplePlugin = "ext_nodescraper_plugins.sample.sample_plugin:SamplePlugin"
+   AnotherPlugin = "ext_nodescraper_plugins.another:AnotherPlugin"
+   ```
+3. **Reinstall** to register the new entry point:
+   ```bash
+   pip install -e . --force-reinstall --no-deps
+   ```

docs/node-scraper-external/pyproject.toml

@@ -4,6 +4,9 @@ version = "0.1.0"
 requires-python = ">=3.10"
 dependencies = ["node-scraper"]
 
+[project.entry-points."nodescraper.plugins"]
+SamplePlugin = "ext_nodescraper_plugins.sample.sample_plugin:SamplePlugin"
+
 [build-system]
 requires = ["setuptools", "wheel"]
 build-backend = "setuptools.build_meta"

nodescraper/cli/cli.py

@@ -54,13 +54,6 @@
 from nodescraper.pluginexecutor import PluginExecutor
 from nodescraper.pluginregistry import PluginRegistry
 
-try:
-    import ext_nodescraper_plugins as ext_pkg
-
-    extra_pkgs = [ext_pkg]
-except ImportError:
-    extra_pkgs = []
-
 
 def build_parser(
     plugin_reg: PluginRegistry,
@@ -376,7 +369,7 @@ def main(arg_input: Optional[list[str]] = None):
     if arg_input is None:
         arg_input = sys.argv[1:]
 
-    plugin_reg = PluginRegistry(plugin_pkg=extra_pkgs)
+    plugin_reg = PluginRegistry()
 
     config_reg = ConfigRegistry()
     parser, plugin_subparser_map = build_parser(plugin_reg, config_reg)

nodescraper/pluginregistry.py

@@ -24,6 +24,7 @@
 #
 ###############################################################################
 import importlib
+import importlib.metadata
 import inspect
 import pkgutil
 import types
@@ -45,12 +46,14 @@ def __init__(
         self,
         plugin_pkg: Optional[list[types.ModuleType]] = None,
         load_internal_plugins: bool = True,
+        load_entry_point_plugins: bool = True,
     ) -> None:
         """Initialize the PluginRegistry with optional plugin packages.
 
         Args:
             plugin_pkg (Optional[list[types.ModuleType]], optional): The module to search for plugins in. Defaults to None.
             load_internal_plugins (bool, optional): Whether internal plugin should be loaded. Defaults to True.
+            load_entry_point_plugins (bool, optional): Whether to load plugins from entry points. Defaults to True.
         """
         if load_internal_plugins:
             self.plugin_pkg = [internal_plugins, internal_connections, internal_collators]
@@ -70,6 +73,10 @@ def __init__(
             PluginResultCollator, self.plugin_pkg
         )
 
+        if load_entry_point_plugins:
+            entry_point_plugins = self.load_plugins_from_entry_points()
+            self.plugins.update(entry_point_plugins)
+
     @staticmethod
     def load_plugins(
         base_class: type,
@@ -104,3 +111,42 @@ def _recurse_pkg(pkg: types.ModuleType, base_class: type) -> None:
         for pkg in search_modules:
             _recurse_pkg(pkg, base_class)
         return registry
+
+    @staticmethod
+    def load_plugins_from_entry_points() -> dict[str, type]:
+        """Load plugins registered via entry points.
+
+        Returns:
+            dict[str, type]: A dictionary mapping plugin names to their classes.
+        """
+        plugins = {}
+
+        try:
+            # Python 3.10+ supports group parameter
+            try:
+                eps = importlib.metadata.entry_points(group="nodescraper.plugins")  # type: ignore[call-arg]
+            except TypeError:
+                # Python 3.9 - entry_points() returns dict-like object
+                all_eps = importlib.metadata.entry_points()  # type: ignore[assignment]
+                eps = all_eps.get("nodescraper.plugins", [])  # type: ignore[assignment, attr-defined]
+
+            for entry_point in eps:
+                try:
+                    plugin_class = entry_point.load()  # type: ignore[attr-defined]
+
+                    if (
+                        inspect.isclass(plugin_class)
+                        and issubclass(plugin_class, PluginInterface)
+                        and not inspect.isabstract(plugin_class)
+                    ):
+                        if hasattr(plugin_class, "is_valid") and not plugin_class.is_valid():
+                            continue
+
+                        plugins[plugin_class.__name__] = plugin_class
+                except Exception:
+                    pass
+
+        except Exception:
+            pass
+
+        return plugins

pyproject.toml

@@ -12,9 +12,10 @@ classifiers = ["Topic :: Software Development"]
 
 dependencies = [
     "pydantic>=2.8.2",
-    "paramiko~=3.5.1",
+    "paramiko>=3.2.0,<4.0.0",
     "requests",
-    "pytz"
+    "pytz",
+    "urllib3>=1.26.15,<2.0.0"
 ]
 
 [project.optional-dependencies]