feat: make @huggingface/transformers optional and improve error handling in EmbeddingService

Author: frontegg-david

libs/enclave-vm/package.json

@@ -42,9 +42,6 @@
     "astring": "1.9.0",
     "zod": "^4.1.13"
   },
-  "optionalDependencies": {
-    "@huggingface/transformers": "^3.0.0"
-  },
   "peerDependencies": {
     "@huggingface/transformers": "^3.0.0"
   },

libs/vectoriadb/package.json

@@ -37,8 +37,13 @@
       "default": "./dist/src/index.js"
     }
   },
-  "dependencies": {
-    "@huggingface/transformers": "^3.2.2"
+  "peerDependencies": {
+    "@huggingface/transformers": "^3.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@huggingface/transformers": {
+      "optional": true
+    }
   },
   "devDependencies": {
     "typescript": "^5.9.3"

libs/vectoriadb/src/embedding.service.ts

@@ -1,8 +1,15 @@
-import { pipeline } from '@huggingface/transformers';
-import { EmbeddingError } from './errors';
+import { EmbeddingError, ConfigurationError } from './errors';
+
+// Dynamic import type for @huggingface/transformers
+type PipelineFunction = typeof import('@huggingface/transformers').pipeline;
 
 /**
  * Service for generating embeddings using transformers.js
+ *
+ * NOTE: This service requires @huggingface/transformers to be installed.
+ * Install it with: npm install @huggingface/transformers
+ *
+ * For a zero-dependency alternative, use TFIDFEmbeddingService instead.
  */
 export class EmbeddingService {
   private pipeline: any = null;
@@ -17,6 +24,23 @@ export class EmbeddingService {
     this.cacheDir = cacheDir;
   }
 
+  /**
+   * Dynamically import @huggingface/transformers
+   * This allows the package to be optional - only loaded when actually used
+   */
+  private async loadTransformers(): Promise<PipelineFunction> {
+    try {
+      const transformers = await import('@huggingface/transformers');
+      return transformers.pipeline;
+    } catch (error) {
+      throw new ConfigurationError(
+        '@huggingface/transformers is not installed. ' +
+          'Install it with: npm install @huggingface/transformers\n' +
+          'Or use TFIDFVectoria/TFIDFEmbeddingService for a zero-dependency alternative.',
+      );
+    }
+  }
+
   /**
    * Initialize the embedding model
    */
@@ -36,8 +60,11 @@ export class EmbeddingService {
 
   private async _initialize(): Promise<void> {
     try {
+      // Dynamically load transformers
+      const pipelineFn = await this.loadTransformers();
+
       // Create feature extraction pipeline
-      this.pipeline = await pipeline('feature-extraction', this.modelName, {
+      this.pipeline = await pipelineFn('feature-extraction', this.modelName, {
         // Use local models directory to cache models
         cache_dir: this.cacheDir,
         // // Don't require progress bars in production
@@ -54,6 +81,9 @@ export class EmbeddingService {
       this.isInitialized = true;
     } catch (error) {
       this.initializationPromise = null;
+      if (error instanceof ConfigurationError) {
+        throw error;
+      }
       throw new EmbeddingError(
         `Failed to initialize embedding model: ${error instanceof Error ? error.message : String(error)}`,
         error instanceof Error ? error : undefined,

package.json

@@ -19,9 +19,6 @@
     "*.{json,css,scss,md,html,yml,yaml}": "prettier --write"
   },
   "private": true,
-  "dependencies": {
-    "@huggingface/transformers": "^3.2.2"
-  },
   "devDependencies": {
     "@eslint/js": "^9.8.0",
     "@mdx-js/mdx": "^3.1.1",