fix: preserve explicit spaCy model behavior

Author: sidmohan0

.github/workflows/ci.yml

@@ -60,13 +60,13 @@ jobs:
         if: matrix.install-profile == 'nlp'
         run: |
           pip install -e ".[test,cli,nlp]" -r requirements-test.txt
-          python -m spacy download en_core_web_sm
+          python -m spacy download en_core_web_lg
 
       - name: Install dependencies (nlp-advanced)
         if: matrix.install-profile == 'nlp-advanced'
         run: |
           pip install -e ".[test,cli,nlp,nlp-advanced]" -r requirements-test.txt
-          python -m spacy download en_core_web_sm
+          python -m spacy download en_core_web_lg
           datafog download-model urchade/gliner_multi_pii-v1 --engine gliner
 
       - name: Run tests (core)

.github/workflows/release.yml

@@ -128,7 +128,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install -e ".[all,test]"
           pip install -r requirements-test.txt
-          pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1.tar.gz
+          python -m spacy download en_core_web_lg
           datafog download-model urchade/gliner_multi_pii-v1 --engine gliner
 
       - name: Run tests with segfault protection

datafog/client.py

@@ -181,7 +181,7 @@ def download_model(
     Download a model for specified engine.
 
     Examples:
-        spaCy: datafog download-model en_core_web_sm --engine spacy
+        spaCy: datafog download-model en_core_web_lg --engine spacy
         GLiNER: datafog download-model urchade/gliner_multi_pii-v1 --engine gliner
     """
     if engine == "spacy":

datafog/models/spacy_nlp.py

@@ -13,7 +13,7 @@
 
 from .annotator import AnnotationResult, AnnotatorRequest
 
-DEFAULT_SPACY_MODEL = "en_core_web_sm"
+DEFAULT_SPACY_MODEL = "en_core_web_lg"
 
 
 class SpacyAnnotator:
@@ -33,7 +33,7 @@ def load_model(self):
             self.nlp = spacy.load(self.model_name)
         except OSError as exc:
             raise ImportError(
-                f"spaCy model '{self.model_name}' is not installed. "
+                f"spaCy model {self.model_name!r} is not installed. "
                 f"Download it explicitly with: datafog download-model {self.model_name} --engine spacy"
             ) from exc
 
@@ -83,7 +83,7 @@ def list_entities(model_name: str = DEFAULT_SPACY_MODEL) -> List[str]:
             nlp = spacy.load(model_name)
         except OSError as exc:
             raise ImportError(
-                f"spaCy model '{model_name}' is not installed. "
+                f"spaCy model {model_name!r} is not installed. "
                 f"Download it explicitly with: datafog download-model {model_name} --engine spacy"
             ) from exc
         return [ent for ent in nlp.pipe_labels["ner"]]

datafog/processing/image_processing/donut_processor.py

@@ -110,7 +110,7 @@ async def extract_text_from_image(self, image: "Image.Image") -> str:
                 )
             except OSError as exc:
                 raise RuntimeError(
-                    f"Donut model '{self.model_path}' is not available locally. "
+                    f"Donut model {self.model_path!r} is not available locally. "
                     "Download it explicitly before using Donut OCR, or pass a local "
                     "model path."
                 ) from exc

datafog/processing/spark_processing/pyspark_udfs.py

@@ -11,7 +11,7 @@
 
 PII_ANNOTATION_LABELS = ["DATE_TIME", "LOC", "NRP", "ORG", "PER"]
 MAXIMAL_STRING_SIZE = 1000000
-DEFAULT_SPACY_MODEL = "en_core_web_sm"
+DEFAULT_SPACY_MODEL = "en_core_web_lg"
 
 
 def pii_annotator(text: str, broadcasted_nlp) -> list[list[str]]:

datafog/processing/text_processing/gliner_annotator.py

@@ -87,7 +87,7 @@ def create(
         except Exception as e:
             logging.error(f"Failed to load GLiNER model {model_name}: {str(e)}")
             raise RuntimeError(
-                f"GLiNER model '{model_name}' is not available locally. "
+                f"GLiNER model {model_name!r} is not available locally. "
                 "Download it explicitly with: "
                 f"datafog download-model {model_name} --engine gliner"
             ) from e

datafog/processing/text_processing/spacy_pii_annotator.py

@@ -24,7 +24,7 @@
     "WORK_OF_ART",
 ]
 MAXIMAL_STRING_SIZE = 1000000
-DEFAULT_SPACY_MODEL = "en_core_web_sm"
+DEFAULT_SPACY_MODEL = "en_core_web_lg"
 
 
 class SpacyPIIAnnotator(BaseModel):
@@ -47,7 +47,7 @@ def create(cls, model_name: str = DEFAULT_SPACY_MODEL) -> "SpacyPIIAnnotator":
             nlp = spacy.load(model_name)
         except OSError as exc:
             raise ImportError(
-                f"spaCy model '{model_name}' is not installed. "
+                f"spaCy model {model_name!r} is not installed. "
                 f"Download it explicitly with: datafog download-model {model_name} --engine spacy"
             ) from exc
 

tests/test_runtime_dependency_safety.py

@@ -33,9 +33,7 @@ def load(_model_name):
 
     monkeypatch.setitem(sys.modules, "spacy", FakeSpacy())
 
-    from datafog.processing.text_processing.spacy_pii_annotator import (
-        SpacyPIIAnnotator,
-    )
+    from datafog.processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator
 
     with pytest.raises(ImportError, match="Download it explicitly"):
         SpacyPIIAnnotator.create()