Relax version requirement for stanfordnlp to include v0.2.0. (#15)

* For compatibility with native Spacy language classes allow passing of empty text strings. This will produce 0-length docs, rather than raising an exception.

* Increment minor version.

* Relax version requirement for stanfordnlp to include new 0.2.0.

* Make tests pass with stanfordnlp 0.2.0. Some POS tag predictions have flipped in ambiguous cases (e.g. DET/PRON, VERB/AUX).

Author: buhrmann

requirements.txt

@@ -1,4 +1,4 @@
 spacy>=2.1.0
-stanfordnlp>=0.1.0,<0.2.0
+stanfordnlp>=0.1.0,<0.2.1
 # Development dependencies
 pytest>=4.0.0,<5.0.0

spacy_stanfordnlp/about.py

@@ -1,5 +1,5 @@
 __title__ = "spacy-stanfordnlp"
-__version__ = "0.1.1"
+__version__ = "0.1.2"
 __summary__ = "Use the latest StanfordNLP research models directly in spaCy"
 __uri__ = "https://explosion.ai"
 __author__ = "Ines Montani"

tests/test_language.py

@@ -16,17 +16,32 @@ def lang():
     return "en"
 
 
+def tags_equal(act, exp):
+    """Check if each actual tag in act is equal to one or more expected tags in exp."""
+    return all(a == e if isinstance(e, str) else a in e for a, e in zip(act, exp))
+
+
 def test_spacy_stanfordnlp(lang, models_dir):
-    snlp = stanfordnlp.Pipeline(lang=lang, models_dir=models_dir)
+    try:
+        snlp = stanfordnlp.Pipeline(lang=lang, models_dir=models_dir)
+    except:
+        snlp = stanfordnlp.Pipeline(lang=lang)
     nlp = StanfordNLPLanguage(snlp)
     assert nlp.lang == "stanfordnlp_" + lang
 
     doc = nlp("Hello world! This is a test.")
 
+    # Expected POS tags. Note: Different versions of stanfordnlp result in different POS tags.
+    # In particular, "this" can be a determiner or pronoun, their distinction is pretty vague in
+    # general. And "is" in "This is a test" can be interpreted either as simply a verb or as an
+    # auxiliary (linking) verb. Neither interpretation is necessarily more or less correct.
     # fmt: off
+    pos_exp = ["INTJ", "NOUN", "PUNCT", ("DET", "PRON"), ("VERB", "AUX"), "DET", "NOUN", "PUNCT"]
+
     assert [t.text for t in doc] == ["Hello", "world", "!", "This", "is", "a", "test", "."]
     assert [t.lemma_ for t in doc] == ["hello", "world", "!", "this", "be", "a", "test", "."]
-    assert [t.pos_ for t in doc] == ["INTJ", "NOUN", "PUNCT", "DET", "VERB", "DET", "NOUN", "PUNCT"]
+    assert tags_equal([t.pos_ for t in doc], pos_exp)
+
     assert [t.tag_ for t in doc] == ["UH", "NN", ".", "DT", "VBZ", "DT", "NN", '.']
     assert [t.dep_ for t in doc] == ["root", "vocative", "punct", "nsubj", "cop", "det", "root", "punct"]
     assert [t.is_sent_start for t in doc] == [True, None, None, True, None, None, None, None]
@@ -41,7 +56,7 @@ def test_spacy_stanfordnlp(lang, models_dir):
     assert docs[0].text == "Hello world"
     assert [t.pos_ for t in docs[0]] == ["INTJ", "NOUN"]
     assert docs[1].text == "This is a test"
-    assert [t.pos_ for t in docs[1]] == ["DET", "VERB", "DET", "NOUN"]
+    assert tags_equal([t.pos_ for t in docs[1]], pos_exp[3:-1])
 
 
 def test_get_defaults():