Merge pull request #830 from qiyanjun/fix/issue-triage-quick-fixes

Fix bert-attack slowness, CLARE <SPLIT>-token corruption, and WordSwapInflections POS mismatch

Author: qiyanjun

tests/test_attacked_text.py

@@ -48,6 +48,16 @@ def attacked_text_pair():
     return textattack.shared.AttackedText(raw_text_pair)
 
 
+raw_text_pair_split_word_collision = collections.OrderedDict(
+    [("premise", "hello there"), ("hypothesis", "I am fine.")]
+)
+
+
+@pytest.fixture
+def attacked_text_pair_split_word_collision():
+    return textattack.shared.AttackedText(raw_text_pair_split_word_collision)
+
+
 class TestAttackedText:
     def test_words(self, attacked_text, pokemon_attacked_text):
         # fmt: off
@@ -144,6 +154,18 @@ def test_word_insertion(self, attacked_text):
             == "A person walks a long way up stairs into a room and sees beer poured from a keg and people on the couch talking."
         )
 
+    def test_pair_word_replacement_split_token_collision(
+        self, attacked_text_pair_split_word_collision
+    ):
+        # Regression test for https://github.com/QData/TextAttack/issues/631:
+        # replacing the first word of the second segment ("I") used to match
+        # inside the "<SPLIT>" join token instead of the real word, since "I"
+        # is a character substring of "<SPLIT>".
+        new_text = attacked_text_pair_split_word_collision.replace_word_at_index(
+            2, "You"
+        )
+        assert new_text.text == "hello there\nYou am fine."
+
     def test_pair_word_insertion(self, attacked_text_pair):
         new_text = attacked_text_pair.insert_text_after_word_index(3, "old decrepit")
         assert new_text.text == (
@@ -204,3 +226,20 @@ def test_hyphen_apostrophe_words(self, hyphenated_text):
         ]
 
     # TODO: test align_words_with_tokens
+
+
+def test_pos_of_word_index_after_ner_of_word_index():
+    # Regression test for a CI failure (KeyError: 'upos', later reproduced
+    # locally as an empty flair annotation layer): `flair_tag` cached a
+    # single `SequenceTagger` in a module-global slot keyed by nothing, so
+    # once anything called `ner_of_word_index` (loading the "ner" tagger)
+    # in the same process, every later `pos_of_word_index` call silently
+    # reused that NER tagger instead of loading "upos-fast", producing
+    # wrong or empty POS labels. Calling NER before POS on a fresh
+    # AttackedText reproduces the exact ordering that broke in CI.
+    ner_text = textattack.shared.AttackedText("I am in Dallas.")
+    ner_text.ner_of_word_index(3)
+
+    pos_text = textattack.shared.AttackedText("The cats were running quickly.")
+    assert pos_text.pos_of_word_index(1) == "NOUN"
+    assert pos_text.pos_of_word_index(2) == "VERB"

tests/test_transformations.py

@@ -149,6 +149,62 @@ def test_chinese_word_swap_hownet():
     assert augmented_s or s in augmented_text_list
 
 
+def test_zip_flair_result_annotation_layer_key_independent():
+    # zip_flair_result used to hardcode the annotation layer's key name
+    # ("upos"), the same failure class as #727 (which hardcoded "pos" and
+    # broke when flair started using "upos"). The actual CI failure this was
+    # investigating turned out to be caused by a different bug (flair_tag's
+    # single-slot tagger cache getting poisoned by a different tag_type, see
+    # test_pos_of_word_index_after_ner_of_word_index in test_attacked_text.py
+    # for the real regression test), but hardcoding the annotation key name
+    # is still fragile on its own, so keep this as a defense-in-depth check
+    # that zip_flair_result doesn't assume one specific key name.
+    from flair.data import Sentence
+
+    from textattack.shared.utils import zip_flair_result
+
+    sentence = Sentence("cats run")
+    for token, tag in zip(sentence.tokens, ["NOUN", "VERB"]):
+        token.add_label("pos", tag)
+
+    word_list, pos_list = zip_flair_result(sentence, tag_type="upos-fast")
+    assert word_list == ["cats", "run"]
+    assert pos_list == ["NOUN", "VERB"]
+
+
+def test_word_swap_inflections_pos_matching():
+    # Regression test for https://github.com/QData/TextAttack/issues/713 and
+    # https://github.com/QData/TextAttack/issues/727: AttackedText.pos_of_word_index
+    # returns flair's upos-fast tags (e.g. "NOUN", "VERB"), so
+    # WordSwapInflections's POS-to-lemma mapping must have entries for those
+    # tags, not just legacy fine-grained en-ptb tags (e.g. "NN", "VBD"), or it
+    # silently returns zero candidates for ordinary words.
+    import textattack
+    from textattack.transformations.word_swaps import WordSwapInflections
+
+    transformation = WordSwapInflections()
+    attacked_text = textattack.shared.AttackedText("The cats were running quickly.")
+
+    # Confirm the tagger is actually giving us the upos-fast tag, not a
+    # legacy en-ptb one, so this test exercises the real mismatch and isn't
+    # trivially passing for the wrong reason.
+    cats_index = attacked_text.words.index("cats")
+    cats_pos = attacked_text.pos_of_word_index(cats_index)
+    assert cats_pos == "NOUN"
+    # Before the fix, "NOUN" wasn't a key in the mapping (only "NN" was), so
+    # this lookup missed and _get_replacement_words returned [] for every
+    # ordinary noun.
+    noun_candidates = transformation._get_replacement_words("cats", cats_pos)
+    assert "cat" in noun_candidates
+
+    were_index = attacked_text.words.index("were")
+    were_pos = attacked_text.pos_of_word_index(were_index)
+    assert were_pos == "VERB"
+    # Same failure mode as above, for verbs ("VERB" vs. the legacy "VBD").
+    verb_candidates = transformation._get_replacement_words("were", were_pos)
+    assert "was" in verb_candidates
+
+
 def test_chinese_word_swap_masked():
     from textattack.augmentation import Augmenter
     from textattack.transformations.word_swaps.chn_transformations import (

textattack/attack_recipes/bert_attack_li_2020.py

@@ -4,10 +4,11 @@
 
 (BERT-Attack: Adversarial Attack Against BERT Using BERT)
 
-.. warning::
-    This attack is super slow
-    (see https://github.com/QData/TextAttack/issues/586)
-    Consider using smaller values for "max_candidates".
+.. note::
+    Candidate size ``max_candidates`` defaults to 8 (down from the paper's 48)
+    to keep runtime tractable; see https://github.com/QData/TextAttack/issues/586.
+    Pass a larger ``max_candidates`` to ``WordSwapMaskedLM`` for closer
+    fidelity to the original paper at the cost of much slower attacks.
 
 """
 
@@ -39,7 +40,11 @@ class BERTAttackLi2020(AttackRecipe):
     def build(model_wrapper):
         # [from correspondence with the author]
         # Candidate size K is set to 48 for all data-sets.
-        transformation = WordSwapMaskedLM(method="bert-attack", max_candidates=48)
+        # In practice K=48 makes the number of masked-LM candidate combinations
+        # explode for multi-subword tokens (48**n), causing multi-hour runtimes;
+        # see https://github.com/QData/TextAttack/issues/586. K=8 keeps the attack
+        # tractable with a small accuracy/ASR tradeoff.
+        transformation = WordSwapMaskedLM(method="bert-attack", max_candidates=8)
         #
         # Don't modify the same word twice or stopwords.
         #

textattack/shared/attacked_text.py

@@ -414,7 +414,26 @@ def generate_new_attacked_text(self, new_words: Iterable[str]) -> AttackedText:
         # Create the new attacked text by swapping out words from the original
         # text with a sequence of 0+ words in the new text.
         for i, (input_word, adv_word_seq) in enumerate(zip(self.words, new_words)):
-            word_start = original_text.index(input_word)
+            # `input_word` can be a character substring of `SPLIT_TOKEN` itself
+            # (e.g. "I" is a substring of "<SPLIT>"), which makes a naive
+            # `.index(input_word)` match inside the token instead of the real
+            # word when it directly follows a split. Route around the split
+            # token's span in that case. See
+            # https://github.com/QData/TextAttack/issues/631
+            if (
+                input_word in AttackedText.SPLIT_TOKEN
+                and AttackedText.SPLIT_TOKEN in original_text
+            ):
+                split_start = original_text.index(AttackedText.SPLIT_TOKEN)
+                split_end = split_start + len(AttackedText.SPLIT_TOKEN)
+                if split_start <= original_text.index(input_word) < split_end:
+                    word_start = original_text.replace(
+                        AttackedText.SPLIT_TOKEN, ""
+                    ).index(input_word) + len(AttackedText.SPLIT_TOKEN)
+                else:
+                    word_start = original_text.index(input_word)
+            else:
+                word_start = original_text.index(input_word)
             word_end = word_start + len(input_word)
             perturbed_text += original_text[:word_start]
             original_text = original_text[word_end:]

textattack/shared/utils/strings.py

@@ -215,17 +215,21 @@ def color_text(text, color=None, method=None):
         return "[[" + text + "]]"
 
 
-_flair_pos_tagger = None
+_flair_taggers = {}
 
 
 def flair_tag(sentence, tag_type="upos-fast"):
     """Tags a `Sentence` object using `flair` part-of-speech tagger."""
-    global _flair_pos_tagger
-    if not _flair_pos_tagger:
+    # Cache one tagger per `tag_type`: this function is used for both POS
+    # ("upos-fast") and NER ("ner", "flair/ner-french", ...) tagging, and a
+    # single shared slot used to silently reuse whichever tagger loaded
+    # first for every later call regardless of the requested `tag_type`,
+    # corrupting results for the other tagging purpose.
+    if tag_type not in _flair_taggers:
         from flair.models import SequenceTagger
 
-        _flair_pos_tagger = SequenceTagger.load(tag_type)
-    _flair_pos_tagger.predict(sentence, force_token_predictions=True)
+        _flair_taggers[tag_type] = SequenceTagger.load(tag_type)
+    _flair_taggers[tag_type].predict(sentence, force_token_predictions=True)
 
 
 def zip_flair_result(pred, tag_type="upos-fast"):
@@ -242,7 +246,18 @@ def zip_flair_result(pred, tag_type="upos-fast"):
     for token in tokens:
         word_list.append(token.text)
         if "pos" in tag_type:
-            pos_list.append(token.annotation_layers["upos"][0]._value)
+            # The annotation layer's key (e.g. "pos", "upos") is set by
+            # whichever flair tagger/version produced these labels, and has
+            # changed across flair releases (see #727). Read the key that's
+            # actually present on the token instead of hardcoding one.
+            pos_label_type = next(iter(token.annotation_layers), None)
+            if pos_label_type is None:
+                raise ValueError(
+                    f"No part-of-speech label found for token {token.text!r}; "
+                    f"the flair `Sentence` may have been tagged with a "
+                    f"tagger that doesn't match tag_type={tag_type!r}."
+                )
+            pos_list.append(token.annotation_layers[pos_label_type][0]._value)
         elif tag_type == "ner":
             pos_list.append(token.get_label("ner"))
 

textattack/transformations/word_swaps/word_swap_inflections.py

@@ -26,9 +26,25 @@ class WordSwapInflections(WordSwap):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        # fine-grained en-ptb POS to universal POS mapping
-        # (mapping info: https://github.com/slavpetrov/universal-pos-tags)
+        # `AttackedText.pos_of_word_index` now returns flair's "upos-fast" tags
+        # directly (Universal Dependencies UPOS, e.g. "NOUN", "VERB", "PROPN"),
+        # not the old fine-grained en-ptb tags this dict's keys used to assume
+        # (see https://github.com/QData/TextAttack/issues/713,
+        # https://github.com/QData/TextAttack/issues/727). Without the plain
+        # "NOUN"/"VERB"/"ADJ" entries below, this lookup always misses and the
+        # transformation silently returns no candidates for ordinary words.
+        # Kept the legacy en-ptb keys too in case a caller wires up a
+        # PTB-style tagger.
+        # (mapping info: https://universaldependencies.org/u/pos/,
+        # https://github.com/slavpetrov/universal-pos-tags)
         self._enptb_to_universal = {
+            # current flair "upos-fast" (Universal Dependencies) tags
+            "NOUN": "NOUN",
+            "PROPN": "NOUN",
+            "VERB": "VERB",
+            "AUX": "VERB",
+            "ADJ": "ADJ",
+            # legacy fine-grained en-ptb tags
             "JJRJR": "ADJ",
             "VBN": "VERB",
             "VBP": "VERB",