Allow various compat decompositions in confusables data (#1308)

roozbehp · web-flow · commit 403cc7762609 · 2026-02-20T14:11:06.000-08:00
- This fixes part of unicode-org/properties#460 - Also fix a data error for U+1F16D which included a TAB character.
diff --git a/unicodetools/data/security/dev/confusables.txt b/unicodetools/data/security/dev/confusables.txt
diff --git a/unicodetools/data/security/dev/confusablesSummary.txt b/unicodetools/data/security/dev/confusablesSummary.txt
diff --git a/unicodetools/data/security/dev/data/confusablesSummaryIdentifier.txt b/unicodetools/data/security/dev/data/confusablesSummaryIdentifier.txt
@@ -1,6 +1,6 @@
 # confusablesSummaryIdentifier.txt
-# Date: 2025-12-07, 22:02:22 GMT
-# © 2025 Unicode®, Inc.
+# Date: 2026-02-20, 09:37:08 GMT
+# © 2026 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
 # For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
@@ -18,7 +18,7 @@
 
 #	'	`	ʻ	ʼ	י	׳	’
 	(‎ ' ‎)	0027	 APOSTROPHE
-←	(‎ ` ‎)	0060	 GRAVE ACCENT	# →ˋ→→｀→→‘→
+←	(‎ ` ‎)	0060	 GRAVE ACCENT	# →｀→→‘→
 ←	(‎ ʻ ‎)	02BB	 MODIFIER LETTER TURNED COMMA	# →‘→
 ←	(‎ ʼ ‎)	02BC	 MODIFIER LETTER APOSTROPHE	# →′→
 ←	(‎ י ‎)	05D9	 HEBREW LETTER YOD
diff --git a/unicodetools/data/security/dev/data/source/confusables-source.txt b/unicodetools/data/security/dev/data/source/confusables-source.txt
@@ -3577,10 +3577,8 @@ FF39 ;  0059    # ( Ｙ → Y) FULLWIDTH LATIN CAPITAL LETTER Y → LATIN CAPITA
 FF3A ;  005A    # ( Ｚ → Z) FULLWIDTH LATIN CAPITAL LETTER Z → LATIN CAPITAL LETTER Z
 005A ;  0396
 FF3B ;  005B    # ( ［ → [) FULLWIDTH LEFT SQUARE BRACKET → LEFT SQUARE BRACKET
-FF3B ;  3014    # ( ［ → 〔) FULLWIDTH LEFT SQUARE BRACKET → LEFT TORTOISE SHELL BRACKET
 FF3C ;  005C    # ( ＼ → \) FULLWIDTH REVERSE SOLIDUS → REVERSE SOLIDUS
 FF3D ;  005D    # ( ］ → ]) FULLWIDTH RIGHT SQUARE BRACKET → RIGHT SQUARE BRACKET
-FF3D ;  3015    # ( ］ → 〕) FULLWIDTH RIGHT SQUARE BRACKET → RIGHT TORTOISE SHELL BRACKET
 FF3E ;  005E    # ( ＾ → ^) FULLWIDTH CIRCUMFLEX ACCENT → CIRCUMFLEX ACCENT
 FF3E ;  FE3F    # ( ＾ → ︿) FULLWIDTH CIRCUMFLEX ACCENT → PRESENTATION FORM FOR VERTICAL LEFT ANGLE BRACKET
 FF3F ;  005F    # ( ＿ → _) FULLWIDTH LOW LINE → LOW LINE
@@ -5424,7 +5422,7 @@ ABBB; 0473; V8_0; ꮻ => ѳ; CHEROKEE SMALL LETTER WI => CYRILLIC SMALL LETTER F
 1F10D ; ⓪	; V13_0 ; CIRCLED ZERO WITH SLASH
 1F10E ; 21BA ; V13_0 ; CIRCLED ANTICLOCKWISE ARROW
 1F10F; 0024 20E0	; V11_0 ; CIRCLED DOLLAR SIGN WITH OVERLAID BACKSLASH
-1F16D ; ㏄	20DD ; V11_0 ; CIRCLED CC
+1F16D ; ㏄ 20DD ; V11_0 ; CIRCLED CC
 1F16E ; C 20E0 	; V11_0 ; CIRCLED C WITH OVERLAID BACKSLASH
 # 1F16F ; 🚹	; V11_0 ; CIRCLED HUMAN FIGURE
 
diff --git a/unicodetools/data/security/dev/data/source/confusables-winFonts.txt b/unicodetools/data/security/dev/data/source/confusables-winFonts.txt
@@ -1583,10 +1583,6 @@
 
 3003 ;	2033	#* ( 〃 ~ ″ ) DITTO MARK ~ DOUBLE PRIME
 
-3014 ;	FF3B	#* ( 〔 ~ ［ ) LEFT TORTOISE SHELL BRACKET ~ FULLWIDTH LEFT SQUARE BRACKET
-
-3015 ;	FF3D	#* ( 〕 ~ ］ ) RIGHT TORTOISE SHELL BRACKET ~ FULLWIDTH RIGHT SQUARE BRACKET
-
 301C ;	FF5E	#* ( 〜 ~ ～ ) WAVE DASH ~ FULLWIDTH TILDE
 
 #3030 ;	FE4B	#* ( 〰 ~ ﹋ ) WAVY DASH ~ WAVY OVERLINE
diff --git a/unicodetools/data/security/dev/data/source/formatted-source.txt b/unicodetools/data/security/dev/data/source/formatted-source.txt
@@ -1,5 +1,5 @@
 # formatted-source.txt
-# Date: 2026-02-06, 21:41:43 GMT
+# Date: 2026-02-20, 09:37:07 GMT
 # © 2026 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
 # For terms of use and license, see https://www.unicode.org/terms_of_use.html
@@ -4734,10 +4734,6 @@
 
 3005 ;	206A4	# ( 々 ~ 𠚤 ) IDEOGRAPHIC ITERATION MARK ~ CJK UNIFIED IDEOGRAPH-206A4
 
-3014 ;	FF3B	#* ( 〔 ~ ［ ) LEFT TORTOISE SHELL BRACKET ~ FULLWIDTH LEFT SQUARE BRACKET
-
-3015 ;	FF3D	#* ( 〕 ~ ］ ) RIGHT TORTOISE SHELL BRACKET ~ FULLWIDTH RIGHT SQUARE BRACKET
-
 301C ;	FF5E	#* ( 〜 ~ ～ ) WAVE DASH ~ FULLWIDTH TILDE
 
 302C ;	309A	# ( 〬 ~ ゚ ) IDEOGRAPHIC DEPARTING TONE MARK ~ COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK
@@ -4785,7 +4781,7 @@
 
 312D ;	5E00	# ( ㄭ ~ 帀 ) BOPOMOFO LETTER IH ~ CJK UNIFIED IDEOGRAPH-5E00
 
-33C4 0009 20DD ;	1F16D	#* (  ~ 🅭 ) SQUARE CC, <CHARACTER TABULATION>, COMBINING ENCLOSING CIRCLE ~ CIRCLED CC
+33C4 20DD ;	1F16D	#* ( ㏄⃝ ~ 🅭 ) SQUARE CC, COMBINING ENCLOSING CIRCLE ~ CIRCLED CC
 
 3588 ;	439B	# ( 㖈 ~ 䎛 ) CJK UNIFIED IDEOGRAPH-3588 ~ CJK UNIFIED IDEOGRAPH-439B
 
diff --git a/unicodetools/data/security/dev/data/source/formatted-winFonts.txt b/unicodetools/data/security/dev/data/source/formatted-winFonts.txt
@@ -1,6 +1,6 @@
 # formatted-winFonts.txt
-# Date: 2025-11-13, 04:42:20 GMT
-# © 2025 Unicode®, Inc.
+# Date: 2026-02-20, 09:37:07 GMT
+# © 2026 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
 # For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
@@ -973,10 +973,6 @@
 
 3003 ;	2033	#* ( 〃 ~ ″ ) DITTO MARK ~ DOUBLE PRIME
 
-3014 ;	FF3B	#* ( 〔 ~ ［ ) LEFT TORTOISE SHELL BRACKET ~ FULLWIDTH LEFT SQUARE BRACKET
-
-3015 ;	FF3D	#* ( 〕 ~ ］ ) RIGHT TORTOISE SHELL BRACKET ~ FULLWIDTH RIGHT SQUARE BRACKET
-
 301C ;	FF5E	#* ( 〜 ~ ～ ) WAVE DASH ~ FULLWIDTH TILDE
 
 3078 ;	30D8	# ( へ ~ ヘ ) HIRAGANA LETTER HE ~ KATAKANA LETTER HE
diff --git a/unicodetools/src/main/java/org/unicode/text/UCD/GenerateConfusables.java b/unicodetools/src/main/java/org/unicode/text/UCD/GenerateConfusables.java
@@ -712,26 +712,33 @@ private static UnicodeSet getSkipNFKD() {
                 }
                 final int decompType = DEFAULT_UCD.getDecompositionType(cp);
                 final String nfc = Default.nfc().normalize(cp);
+                final String mapped = NFKD.normalize(cp);
                 if (decompType == UCD_Types.CANONICAL) {
                     nfcMap.put(cp, nfc);
                 }
                 if (decompType == UCD_Types.COMPAT_CIRCLE
                         || decompType == UCD_Types.COMPAT_SUPER
                         || decompType == UCD_Types.COMPAT_SUB
                         || decompType == UCD_Types.COMPAT_VERTICAL
-                        || decompType == UCD_Types.COMPAT_SMALL
-                        || decompType == UCD_Types.COMPAT_SQUARE
                         || decompType == UCD_Types.COMPAT_FRACTION
-                        || decompType == UCD_Types.COMPAT_NARROW
-                        || decompType == UCD_Types.COMPAT_WIDE
-                        || decompType == UCD_Types.COMPAT_WIDE
+                        || (0x3300 <= cp && cp <= 0x3357) // Skip squared Katakana words
+                        || cp == 0x337F // Skip SQUARE CORPORATION
+                        || cp == 0xFE58 // Skip SMALL EM DASH since it would merge two classes
+                        // Skip FULLWIDTH HYPHEN-MINUS since it would merge two classes
+                        || cp == 0xFF0D
+                        || (0x1F130 <= cp && cp <= 0x1F14F) // Skip squared Latin letters
+                        || (0x1F200 <= cp && cp <= 0x1F23B) // Skip Enclosed Ideographic Supplement
+                        // Skip square compatibility characters that have a power of 2 or 3.
+                        // This needs to be fixed in the future so we can have confusability
+                        // with the decomposition using superscript 2 and 3.
+                        || (decompType == UCD_Types.COMPAT_SQUARE
+                                && (mapped.contains("2") || mapped.contains("3")))
                         || cp == '﬩'
                         || cp == '︒') {
                     _skipNFKD.add(cp);
                     continue;
                 }
                 final String source = UTF16.valueOf(cp);
-                final String mapped = NFKD.normalize(cp);
                 String kmapped = ModifiedNFKD.normalize(source);
                 if (!kmapped.equals(source) && !kmapped.equals(nfc)) {
                     if (kmapped.startsWith(" ") || kmapped.startsWith("\u0640")) {
