common: split utf8_check() into permissive and strict variants

Changelog-None

Author: Andezion

common/bolt11.c

@@ -212,7 +212,7 @@ static const char *decode_d(struct bolt11 *b11,
 		return err;
 
 	*have_d = true;
-	b11->description = utf8_str(b11, take(desc), tal_bytelen(desc));
+	b11->description = utf8_str_text(b11, take(desc), tal_bytelen(desc));
 	if (b11->description)
 		return NULL;
 

common/bolt12_proof.c

@@ -157,7 +157,7 @@ struct tlv_payer_proof *make_unsigned_proof_(const tal_t *ctx,
 	if (note) {
 		/* Not nul-terminated! */
 		pptlv->proof_note = tal_dup_arr(pptlv, utf8, note, strlen(note), 0);
-		assert(utf8_check(pptlv->proof_note, tal_bytelen(pptlv->proof_note)));
+		assert(utf8_check_text(pptlv->proof_note, tal_bytelen(pptlv->proof_note)));
 	}
 
 	/* Make sure pptlv->fields correctly reflects values */

common/json_param.c

@@ -463,7 +463,7 @@ struct command_result *param_escaped_utf8_string(struct command *cmd,
 	if (ret)
 		return ret;
 
-	if (!utf8_check(*str, strlen(*str)))
+	if (!utf8_check_text(*str, strlen(*str)))
 		return command_fail_badparam(cmd, name, buffer, tok,
 					     "should not contain control, format, private-use or unassigned Unicode characters");
 	return NULL;
@@ -486,7 +486,7 @@ struct command_result *param_utf8_string(struct command *cmd, const char *name,
 	if (ret)
 		return ret;
 
-	if (!utf8_check(*str, strlen(*str)))
+	if (!utf8_check_text(*str, strlen(*str)))
 		return command_fail_badparam(cmd, name, buffer, tok,
 					     "should not contain control, format, private-use or unassigned Unicode characters");
 	return NULL;

common/json_param.h

@@ -193,7 +193,9 @@ struct command_result *param_escaped_string(struct command *cmd,
 					    const jsmntok_t *tok,
 					    const char **str);
 
-
+/* Extract an escaped string (and unescape it), and reject it if it
+ * contains characters banned from protocol text fields - see
+ * param_utf8_string() below. */
 struct command_result *param_escaped_utf8_string(struct command *cmd,
 						  const char *name,
 						  const char *buffer,

common/test/run-utils-utf8_check.c

@@ -1,5 +1,6 @@
 #include "config.h"
 #include <assert.h>
+#include <ccan/array_size/array_size.h>
 #include <common/amount.h>
 #include <common/pseudorand.h>
 #include <common/setup.h>
@@ -106,47 +107,67 @@ void towire_u8_array(u8 **pptr UNNEEDED, const u8 *arr UNNEEDED, size_t num UNNE
 static void test_valid(void)
 {
 	assert(utf8_check("hello world", strlen("hello world")));
+	assert(utf8_check_text("hello world", strlen("hello world")));
 
 	{
 		static const u8 nansensu[] = {
 			0xe3, 0x83, 0x8a, 0xe3, 0x83, 0xb3, 0xe3, 0x82,
 			0xbb, 0xe3, 0x83, 0xb3, 0xe3, 0x82, 0xb9
 		};
 		assert(utf8_check(nansensu, sizeof(nansensu)));
+		assert(utf8_check_text(nansensu, sizeof(nansensu)));
 	}
 
 	assert(utf8_check("", 0));
+	assert(utf8_check_text("", 0));
 }
 
-static void test_banned(void)
+/* NUL, surrogates and overlong encodings are rejected by ccan/utf8
+ * itself (utf8_decode()), regardless of category filtering - even
+ * plain utf8_check() rejects those */
+static void test_encoding_banned(void)
 {
 	static const u8 embedded_nul[] = { 'a', 0x00, 'b' };
 	assert(!utf8_check(embedded_nul, sizeof(embedded_nul)));
+	assert(!utf8_check_text(embedded_nul, sizeof(embedded_nul)));
+}
 
-	assert(!utf8_check("a\tb", 3));
-
+/* Codepoints in Unicode categories Cc/Cf/Co/Cn are valid UTF-8
+ * encoding (plain utf8_check() accepts them - it's used for general
+ * JSON-RPC input, datastore, which shouldn't be restricted this
+ * way), but utf8_check_text() must reject them for protocol text
+ * fields */
+static void test_text_banned(void)
+{
+	static const u8 tab[] = { 'a', '\t', 'b' };
 	static const u8 del[] = { 'a', 0x7f, 'b' };
-	assert(!utf8_check(del, sizeof(del)));
-
 	static const u8 c1_control[] = { 'a', 0xc2, 0x85, 'b' };
-	assert(!utf8_check(c1_control, sizeof(c1_control)));
-
 	static const u8 rtl_override[] = { 'a', 0xe2, 0x80, 0xae, 'b' };
-	assert(!utf8_check(rtl_override, sizeof(rtl_override)));
-
 	static const u8 private_use[] = { 'a', 0xee, 0x80, 0x80, 'b' };
-	assert(!utf8_check(private_use, sizeof(private_use)));
-
 	static const u8 unassigned[] = { 'a', 0xcd, 0xb8, 'b' };
-	assert(!utf8_check(unassigned, sizeof(unassigned)));
+	static const u8 *cases[] = {
+		tab, del, c1_control, rtl_override, private_use, unassigned
+	};
+	static const size_t lens[] = {
+		sizeof(tab), sizeof(del), sizeof(c1_control),
+		sizeof(rtl_override), sizeof(private_use), sizeof(unassigned)
+	};
+
+	for (size_t i = 0; i < ARRAY_SIZE(cases); i++) {
+		/* Plain utf8_check() stays permissive - valid encoding */
+		assert(utf8_check(cases[i], lens[i]));
+		/* utf8_check_text() rejects the banned category */
+		assert(!utf8_check_text(cases[i], lens[i]));
+	}
 }
 
 int main(int argc, char *argv[])
 {
 	common_setup(argv[0]);
 
 	test_valid();
-	test_banned();
+	test_encoding_banned();
+	test_text_banned();
 
 	common_shutdown();
 }

common/unicode_category.c

@@ -1,6 +1,7 @@
-/* This file was generated by devtools/gen-unicode-category.py */
-/* Do not modify this file! Modify the script, or regenerate from a new
- * UnicodeData.txt, instead. */
+
+/* This file generated by devtools/gen-unicode-category.py */
+/* Do not modify this file!!! Modify the script, or regenerate from a new
+ * UnicodeData.txt, instead */
 /* Generated from Unicode 17.0.0 UnicodeData.txt */
 
 #include "config.h"

common/unicode_category.h

@@ -1,13 +1,14 @@
-/* This file was generated by devtools/gen-unicode-category.py */
-/* Do not modify this file! Modify the script, or regenerate from a new
- * UnicodeData.txt, instead. */
 
-#ifndef LIGHTNING_COMMON_UNICODE_CATEGORY_GEN_H
-#define LIGHTNING_COMMON_UNICODE_CATEGORY_GEN_H
+/* This file generated by devtools/gen-unicode-category.py */
+/* Do not modify this file!!! Modify the script, or regenerate from a new
+ * UnicodeData.txt, instead */
+
+#ifndef LIGHTNING_COMMON_UNICODE_CATEGORY_H
+#define LIGHTNING_COMMON_UNICODE_CATEGORY_H
 #include "config.h"
 #include <ccan/short_types/short_types.h>
 #include <stdbool.h>
 
 bool unicode_is_banned_codepoint(u32 cp);
 
-#endif /* LIGHTNING_COMMON_UNICODE_CATEGORY_GEN_H */
+#endif /* LIGHTNING_COMMON_UNICODE_CATEGORY_H */

common/utils.c

@@ -181,8 +181,7 @@ void tal_arr_appendn_(void *p, const void *append TAKES, size_t bytes)
 	tal_arr_append_bytes(p, append, bytes);
 }
 
-/* Check for valid UTF-8 */
-bool utf8_check(const void *vbuf, size_t buflen)
+static bool utf8_check_(const void *vbuf, size_t buflen, bool reject_banned)
 {
 	const u8 *buf = vbuf;
 	struct utf8_state utf8_state = UTF8_STATE_INIT;
@@ -196,17 +195,29 @@ bool utf8_check(const void *vbuf, size_t buflen)
 		need_more = false;
 		if (errno != 0)
 			return false;
-		if (unicode_is_banned_codepoint(utf8_state.c))
+		if (reject_banned && unicode_is_banned_codepoint(utf8_state.c))
 			return false;
 	}
 	return !need_more;
 }
 
-char *utf8_str(const tal_t *ctx, const u8 *buf TAKES, size_t buflen)
+/* Check for valid UTF-8 */
+bool utf8_check(const void *vbuf, size_t buflen)
+{
+	return utf8_check_(vbuf, buflen, false);
+}
+
+bool utf8_check_text(const void *vbuf, size_t buflen)
+{
+	return utf8_check_(vbuf, buflen, true);
+}
+
+static char *utf8_str_(const tal_t *ctx, const u8 *buf TAKES, size_t buflen,
+			bool reject_banned)
 {
 	char *ret;
 
-	if (!utf8_check(buf, buflen)) {
+	if (!utf8_check_(buf, buflen, reject_banned)) {
 		tal_free_if_taken(buf);
 		return NULL;
 	}
@@ -217,6 +228,16 @@ char *utf8_str(const tal_t *ctx, const u8 *buf TAKES, size_t buflen)
 	return ret;
 }
 
+char *utf8_str(const tal_t *ctx, const u8 *buf TAKES, size_t buflen)
+{
+	return utf8_str_(ctx, buf, buflen, false);
+}
+
+char *utf8_str_text(const tal_t *ctx, const u8 *buf TAKES, size_t buflen)
+{
+	return utf8_str_(ctx, buf, buflen, true);
+}
+
 char *tal_strdup_or_null(const tal_t *ctx, const char *str)
 {
 	if (!str) {

common/utils.h

@@ -139,6 +139,13 @@ bool utf8_check(const void *buf, size_t buflen);
 /* Check it's UTF-8, return copy (or same if TAKES), or NULL if not valid. */
 char *utf8_str(const tal_t *ctx, const u8 *buf TAKES, size_t buflen);
 
+/* Like utf8_check(), but also reject codepoints in Unicode general
+ * categories Cc/Cf/Co/Cn (control, format, private-use, unassigned) */
+bool utf8_check_text(const void *buf, size_t buflen);
+
+/* Like utf8_str(), but using utf8_check_text() instead of utf8_check(). */
+char *utf8_str_text(const tal_t *ctx, const u8 *buf TAKES, size_t buflen);
+
 /* Strdup, or pass through NULL */
 char *tal_strdup_or_null(const tal_t *ctx, const char *str);
 

wire/fromwire.c

@@ -232,7 +232,7 @@ void fromwire_u8_array(const u8 **cursor, size_t *max, u8 *arr, size_t num)
 void fromwire_utf8_array(const u8 **cursor, size_t *max, char *arr, size_t num)
 {
 	fromwire(cursor, max, arr, num);
-	if (!utf8_check(arr, num))
+	if (!utf8_check_text(arr, num))
 		fromwire_fail(cursor, max);
 }