Updated ImapTokenCache to be able to handle non-ASCII tokens

Author: jstedfast

MailKit/Net/Imap/ImapToken.cs

@@ -127,19 +127,6 @@ public static ImapToken Create (ImapTokenType type, int literalLength)
 			return new ImapToken (type, literalLength);
 		}
 
-		static bool IsAscii (ByteArrayBuilder builder)
-		{
-			for (int i = 0; i < builder.Length; i++) {
-				byte c = builder[i];
-
-				// Disregard any non-ASCII tokens.
-				if (c < 32 || c >= 127)
-					return false;
-			}
-
-			return true;
-		}
-
 		static bool IsCacheable (ByteArrayBuilder builder)
 		{
 			if (builder.Length < 2 || builder.Length > 32)
@@ -153,7 +140,7 @@ static bool IsCacheable (ByteArrayBuilder builder)
 			if (builder[0] >= (byte) 'A' && builder[0] <= (byte) 'Z' && builder[1] >= (byte) '0' && builder[1] <= (byte) '9')
 				return false;
 
-			return IsAscii (builder);
+			return true;
 		}
 
 		public static ImapToken Create (ImapTokenType type, ByteArrayBuilder builder)
@@ -169,7 +156,7 @@ public static ImapToken Create (ImapTokenType type, ByteArrayBuilder builder)
 						return token;
 				}
 
-				cachable = IsAscii (builder);
+				cachable = true;
 			} else if (type == ImapTokenType.Atom) {
 				if (builder.Equals ("NIL", true)) {
 					// Look for the cached NIL token that matches this capitalization.
@@ -218,7 +205,7 @@ public static ImapToken Create (ImapTokenType type, ByteArrayBuilder builder)
 
 				cachable = IsCacheable (builder);
 			} else if (type == ImapTokenType.QString) {
-				cachable = IsAscii (builder);
+				cachable = true;
 			}
 
 			if (cachable)

MailKit/Net/Imap/ImapTokenCache.cs

@@ -37,19 +37,29 @@ class ImapTokenCache
 		readonly Dictionary<ImapTokenKey, LinkedListNode<ImapTokenItem>> cache;
 		readonly LinkedList<ImapTokenItem> list;
 		readonly ImapTokenKey lookupKey;
+		readonly Decoder[] decoders;
+		readonly char[] chars;
 
 		public ImapTokenCache ()
 		{
 			cache = new Dictionary<ImapTokenKey, LinkedListNode<ImapTokenItem>> ();
 			list = new LinkedList<ImapTokenItem> ();
 			lookupKey = new ImapTokenKey ();
+
+			// Start with the assumption that token values will be valid UTF-8 and then fall back to iso-8859-1.
+			decoders = new Decoder[2] {
+				TextEncodings.UTF8.GetDecoder (),
+				TextEncodings.Latin1.GetDecoder ()
+			};
+
+			chars = new char[128];
 		}
 
 		public ImapToken AddOrGet (ImapTokenType type, ByteArrayBuilder builder)
 		{
 			lock (cache) {
 				// lookupKey is a pre-allocated key used for lookups
-				lookupKey.Init (type, builder.GetBuffer (), builder.Length);
+				lookupKey.Init (decoders, chars, type, builder.GetBuffer (), builder.Length);
 
 				if (cache.TryGetValue (lookupKey, out var node)) {
 					// move the node to the head of the list
@@ -102,7 +112,7 @@ public ImapTokenKey (ImapTokenType type, string key)
 				Init (type, key);
 			}
 
-			public void Init (ImapTokenType type, byte[] key, int length)
+			public void Init (Decoder[] decoders, char[] chars, ImapTokenType type, byte[] key, int length)
 			{
 				this.type = type;
 				this.byteArrayKey = key;
@@ -111,8 +121,30 @@ public void Init (ImapTokenType type, byte[] key, int length)
 
 				var hash = new HashCode ();
 				hash.Add ((int) type);
-				for (int i = 0; i < length; i++)
-					hash.Add ((char) key[i]);
+
+				foreach (var decoder in decoders) {
+					bool completed;
+					int index = 0;
+
+					do {
+						try {
+							decoder.Convert (key, index, length - index, chars, 0, chars.Length, true, out var bytesUsed, out var charsUsed, out completed);
+							index += bytesUsed;
+
+							for (int i = 0; i < charsUsed; i++)
+								hash.Add (chars[i]);
+						} catch (DecoderFallbackException) {
+							// Restart the hash...
+							hash = new HashCode ();
+							hash.Add ((int) type);
+							completed = false;
+							break;
+						}
+					} while (!completed);
+
+					if (completed)
+						break;
+				}
 
 				this.hashCode = hash.ToHashCode ();
 			}