Skip to content

Commit a511a6b

Browse files
RasulOsRasulOs
authored
Merge pull request #149 from droidrun/fix/atomic-text-input
Fix atomic IME text replacement
2 parents 6de989d + b1bd641 commit a511a6b

6 files changed

Lines changed: 1015 additions & 42 deletions

File tree

‎app/src/main/java/com/mobilerun/portal/api/ApiHandler.kt‎

Lines changed: 41 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import android.view.accessibility.AccessibilityNodeInfo
1717
import android.net.Uri
1818
import android.provider.Settings
1919
import com.mobilerun.portal.input.MobilerunKeyboardIME
20+
import com.mobilerun.portal.input.TextInputResult
2021
import com.mobilerun.portal.core.JsonBuilders
2122
import com.mobilerun.portal.core.StateRepository
2223
import com.mobilerun.portal.service.GestureController
@@ -314,8 +315,26 @@ class ApiHandler(
314315
fun keyboardInput(base64Text: String, clear: Boolean): ApiResponse {
315316
val ime = getKeyboardIME()
316317
if (ime != null) {
317-
if (ime.inputB64Text(base64Text, clear)) {
318-
return ApiResponse.Success("input done via IME (clear=$clear)")
318+
when (ime.inputB64TextResult(base64Text, clear)) {
319+
TextInputResult.Verified -> {
320+
return ApiResponse.Success("input done via IME (clear=$clear)")
321+
}
322+
TextInputResult.AcceptedUnverified -> {
323+
return ApiResponse.Error(
324+
"input accepted via IME but could not be verified; fallback skipped",
325+
)
326+
}
327+
TextInputResult.CommitOutcomeUnknown -> {
328+
return ApiResponse.Error(
329+
"IME commit outcome unknown; fallback skipped",
330+
)
331+
}
332+
TextInputResult.InputSessionChanged -> {
333+
return ApiResponse.Error(
334+
"input session changed during IME input; fallback skipped",
335+
)
336+
}
337+
TextInputResult.Rejected -> Unit
319338
}
320339
}
321340

@@ -338,10 +357,27 @@ class ApiHandler(
338357
val ime = getKeyboardIME()
339358

340359
if (ime != null && ime.hasInputConnection()) {
341-
if (ime.clearText()) {
342-
return ApiResponse.Success("Text cleared via IME")
360+
when (ime.clearTextResult()) {
361+
TextInputResult.Verified -> {
362+
return ApiResponse.Success("Text cleared via IME")
363+
}
364+
TextInputResult.AcceptedUnverified -> {
365+
return ApiResponse.Error(
366+
"clear accepted via IME but could not be verified; fallback skipped",
367+
)
368+
}
369+
TextInputResult.CommitOutcomeUnknown -> {
370+
return ApiResponse.Error(
371+
"IME clear commit outcome unknown; fallback skipped",
372+
)
373+
}
374+
TextInputResult.InputSessionChanged -> {
375+
return ApiResponse.Error(
376+
"input session changed during IME clear; fallback skipped",
377+
)
378+
}
379+
TextInputResult.Rejected -> Unit
343380
}
344-
Log.w(TAG, "IME clearText() failed, falling back to Accessibility")
345381
}
346382

347383
return if (stateRepo.inputText("", clear = true)) {

‎app/src/main/java/com/mobilerun/portal/input/InputConnectionTextEditor.kt‎

Lines changed: 296 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,296 @@
1+
package com.mobilerun.portal.input
2+
3+
import android.view.inputmethod.ExtractedTextRequest
4+
import android.view.inputmethod.InputConnection
5+
6+
internal sealed class TextInputResult {
7+
data object Verified : TextInputResult()
8+
data object Rejected : TextInputResult()
9+
data object AcceptedUnverified : TextInputResult()
10+
data object CommitOutcomeUnknown : TextInputResult()
11+
data object InputSessionChanged : TextInputResult()
12+
}
13+
14+
internal class InputConnectionTextEditor(
15+
private val connectionProvider: () -> InputConnection?,
16+
private val generationProvider: () -> Long,
17+
private val sleep: (Long) -> Unit,
18+
private val retryDelayMs: Long = DEFAULT_RETRY_DELAY_MS,
19+
) {
20+
companion object {
21+
private const val DEFAULT_RETRY_DELAY_MS = 100L
22+
private const val MAX_ATTEMPTS = 2
23+
private const val ZERO_WIDTH_SPACE = '\u200B'
24+
private const val ZERO_WIDTH_NO_BREAK_SPACE = '\uFEFF'
25+
}
26+
27+
private data class Snapshot(
28+
val text: String,
29+
val startOffset: Int,
30+
val partialStartOffset: Int,
31+
val selectionStart: Int,
32+
val selectionEnd: Int,
33+
val reachesFieldEnd: Boolean,
34+
) {
35+
val isCompleteReport: Boolean
36+
get() = partialStartOffset == -1
37+
38+
val coversEntireField: Boolean
39+
get() = startOffset == 0 && isCompleteReport && reachesFieldEnd
40+
}
41+
42+
fun inputText(text: String, clear: Boolean): TextInputResult {
43+
val inputSessionGeneration = generationProvider()
44+
45+
repeat(MAX_ATTEMPTS) { attempt ->
46+
if (!isSameInputSession(inputSessionGeneration)) {
47+
return TextInputResult.InputSessionChanged
48+
}
49+
50+
val connection = connectionOrNull()
51+
if (connection == null) {
52+
if (attempt < MAX_ATTEMPTS - 1) {
53+
sleep(retryDelayMs)
54+
return@repeat
55+
}
56+
return TextInputResult.Rejected
57+
}
58+
59+
if (!isSameInputSession(inputSessionGeneration)) {
60+
return TextInputResult.InputSessionChanged
61+
}
62+
val compositionFinished = try {
63+
connection.finishComposingText()
64+
} catch (_: Exception) {
65+
false
66+
}
67+
if (!compositionFinished) {
68+
if (!clear) {
69+
when (val result = commitWithoutVerification(connection, text, inputSessionGeneration)) {
70+
TextInputResult.Rejected -> {
71+
if (attempt < MAX_ATTEMPTS - 1) {
72+
sleep(retryDelayMs)
73+
return@repeat
74+
}
75+
return TextInputResult.Rejected
76+
}
77+
else -> return result
78+
}
79+
}
80+
if (!isSameInputSession(inputSessionGeneration)) {
81+
return TextInputResult.InputSessionChanged
82+
}
83+
if (attempt < MAX_ATTEMPTS - 1) {
84+
sleep(retryDelayMs)
85+
return@repeat
86+
}
87+
return TextInputResult.Rejected
88+
}
89+
if (!isSameInputSession(inputSessionGeneration)) {
90+
return TextInputResult.InputSessionChanged
91+
}
92+
93+
val before = readSnapshot(connection, requireFieldEnd = clear)
94+
if (before == null && !clear) {
95+
when (val result = commitWithoutVerification(connection, text, inputSessionGeneration)) {
96+
TextInputResult.Rejected -> {
97+
if (attempt < MAX_ATTEMPTS - 1) {
98+
sleep(retryDelayMs)
99+
return@repeat
100+
}
101+
return TextInputResult.Rejected
102+
}
103+
else -> return result
104+
}
105+
}
106+
107+
if (before == null || (clear && !before.coversEntireField)) {
108+
if (attempt < MAX_ATTEMPTS - 1) {
109+
sleep(retryDelayMs)
110+
return@repeat
111+
}
112+
return TextInputResult.Rejected
113+
}
114+
115+
val selection = selectionFor(before, clear)
116+
val expected = expectedText(before, text, clear)
117+
val selectionAccepted = try {
118+
connection.setSelection(selection.first, selection.second)
119+
} catch (_: Exception) {
120+
false
121+
}
122+
if (!selectionAccepted) {
123+
if (!isSameInputSession(inputSessionGeneration)) {
124+
return TextInputResult.InputSessionChanged
125+
}
126+
if (attempt < MAX_ATTEMPTS - 1) {
127+
sleep(retryDelayMs)
128+
return@repeat
129+
}
130+
return TextInputResult.Rejected
131+
}
132+
133+
if (!isSameInputSession(inputSessionGeneration)) {
134+
return TextInputResult.InputSessionChanged
135+
}
136+
val commitAccepted = try {
137+
connection.commitText(text, 1)
138+
} catch (_: Exception) {
139+
return if (isSameInputSession(inputSessionGeneration)) {
140+
TextInputResult.CommitOutcomeUnknown
141+
} else {
142+
TextInputResult.InputSessionChanged
143+
}
144+
}
145+
if (!commitAccepted) {
146+
if (!isSameInputSession(inputSessionGeneration)) {
147+
return TextInputResult.InputSessionChanged
148+
}
149+
if (attempt < MAX_ATTEMPTS - 1) {
150+
sleep(retryDelayMs)
151+
return@repeat
152+
}
153+
return TextInputResult.Rejected
154+
}
155+
156+
sleep(retryDelayMs)
157+
if (!isSameInputSession(inputSessionGeneration)) {
158+
return TextInputResult.InputSessionChanged
159+
}
160+
val afterConnection = connectionOrNull()
161+
val after = afterConnection?.let {
162+
readSnapshot(it, requireFieldEnd = clear)
163+
}
164+
if (!isSameInputSession(inputSessionGeneration)) {
165+
return TextInputResult.InputSessionChanged
166+
}
167+
if (after != null &&
168+
canVerify(before, after, clear) &&
169+
matches(after.text, expected, text)
170+
) {
171+
return TextInputResult.Verified
172+
}
173+
174+
return TextInputResult.AcceptedUnverified
175+
}
176+
177+
return TextInputResult.Rejected
178+
}
179+
180+
private fun isSameInputSession(generation: Long): Boolean {
181+
return generationProvider() == generation
182+
}
183+
184+
private fun commitWithoutVerification(
185+
connection: InputConnection,
186+
text: String,
187+
inputSessionGeneration: Long,
188+
): TextInputResult {
189+
if (!isSameInputSession(inputSessionGeneration)) {
190+
return TextInputResult.InputSessionChanged
191+
}
192+
val accepted = try {
193+
connection.commitText(text, 1)
194+
} catch (_: Exception) {
195+
return if (isSameInputSession(inputSessionGeneration)) {
196+
TextInputResult.CommitOutcomeUnknown
197+
} else {
198+
TextInputResult.InputSessionChanged
199+
}
200+
}
201+
if (!isSameInputSession(inputSessionGeneration)) {
202+
return TextInputResult.InputSessionChanged
203+
}
204+
return if (accepted) TextInputResult.AcceptedUnverified else TextInputResult.Rejected
205+
}
206+
207+
private fun connectionOrNull(): InputConnection? {
208+
return try {
209+
connectionProvider()
210+
} catch (_: Exception) {
211+
null
212+
}
213+
}
214+
215+
private fun readSnapshot(
216+
connection: InputConnection,
217+
requireFieldEnd: Boolean = false,
218+
): Snapshot? {
219+
return try {
220+
val extracted = connection.getExtractedText(ExtractedTextRequest(), 0) ?: return null
221+
val value = extracted.text?.toString() ?: ""
222+
val relativeStart = extracted.selectionStart
223+
val relativeEnd = extracted.selectionEnd
224+
if (relativeStart < 0 || relativeEnd < 0 || relativeStart > value.length || relativeEnd > value.length) {
225+
return null
226+
}
227+
val structurallyComplete = extracted.startOffset == 0 && extracted.partialStartOffset == -1
228+
val fieldEndConfirmed = !requireFieldEnd ||
229+
(structurallyComplete && reachesFieldEnd(connection, value, relativeStart, relativeEnd))
230+
Snapshot(
231+
text = value,
232+
startOffset = extracted.startOffset,
233+
partialStartOffset = extracted.partialStartOffset,
234+
selectionStart = relativeStart,
235+
selectionEnd = relativeEnd,
236+
reachesFieldEnd = fieldEndConfirmed,
237+
)
238+
} catch (_: Exception) {
239+
return null
240+
}
241+
}
242+
243+
private fun reachesFieldEnd(
244+
connection: InputConnection,
245+
text: String,
246+
selectionStart: Int,
247+
selectionEnd: Int,
248+
): Boolean {
249+
val cursor = maxOf(selectionStart, selectionEnd)
250+
val representedSuffix = text.substring(cursor)
251+
val actualSuffix = connection.getTextAfterCursor(representedSuffix.length + 1, 0)
252+
?.toString()
253+
?: return false
254+
return actualSuffix == representedSuffix
255+
}
256+
257+
private fun selectionFor(snapshot: Snapshot, clear: Boolean): Pair<Int, Int> {
258+
val relativeStart = if (clear) 0 else minOf(snapshot.selectionStart, snapshot.selectionEnd)
259+
val relativeEnd = if (clear) snapshot.text.length else maxOf(snapshot.selectionStart, snapshot.selectionEnd)
260+
return Pair(snapshot.startOffset + relativeStart, snapshot.startOffset + relativeEnd)
261+
}
262+
263+
private fun expectedText(snapshot: Snapshot, text: String, clear: Boolean): String {
264+
if (clear) return text
265+
266+
val selectionStart = minOf(snapshot.selectionStart, snapshot.selectionEnd)
267+
val selectionEnd = maxOf(snapshot.selectionStart, snapshot.selectionEnd)
268+
return snapshot.text.replaceRange(selectionStart, selectionEnd, text)
269+
}
270+
271+
private fun canVerify(
272+
before: Snapshot,
273+
after: Snapshot,
274+
clear: Boolean,
275+
): Boolean {
276+
if (clear) return after.coversEntireField
277+
return before.isCompleteReport &&
278+
after.isCompleteReport &&
279+
before.startOffset == after.startOffset
280+
}
281+
282+
private fun matches(actual: String, expected: String, requested: String): Boolean {
283+
if (requested.any(::isIgnoredSentinel)) {
284+
return actual == expected
285+
}
286+
return normalize(actual) == normalize(expected)
287+
}
288+
289+
private fun normalize(value: String): String {
290+
return value.filterNot(::isIgnoredSentinel)
291+
}
292+
293+
private fun isIgnoredSentinel(value: Char): Boolean {
294+
return value == ZERO_WIDTH_SPACE || value == ZERO_WIDTH_NO_BREAK_SPACE
295+
}
296+
}

0 commit comments

Comments
 (0)