Skip to content

Fix atomic IME text replacement #149

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
RasulOs merged 9 commits into from
Jul 21, 2026
Merged

Fix atomic IME text replacement #149

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
b29ba45
fix: replace IME text atomically
RasulOs Jul 20, 2026
c8d1d3d
fix: address IME verification review
RasulOs Jul 21, 2026
a9e4442
fix: keep IME retries in input session
RasulOs Jul 21, 2026
eb3ebd1
fix: preserve IME verification outcomes
RasulOs Jul 21, 2026
b29890c
fix: preserve atomic replacement outcomes
RasulOs Jul 21, 2026
ad8f635
fix: prevent accepted text input replay
RasulOs Jul 21, 2026
86a7191
fix: distinguish unknown IME commit outcomes
RasulOs Jul 21, 2026
4f8266f
fix: renew IME sessions on restart
RasulOs Jul 21, 2026
b1bd641
fix: confirm complete extracted fields
RasulOs Jul 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 41 additions & 5 deletions app/src/main/java/com/mobilerun/portal/api/ApiHandler.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import android.view.accessibility.AccessibilityNodeInfo
import android.net.Uri
import android.provider.Settings
import com.mobilerun.portal.input.MobilerunKeyboardIME
import com.mobilerun.portal.input.TextInputResult
import com.mobilerun.portal.core.JsonBuilders
import com.mobilerun.portal.core.StateRepository
import com.mobilerun.portal.service.GestureController
Expand Down Expand Up @@ -314,8 +315,26 @@ class ApiHandler(
fun keyboardInput(base64Text: String, clear: Boolean): ApiResponse {
val ime = getKeyboardIME()
if (ime != null) {
if (ime.inputB64Text(base64Text, clear)) {
return ApiResponse.Success("input done via IME (clear=$clear)")
when (ime.inputB64TextResult(base64Text, clear)) {
TextInputResult.Verified -> {
return ApiResponse.Success("input done via IME (clear=$clear)")
}
TextInputResult.AcceptedUnverified -> {
return ApiResponse.Error(
"input accepted via IME but could not be verified; fallback skipped",
)
}
TextInputResult.CommitOutcomeUnknown -> {
return ApiResponse.Error(
"IME commit outcome unknown; fallback skipped",
)
}
TextInputResult.InputSessionChanged -> {
return ApiResponse.Error(
"input session changed during IME input; fallback skipped",
)
}
TextInputResult.Rejected -> Unit
}
}

Expand All @@ -338,10 +357,27 @@ class ApiHandler(
val ime = getKeyboardIME()

if (ime != null && ime.hasInputConnection()) {
if (ime.clearText()) {
return ApiResponse.Success("Text cleared via IME")
when (ime.clearTextResult()) {
TextInputResult.Verified -> {
return ApiResponse.Success("Text cleared via IME")
}
TextInputResult.AcceptedUnverified -> {
return ApiResponse.Error(
"clear accepted via IME but could not be verified; fallback skipped",
)
}
TextInputResult.CommitOutcomeUnknown -> {
return ApiResponse.Error(
"IME clear commit outcome unknown; fallback skipped",
)
}
TextInputResult.InputSessionChanged -> {
return ApiResponse.Error(
"input session changed during IME clear; fallback skipped",
)
}
TextInputResult.Rejected -> Unit
}
Log.w(TAG, "IME clearText() failed, falling back to Accessibility")
}

return if (stateRepo.inputText("", clear = true)) {
Expand Down
296 changes: 296 additions & 0 deletions app/src/main/java/com/mobilerun/portal/input/InputConnectionTextEditor.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,296 @@
package com.mobilerun.portal.input

import android.view.inputmethod.ExtractedTextRequest
import android.view.inputmethod.InputConnection

internal sealed class TextInputResult {
data object Verified : TextInputResult()
data object Rejected : TextInputResult()
data object AcceptedUnverified : TextInputResult()
data object CommitOutcomeUnknown : TextInputResult()
data object InputSessionChanged : TextInputResult()
}

internal class InputConnectionTextEditor(
private val connectionProvider: () -> InputConnection?,
private val generationProvider: () -> Long,
private val sleep: (Long) -> Unit,
private val retryDelayMs: Long = DEFAULT_RETRY_DELAY_MS,
) {
companion object {
private const val DEFAULT_RETRY_DELAY_MS = 100L
private const val MAX_ATTEMPTS = 2
private const val ZERO_WIDTH_SPACE = '\u200B'
private const val ZERO_WIDTH_NO_BREAK_SPACE = '\uFEFF'
}

private data class Snapshot(
val text: String,
val startOffset: Int,
val partialStartOffset: Int,
val selectionStart: Int,
val selectionEnd: Int,
val reachesFieldEnd: Boolean,
) {
val isCompleteReport: Boolean
get() = partialStartOffset == -1

val coversEntireField: Boolean
get() = startOffset == 0 && isCompleteReport && reachesFieldEnd
}

fun inputText(text: String, clear: Boolean): TextInputResult {
val inputSessionGeneration = generationProvider()

repeat(MAX_ATTEMPTS) { attempt ->
if (!isSameInputSession(inputSessionGeneration)) {
return TextInputResult.InputSessionChanged
}

val connection = connectionOrNull()
if (connection == null) {
if (attempt < MAX_ATTEMPTS - 1) {
sleep(retryDelayMs)
return@repeat
}
return TextInputResult.Rejected
}

if (!isSameInputSession(inputSessionGeneration)) {
return TextInputResult.InputSessionChanged
}
val compositionFinished = try {
connection.finishComposingText()
} catch (_: Exception) {
false
}
if (!compositionFinished) {
if (!clear) {
when (val result = commitWithoutVerification(connection, text, inputSessionGeneration)) {
TextInputResult.Rejected -> {
if (attempt < MAX_ATTEMPTS - 1) {
sleep(retryDelayMs)
return@repeat
}
return TextInputResult.Rejected
}
else -> return result
}
}
if (!isSameInputSession(inputSessionGeneration)) {
return TextInputResult.InputSessionChanged
}
if (attempt < MAX_ATTEMPTS - 1) {
sleep(retryDelayMs)
return@repeat
}
return TextInputResult.Rejected
}
if (!isSameInputSession(inputSessionGeneration)) {
return TextInputResult.InputSessionChanged
}

val before = readSnapshot(connection, requireFieldEnd = clear)
if (before == null && !clear) {
when (val result = commitWithoutVerification(connection, text, inputSessionGeneration)) {
TextInputResult.Rejected -> {
if (attempt < MAX_ATTEMPTS - 1) {
sleep(retryDelayMs)
return@repeat
}
return TextInputResult.Rejected
}
else -> return result
}
}

if (before == null || (clear && !before.coversEntireField)) {
if (attempt < MAX_ATTEMPTS - 1) {
sleep(retryDelayMs)
return@repeat
}
return TextInputResult.Rejected
}

val selection = selectionFor(before, clear)
val expected = expectedText(before, text, clear)
val selectionAccepted = try {
connection.setSelection(selection.first, selection.second)
} catch (_: Exception) {
false
}
if (!selectionAccepted) {
if (!isSameInputSession(inputSessionGeneration)) {
return TextInputResult.InputSessionChanged
}
if (attempt < MAX_ATTEMPTS - 1) {
sleep(retryDelayMs)
return@repeat
}
return TextInputResult.Rejected
}

if (!isSameInputSession(inputSessionGeneration)) {
return TextInputResult.InputSessionChanged
}
val commitAccepted = try {
connection.commitText(text, 1)
} catch (_: Exception) {
return if (isSameInputSession(inputSessionGeneration)) {
TextInputResult.CommitOutcomeUnknown
} else {
TextInputResult.InputSessionChanged
}
}
if (!commitAccepted) {
if (!isSameInputSession(inputSessionGeneration)) {
return TextInputResult.InputSessionChanged
}
if (attempt < MAX_ATTEMPTS - 1) {
sleep(retryDelayMs)
return@repeat
}
return TextInputResult.Rejected
}

sleep(retryDelayMs)
if (!isSameInputSession(inputSessionGeneration)) {
return TextInputResult.InputSessionChanged
}
val afterConnection = connectionOrNull()
val after = afterConnection?.let {
readSnapshot(it, requireFieldEnd = clear)
}
if (!isSameInputSession(inputSessionGeneration)) {
return TextInputResult.InputSessionChanged
}
if (after != null &&
canVerify(before, after, clear) &&
matches(after.text, expected, text)
) {
return TextInputResult.Verified
}

return TextInputResult.AcceptedUnverified
}

return TextInputResult.Rejected
}

private fun isSameInputSession(generation: Long): Boolean {
return generationProvider() == generation
}

private fun commitWithoutVerification(
connection: InputConnection,
text: String,
inputSessionGeneration: Long,
): TextInputResult {
if (!isSameInputSession(inputSessionGeneration)) {
return TextInputResult.InputSessionChanged
}
val accepted = try {
connection.commitText(text, 1)
} catch (_: Exception) {
return if (isSameInputSession(inputSessionGeneration)) {
TextInputResult.CommitOutcomeUnknown
} else {
TextInputResult.InputSessionChanged
}
}
if (!isSameInputSession(inputSessionGeneration)) {
return TextInputResult.InputSessionChanged
}
return if (accepted) TextInputResult.AcceptedUnverified else TextInputResult.Rejected
}

private fun connectionOrNull(): InputConnection? {
return try {
connectionProvider()
} catch (_: Exception) {
null
}
}

private fun readSnapshot(
connection: InputConnection,
requireFieldEnd: Boolean = false,
): Snapshot? {
return try {
val extracted = connection.getExtractedText(ExtractedTextRequest(), 0) ?: return null
val value = extracted.text?.toString() ?: ""
val relativeStart = extracted.selectionStart
val relativeEnd = extracted.selectionEnd
if (relativeStart < 0 || relativeEnd < 0 || relativeStart > value.length || relativeEnd > value.length) {
return null
}
val structurallyComplete = extracted.startOffset == 0 && extracted.partialStartOffset == -1
val fieldEndConfirmed = !requireFieldEnd ||
(structurallyComplete && reachesFieldEnd(connection, value, relativeStart, relativeEnd))
Snapshot(
text = value,
startOffset = extracted.startOffset,
partialStartOffset = extracted.partialStartOffset,
selectionStart = relativeStart,
selectionEnd = relativeEnd,
reachesFieldEnd = fieldEndConfirmed,
)
} catch (_: Exception) {
return null
}
}

private fun reachesFieldEnd(
connection: InputConnection,
text: String,
selectionStart: Int,
selectionEnd: Int,
): Boolean {
val cursor = maxOf(selectionStart, selectionEnd)
val representedSuffix = text.substring(cursor)
val actualSuffix = connection.getTextAfterCursor(representedSuffix.length + 1, 0)
?.toString()
?: return false
return actualSuffix == representedSuffix
Comment thread
RasulOs marked this conversation as resolved.
}

private fun selectionFor(snapshot: Snapshot, clear: Boolean): Pair<Int, Int> {
val relativeStart = if (clear) 0 else minOf(snapshot.selectionStart, snapshot.selectionEnd)
val relativeEnd = if (clear) snapshot.text.length else maxOf(snapshot.selectionStart, snapshot.selectionEnd)
return Pair(snapshot.startOffset + relativeStart, snapshot.startOffset + relativeEnd)
Comment thread
RasulOs marked this conversation as resolved.
}

private fun expectedText(snapshot: Snapshot, text: String, clear: Boolean): String {
if (clear) return text

val selectionStart = minOf(snapshot.selectionStart, snapshot.selectionEnd)
val selectionEnd = maxOf(snapshot.selectionStart, snapshot.selectionEnd)
return snapshot.text.replaceRange(selectionStart, selectionEnd, text)
}

private fun canVerify(
before: Snapshot,
after: Snapshot,
clear: Boolean,
): Boolean {
if (clear) return after.coversEntireField
Comment thread
RasulOs marked this conversation as resolved.
return before.isCompleteReport &&
after.isCompleteReport &&
before.startOffset == after.startOffset
}

private fun matches(actual: String, expected: String, requested: String): Boolean {
if (requested.any(::isIgnoredSentinel)) {
return actual == expected
}
return normalize(actual) == normalize(expected)
}

private fun normalize(value: String): String {
return value.filterNot(::isIgnoredSentinel)
}

private fun isIgnoredSentinel(value: Char): Boolean {
return value == ZERO_WIDTH_SPACE || value == ZERO_WIDTH_NO_BREAK_SPACE
}
}
Loading
Loading