diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 67361e2..bac4ee9 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -27,8 +27,8 @@ android { applicationId = "me.maxistar.voiceinbox" minSdk = 24 targetSdk = 36 - versionCode = 6 - versionName = "0.3.0" + versionCode = 7 + versionName = "0.4.0" ndk { abiFilters += "arm64-v8a" } diff --git a/app/src/androidTest/java/me/maxistar/voiceinbox/VoiceKeyboardInstrumentedTest.kt b/app/src/androidTest/java/me/maxistar/voiceinbox/VoiceKeyboardInstrumentedTest.kt new file mode 100644 index 0000000..bb0f6c4 --- /dev/null +++ b/app/src/androidTest/java/me/maxistar/voiceinbox/VoiceKeyboardInstrumentedTest.kt @@ -0,0 +1,88 @@ +package me.maxistar.voiceinbox + +import android.content.ComponentName +import android.content.pm.PackageManager +import android.view.LayoutInflater +import android.widget.EditText +import android.widget.ImageButton +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class VoiceKeyboardInstrumentedTest { + @Test + fun manifestDeclaresBoundInputMethodService() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + val service = context.packageManager.getServiceInfo( + ComponentName(context, VoiceKeyboardInputMethodService::class.java), + PackageManager.GET_META_DATA, + ) + + assertEquals("android.permission.BIND_INPUT_METHOD", service.permission) + assertTrue(service.metaData?.containsKey("android.view.im") == true) + assertEquals( + PackageManager.PERMISSION_GRANTED, + context.packageManager.checkPermission(android.Manifest.permission.RECORD_AUDIO, context.packageName), + ) + } + + @Test + fun inputConnectionCommitsAndDeletesText() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + InstrumentationRegistry.getInstrumentation().runOnMainSync { + val editor = EditText(context) + editor.setText("start") + editor.setSelection(editor.length()) + val connection = editor.onCreateInputConnection(android.view.inputmethod.EditorInfo()) + + connection.commitText(" text", 1) + connection.deleteSurroundingTextInCodePoints(1, 0) + + assertEquals("start tex", editor.text.toString()) + } + } + + @Test + fun inputConnectionDeletesEmojiAsOneCodePoint() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + InstrumentationRegistry.getInstrumentation().runOnMainSync { + val editor = EditText(context) + editor.setText("start \uD83D\uDE00") + editor.setSelection(editor.length()) + val connection = editor.onCreateInputConnection(android.view.inputmethod.EditorInfo()) + + assertTrue(connection.deleteSurroundingTextInCodePoints(1, 0)) + + assertEquals("start ", editor.text.toString()) + } + } + + @Test + fun keyboardLayoutInflatesWithAccessibleIconControls() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + InstrumentationRegistry.getInstrumentation().runOnMainSync { + val view = LayoutInflater.from(context).inflate(R.layout.input_view_voice_keyboard, null) + + assertEquals( + context.getString(R.string.voice_keyboard_return_to_previous), + view.findViewById(R.id.voiceKeyboardNextKeyboard).contentDescription, + ) + assertEquals( + context.getString(R.string.voice_keyboard_space), + view.findViewById(R.id.voiceKeyboardSpace).contentDescription, + ) + assertEquals( + context.getString(R.string.voice_keyboard_enter), + view.findViewById(R.id.voiceKeyboardEnter).contentDescription, + ) + assertEquals( + context.getString(R.string.voice_keyboard_backspace), + view.findViewById(R.id.voiceKeyboardBackspace).contentDescription, + ) + } + } +} diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index f5060d2..3ec6f3e 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -6,6 +6,7 @@ + + + + + + + diff --git a/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt b/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt index c23c5af..a034c45 100644 --- a/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt +++ b/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt @@ -2,7 +2,9 @@ package me.maxistar.voiceinbox import me.maxistar.voiceinbox.core.* +import android.Manifest import android.content.Intent +import android.content.pm.PackageManager import android.media.MediaPlayer import android.net.Uri import android.os.Bundle @@ -160,6 +162,16 @@ class MainActivity : AppCompatActivity(), StartupProcessingDialogFragment.Listen if (uri != null) acceptModelFolder(uri) } + private val microphonePermissionRequest = registerForActivityResult( + ActivityResultContracts.RequestPermission(), + ) { granted -> + Toast.makeText( + this, + if (granted) R.string.voice_keyboard_permission_granted else R.string.voice_keyboard_permission_denied, + Toast.LENGTH_LONG, + ).show() + } + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) enableEdgeToEdge() @@ -237,6 +249,7 @@ class MainActivity : AppCompatActivity(), StartupProcessingDialogFragment.Listen intent.removeExtra(EXTRA_OPEN_MODEL_FOLDER_PICKER) modelFolderPicker.launch(null) } + requestMicrophonePermissionIfNeeded(intent) } override fun onStart() { @@ -251,7 +264,9 @@ class MainActivity : AppCompatActivity(), StartupProcessingDialogFragment.Listen override fun onNewIntent(intent: Intent) { super.onNewIntent(intent) + setIntent(intent) handleShareIntent(intent) + requestMicrophonePermissionIfNeeded(intent) } override fun onSaveInstanceState(outState: Bundle) { @@ -307,6 +322,14 @@ class MainActivity : AppCompatActivity(), StartupProcessingDialogFragment.Listen super.onDestroy() } + private fun requestMicrophonePermissionIfNeeded(intent: Intent) { + if (!intent.getBooleanExtra(VoiceKeyboardSetup.EXTRA_REQUEST_MICROPHONE_PERMISSION, false)) return + intent.removeExtra(VoiceKeyboardSetup.EXTRA_REQUEST_MICROPHONE_PERMISSION) + if (checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { + microphonePermissionRequest.launch(Manifest.permission.RECORD_AUDIO) + } + } + private fun bindViews() { importAudio = findViewById(R.id.importAudio) newTab = findViewById(R.id.newTab) @@ -1009,7 +1032,7 @@ class MainActivity : AppCompatActivity(), StartupProcessingDialogFragment.Listen SpeechModelImportPermission.releaseOwnedIfUnused(this) if (shouldHandleModelInstallSuccess(info.id.toString())) { modelReadiness.invalidate() - SpeechModelPreparation.invalidate(NativeTranscriptionBridge::reset) + SpeechModelWarmup.invalidate() } refreshModel() } diff --git a/app/src/main/java/me/maxistar/voiceinbox/SpeechModelDownloadWorker.kt b/app/src/main/java/me/maxistar/voiceinbox/SpeechModelDownloadWorker.kt index 71e173c..cccb464 100644 --- a/app/src/main/java/me/maxistar/voiceinbox/SpeechModelDownloadWorker.kt +++ b/app/src/main/java/me/maxistar/voiceinbox/SpeechModelDownloadWorker.kt @@ -110,7 +110,7 @@ class SpeechModelDownloadWorker( val installedDirectory = repository.activate().getOrElse { return failure(it.message ?: "Failed to activate speech model") } - SpeechModelPreparation.invalidate(NativeTranscriptionBridge::reset) + SpeechModelWarmup.invalidate() return Result.success(workDataOf(KEY_MODEL_PATH to installedDirectory.absolutePath)) } diff --git a/app/src/main/java/me/maxistar/voiceinbox/SpeechModelImportWorker.kt b/app/src/main/java/me/maxistar/voiceinbox/SpeechModelImportWorker.kt index fc07f0a..ca6adb4 100644 --- a/app/src/main/java/me/maxistar/voiceinbox/SpeechModelImportWorker.kt +++ b/app/src/main/java/me/maxistar/voiceinbox/SpeechModelImportWorker.kt @@ -42,7 +42,7 @@ class SpeechModelImportWorker( ).import(treeUri.toString()) { progress -> publishProgress(progress, repository) }.getOrElse { return failure(it.message ?: "Could not import speech model") } - SpeechModelPreparation.invalidate(NativeTranscriptionBridge::reset) + SpeechModelWarmup.invalidate() Result.success( workDataOf(SpeechModelInstallationWork.KEY_MODEL_PATH to installed.absolutePath), ) diff --git a/app/src/main/java/me/maxistar/voiceinbox/SpeechModelWarmup.kt b/app/src/main/java/me/maxistar/voiceinbox/SpeechModelWarmup.kt new file mode 100644 index 0000000..efe60f1 --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/SpeechModelWarmup.kt @@ -0,0 +1,103 @@ +package me.maxistar.voiceinbox + +import java.io.File +import java.util.concurrent.Callable +import java.util.concurrent.Executor +import java.util.concurrent.Executors +import java.util.concurrent.Future +import java.util.concurrent.FutureTask + +internal sealed interface SpeechModelWarmupState { + data object Idle : SpeechModelWarmupState + data class Preparing(val installation: String) : SpeechModelWarmupState + data class Ready(val installation: String) : SpeechModelWarmupState + data class Failed(val installation: String, val message: String) : SpeechModelWarmupState +} + +internal class SpeechModelWarmupCoordinator( + private val executor: Executor = Executors.newSingleThreadExecutor(), + private val prepareModel: (SpeechModelRepository) -> Result, +) { + private data class Attempt( + val installation: String, + val future: Future>, + ) + + private val lock = Any() + private var attempt: Attempt? = null + private var currentState: SpeechModelWarmupState = SpeechModelWarmupState.Idle + + fun state(): SpeechModelWarmupState = synchronized(lock) { currentState } + + fun warmUp(repository: SpeechModelRepository) { + prepare(repository, retryFailed = false) + } + + fun prepare( + repository: SpeechModelRepository, + retryFailed: Boolean, + ): Future> { + val installation = installationIdentity(repository) + synchronized(lock) { + attempt?.takeIf { it.installation == installation }?.let { active -> + if (currentState !is SpeechModelWarmupState.Failed || !retryFailed) { + return active.future + } + } + + lateinit var task: FutureTask> + task = FutureTask(Callable { + val result = prepareModel(repository) + synchronized(lock) { + if (attempt?.future === task) { + currentState = result.fold( + onSuccess = { SpeechModelWarmupState.Ready(installation) }, + onFailure = { error -> + SpeechModelWarmupState.Failed( + installation, + error.message ?: "Speech model preparation failed", + ) + }, + ) + } + } + result + }) + attempt = Attempt(installation, task) + currentState = SpeechModelWarmupState.Preparing(installation) + executor.execute(task) + return task + } + } + + fun invalidate() { + synchronized(lock) { + attempt = null + currentState = SpeechModelWarmupState.Idle + } + } + + private fun installationIdentity(repository: SpeechModelRepository): String = + "${repository.descriptor.backend}:${repository.descriptor.catalogId}:" + + "${repository.manifest.version}:${repository.installedDirectory.canonicalPath}" +} + +internal object SpeechModelWarmup { + private val coordinator = SpeechModelWarmupCoordinator { repository -> + SpeechModelPreparation.prepare(repository, NativeTranscriptionBridge::initialize) + } + + fun warmUp(repository: SpeechModelRepository) { + coordinator.warmUp(repository) + } + + fun prepare( + repository: SpeechModelRepository, + retryFailed: Boolean, + ): Future> = coordinator.prepare(repository, retryFailed) + + fun invalidate() { + SpeechModelPreparation.invalidate(NativeTranscriptionBridge::reset) + coordinator.invalidate() + } +} diff --git a/app/src/main/java/me/maxistar/voiceinbox/TranscriptionWorker.kt b/app/src/main/java/me/maxistar/voiceinbox/TranscriptionWorker.kt index dee7158..2a4d74e 100644 --- a/app/src/main/java/me/maxistar/voiceinbox/TranscriptionWorker.kt +++ b/app/src/main/java/me/maxistar/voiceinbox/TranscriptionWorker.kt @@ -42,7 +42,7 @@ class TranscriptionWorker( applicationContext.noBackupFilesDir.resolve("models"), ) publish("Preparing speech model", null, null, 0, 0, null, null) - SpeechModelPreparation.prepare(modelRepository, NativeTranscriptionBridge::initialize) + SpeechModelWarmup.prepare(modelRepository, retryFailed = true).get() .getOrElse { return@withContext failure(it.message ?: "Speech model preparation failed") } val batch = BatchTranscriptionUseCase( diff --git a/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardAudioRecorder.kt b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardAudioRecorder.kt new file mode 100644 index 0000000..197d42f --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardAudioRecorder.kt @@ -0,0 +1,109 @@ +package me.maxistar.voiceinbox + +import android.media.AudioFormat +import android.media.AudioRecord +import android.media.MediaRecorder +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.Future + +class VoiceKeyboardAudioRecorder( + private val sampleRate: Int = SAMPLE_RATE, + private val recorderFactory: (Int) -> AudioRecord = { bufferSize -> + AudioRecord.Builder() + .setAudioSource(MediaRecorder.AudioSource.VOICE_RECOGNITION) + .setAudioFormat( + AudioFormat.Builder() + .setSampleRate(SAMPLE_RATE) + .setChannelMask(AudioFormat.CHANNEL_IN_MONO) + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .build(), + ) + .setBufferSizeInBytes(bufferSize) + .build() + }, + private val readerExecutor: ExecutorService = Executors.newSingleThreadExecutor(), +) { + private val lock = Any() + private val chunks = mutableListOf() + private var recorder: AudioRecord? = null + private var reader: Future<*>? = null + private var reading = false + + fun start(): Result = runCatching { + synchronized(lock) { + check(!reading) { "Microphone recording is already active" } + val minimum = AudioRecord.getMinBufferSize( + sampleRate, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + ) + check(minimum > 0) { "This device does not support 16 kHz mono recording" } + val created = recorderFactory(minimum * 2) + try { + check(created.state == AudioRecord.STATE_INITIALIZED) { "Microphone recorder could not initialize" } + chunks.clear() + recorder = created + reading = true + created.startRecording() + reader = readerExecutor.submit { readSamples(created) } + } catch (error: Throwable) { + recorder = null + reading = false + created.release() + throw error + } + } + } + + fun stop(): Result = runCatching { + val activeRecorder = synchronized(lock) { + reading = false + recorder + } ?: return@runCatching FloatArray(0) + runCatching { activeRecorder.stop() } + val readerResult = runCatching { reader?.get() } + val result = synchronized(lock) { + recorder = null + reader = null + activeRecorder.release() + val sampleCount = chunks.sumOf(FloatArray::size) + FloatArray(sampleCount).also { result -> + var offset = 0 + chunks.forEach { chunk -> + chunk.copyInto(result, destinationOffset = offset) + offset += chunk.size + } + chunks.clear() + } + } + readerResult.getOrThrow() + result + } + + fun cancel() { + stop() + } + + fun close() { + cancel() + readerExecutor.shutdownNow() + } + + private fun readSamples(activeRecorder: AudioRecord) { + val buffer = ShortArray(READ_BUFFER_SAMPLES) + while (synchronized(lock) { reading }) { + val read = activeRecorder.read(buffer, 0, buffer.size, AudioRecord.READ_BLOCKING) + if (read <= 0) continue + val normalized = FloatArray(read) { index -> buffer[index] / 32768f } + synchronized(lock) { + if (reading) chunks += normalized + } + } + } + + private companion object { + const val SAMPLE_RATE = 16_000 + const val READ_BUFFER_SAMPLES = 2_048 + } +} diff --git a/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardBackspace.kt b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardBackspace.kt new file mode 100644 index 0000000..e9dd7f2 --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardBackspace.kt @@ -0,0 +1,22 @@ +package me.maxistar.voiceinbox + +import android.view.inputmethod.InputConnection + +internal object VoiceKeyboardBackspace { + fun deleteOne(connection: InputConnection?): Boolean { + connection ?: return false + if (!shouldAttemptDeletion(connection.getTextBeforeCursor(1, 0))) return false + return deleteOne( + deleteCodePoint = { connection.deleteSurroundingTextInCodePoints(1, 0) }, + deleteCodeUnit = { connection.deleteSurroundingText(1, 0) }, + ) + } + + internal fun shouldAttemptDeletion(textBeforeCursor: CharSequence?): Boolean = + textBeforeCursor?.isNotEmpty() ?: true + + internal fun deleteOne( + deleteCodePoint: () -> Boolean, + deleteCodeUnit: () -> Boolean, + ): Boolean = deleteCodePoint() || deleteCodeUnit() +} diff --git a/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardBackspaceRepeater.kt b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardBackspaceRepeater.kt new file mode 100644 index 0000000..c5e42f8 --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardBackspaceRepeater.kt @@ -0,0 +1,47 @@ +package me.maxistar.voiceinbox + +internal interface VoiceKeyboardRepeatScheduler { + fun postDelayed(runnable: Runnable, delayMillis: Long) + fun removeCallbacks(runnable: Runnable) +} + +internal class VoiceKeyboardBackspaceRepeater( + private val scheduler: VoiceKeyboardRepeatScheduler, + private val deleteOne: () -> Boolean, + private val repeatIntervalMillis: Long = DEFAULT_REPEAT_INTERVAL_MILLIS, +) { + private var repeating = false + + private val repeatRunnable = object : Runnable { + override fun run() { + if (!repeating) return + if (!deleteOne()) { + cancel() + return + } + scheduler.postDelayed(this, repeatIntervalMillis) + } + } + + fun start() { + if (repeating) return + repeating = true + if (!deleteOne()) { + cancel() + return + } + scheduler.postDelayed(repeatRunnable, repeatIntervalMillis) + } + + fun cancel() { + if (!repeating) return + repeating = false + scheduler.removeCallbacks(repeatRunnable) + } + + fun isRepeating(): Boolean = repeating + + private companion object { + const val DEFAULT_REPEAT_INTERVAL_MILLIS = 55L + } +} diff --git a/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardController.kt b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardController.kt new file mode 100644 index 0000000..5909f18 --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardController.kt @@ -0,0 +1,69 @@ +package me.maxistar.voiceinbox + +enum class VoiceKeyboardPhase { + IDLE, + PREPARING, + RECORDING, + TRANSCRIBING, + RESULT_PENDING, + ERROR, +} + +/** Keeps IME request state deterministic and independent from Android view callbacks. */ +class VoiceKeyboardController { + var phase: VoiceKeyboardPhase = VoiceKeyboardPhase.IDLE + private set + + var pendingText: String? = null + private set + + fun beginPreparation(): Boolean { + if (phase !in setOf(VoiceKeyboardPhase.IDLE, VoiceKeyboardPhase.ERROR)) return false + phase = VoiceKeyboardPhase.PREPARING + return true + } + + fun recordingStarted(): Boolean { + if (phase != VoiceKeyboardPhase.PREPARING) return false + phase = VoiceKeyboardPhase.RECORDING + return true + } + + fun recordingStopped(): Boolean { + if (phase != VoiceKeyboardPhase.RECORDING) return false + phase = VoiceKeyboardPhase.TRANSCRIBING + return true + } + + fun completeTranscription(text: String?): String? { + if (phase != VoiceKeyboardPhase.TRANSCRIBING) return null + phase = VoiceKeyboardPhase.IDLE + return text?.trim()?.takeIf(String::isNotEmpty) + } + + fun deferResult(text: String) { + pendingText = text + phase = VoiceKeyboardPhase.RESULT_PENDING + } + + fun pendingResult(): String? = pendingText + + fun markPendingCommitted() { + pendingText = null + phase = VoiceKeyboardPhase.IDLE + } + + fun dismissPendingResult() { + pendingText = null + if (phase == VoiceKeyboardPhase.RESULT_PENDING) phase = VoiceKeyboardPhase.IDLE + } + + fun fail() { + phase = VoiceKeyboardPhase.ERROR + } + + fun cancel() { + pendingText = null + phase = VoiceKeyboardPhase.IDLE + } +} diff --git a/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardInputMethodService.kt b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardInputMethodService.kt new file mode 100644 index 0000000..213ef7a --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardInputMethodService.kt @@ -0,0 +1,368 @@ +package me.maxistar.voiceinbox + +import android.Manifest +import android.content.Intent +import android.content.pm.PackageManager +import android.inputmethodservice.InputMethodService +import android.os.Build +import android.os.Handler +import android.os.Looper +import android.view.KeyEvent +import android.view.LayoutInflater +import android.view.MotionEvent +import android.view.View +import android.view.inputmethod.EditorInfo +import android.view.inputmethod.InputConnection +import android.view.inputmethod.InputMethodManager +import android.widget.Button +import android.widget.ImageButton +import android.widget.ProgressBar +import android.widget.TextView +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +class VoiceKeyboardInputMethodService : InputMethodService() { + private val controller = VoiceKeyboardController() + private val mainHandler = Handler(Looper.getMainLooper()) + private val workExecutor: ExecutorService = Executors.newSingleThreadExecutor() + private val backspaceRepeater = VoiceKeyboardBackspaceRepeater( + scheduler = object : VoiceKeyboardRepeatScheduler { + override fun postDelayed(runnable: Runnable, delayMillis: Long) { + mainHandler.postDelayed(runnable, delayMillis) + } + + override fun removeCallbacks(runnable: Runnable) { + mainHandler.removeCallbacks(runnable) + } + }, + deleteOne = ::deleteOneBackspace, + ) + private lateinit var recorder: VoiceKeyboardAudioRecorder + private var inputActive = false + private var requestGeneration = 0L + private var warmUpGeneration = 0L + private var warmUpActive = false + + private var statusView: TextView? = null + private var progressView: ProgressBar? = null + private var recordButton: ImageButton? = null + private var dismissButton: Button? = null + private var setupButton: Button? = null + private var backspaceButton: ImageButton? = null + private var spaceButton: ImageButton? = null + private var enterButton: ImageButton? = null + private var nextKeyboardButton: ImageButton? = null + + override fun onCreate() { + super.onCreate() + recorder = VoiceKeyboardAudioRecorder() + } + + override fun onCreateInputView(): View { + val view = LayoutInflater.from(this).inflate(R.layout.input_view_voice_keyboard, null) + statusView = view.findViewById(R.id.voiceKeyboardStatus) + progressView = view.findViewById(R.id.voiceKeyboardProgress) + recordButton = view.findViewById(R.id.voiceKeyboardRecord) + dismissButton = view.findViewById(R.id.voiceKeyboardDismiss) + setupButton = view.findViewById(R.id.voiceKeyboardSetup) + backspaceButton = view.findViewById(R.id.voiceKeyboardBackspace) + spaceButton = view.findViewById(R.id.voiceKeyboardSpace) + enterButton = view.findViewById(R.id.voiceKeyboardEnter) + nextKeyboardButton = view.findViewById(R.id.voiceKeyboardNextKeyboard) + + recordButton?.setOnClickListener { handleRecordButton() } + dismissButton?.setOnClickListener { + controller.dismissPendingResult() + render(R.string.voice_keyboard_ready) + } + setupButton?.setOnClickListener { openVoiceInboxSetup() } + backspaceButton?.apply { + setOnClickListener { + if (!backspaceRepeater.isRepeating()) deleteOneBackspace() + } + setOnLongClickListener { + if (isEditorReady()) backspaceRepeater.start() + true + } + setOnTouchListener { _, event -> + if (event.actionMasked == MotionEvent.ACTION_UP || event.actionMasked == MotionEvent.ACTION_CANCEL) { + backspaceRepeater.cancel() + } + false + } + } + spaceButton?.setOnClickListener { currentInputConnection?.commitText(" ", 1) } + enterButton?.setOnClickListener(::performEnterAction) + nextKeyboardButton?.setOnClickListener { switchKeyboard() } + render(R.string.voice_keyboard_ready) + return view + } + + override fun onStartInput(attribute: EditorInfo, restarting: Boolean) { + super.onStartInput(attribute, restarting) + inputActive = true + } + + override fun onStartInputView(info: EditorInfo, restarting: Boolean) { + super.onStartInputView(info, restarting) + inputActive = true + flushPendingResult() + mainHandler.post(::startVisibleWarmUp) + } + + override fun onFinishInput() { + inputActive = false + warmUpGeneration += 1 + warmUpActive = false + backspaceRepeater.cancel() + super.onFinishInput() + } + + override fun onFinishInputView(finishingInput: Boolean) { + backspaceRepeater.cancel() + super.onFinishInputView(finishingInput) + } + + override fun onDestroy() { + requestGeneration += 1 + warmUpGeneration += 1 + backspaceRepeater.cancel() + recorder.close() + controller.cancel() + workExecutor.shutdownNow() + super.onDestroy() + } + + private fun handleRecordButton() { + if (warmUpActive) return + when (controller.phase) { + VoiceKeyboardPhase.RECORDING -> stopRecording() + VoiceKeyboardPhase.PREPARING, + VoiceKeyboardPhase.TRANSCRIBING, + -> cancelCurrentRequest() + VoiceKeyboardPhase.RESULT_PENDING -> { + controller.dismissPendingResult() + render(R.string.voice_keyboard_ready) + } + VoiceKeyboardPhase.IDLE, + VoiceKeyboardPhase.ERROR, + -> prepareAndStartRecording() + } + } + + private fun prepareAndStartRecording() { + if (checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { + controller.fail() + render(R.string.voice_keyboard_microphone_permission_missing, showSetup = true) + return + } + if (!controller.beginPreparation()) return + val generation = ++requestGeneration + render(R.string.voice_keyboard_preparing) + workExecutor.execute { + val preparation = prepareModelForDictation() + mainHandler.post { + if (generation != requestGeneration || controller.phase != VoiceKeyboardPhase.PREPARING) return@post + preparation.onSuccess { + recorder.start().onSuccess { + if (controller.recordingStarted()) { + render(R.string.voice_keyboard_listening) + mainHandler.postDelayed({ stopForDurationLimit(generation) }, MAX_PHRASE_DURATION_MS) + } + }.onFailure { + controller.fail() + render(R.string.voice_keyboard_microphone_unavailable, showSetup = false) + } + }.onFailure { + controller.fail() + render(R.string.voice_keyboard_model_unavailable, showSetup = true) + } + } + } + } + + private fun startVisibleWarmUp() { + if (warmUpActive || controller.phase != VoiceKeyboardPhase.IDLE) return + val generation = ++warmUpGeneration + warmUpActive = true + render(R.string.voice_keyboard_preparing) + workExecutor.execute { + val repository = SpeechModelRepository.forActive(noBackupFilesDir.resolve("models")) + val preparation = SpeechModelWarmup.prepare(repository, retryFailed = false).get() + mainHandler.post { + if (generation != warmUpGeneration) return@post + warmUpActive = false + render(R.string.voice_keyboard_ready) + } + } + } + + private fun prepareModelForDictation(): Result = runCatching { + val repository = SpeechModelRepository.forActive(noBackupFilesDir.resolve("models")) + SpeechModelWarmup.prepare(repository, retryFailed = true).get().getOrThrow() + } + + private fun stopForDurationLimit(generation: Long) { + if (generation == requestGeneration && controller.phase == VoiceKeyboardPhase.RECORDING) { + stopRecording() + } + } + + private fun stopRecording() { + if (!controller.recordingStopped()) return + mainHandler.removeCallbacksAndMessages(null) + val generation = requestGeneration + render(R.string.voice_keyboard_transcribing) + workExecutor.execute { + val result = recorder.stop().mapCatching { samples -> + if (samples.isEmpty()) null else NativeTranscriptionBridge.transcribeChunk(samples)?.text + } + mainHandler.post { + if (generation != requestGeneration || controller.phase != VoiceKeyboardPhase.TRANSCRIBING) return@post + result.onSuccess { text -> + val completed = controller.completeTranscription(text) + when { + completed == null -> render(R.string.voice_keyboard_no_speech) + commitToCurrentEditor(completed) -> render(R.string.voice_keyboard_ready) + else -> { + controller.deferResult(completed) + render(R.string.voice_keyboard_result_pending) + } + } + }.onFailure { + controller.fail() + render(R.string.voice_keyboard_transcription_failed) + } + } + } + } + + private fun cancelCurrentRequest() { + requestGeneration += 1 + mainHandler.removeCallbacksAndMessages(null) + if (controller.phase == VoiceKeyboardPhase.RECORDING) { + workExecutor.execute { recorder.cancel() } + } + controller.cancel() + render(R.string.voice_keyboard_ready) + } + + private fun flushPendingResult() { + val pending = controller.pendingResult() ?: return + if (commitToCurrentEditor(pending)) { + controller.markPendingCommitted() + render(R.string.voice_keyboard_ready) + } + } + + private fun commitToCurrentEditor(text: String): Boolean { + if (!inputActive || text.isBlank()) return false + return currentInputConnection?.commitText(text, 1) == true + } + + private fun deleteOneBackspace(): Boolean { + if (!isEditorReady()) return false + return VoiceKeyboardBackspace.deleteOne(currentInputConnection) + } + + private fun isEditorReady(): Boolean = inputActive && controller.phase !in setOf( + VoiceKeyboardPhase.PREPARING, + VoiceKeyboardPhase.TRANSCRIBING, + ) + + private fun performEnterAction(@Suppress("UNUSED_PARAMETER") view: View) { + val connection = currentInputConnection ?: return + val info = currentInputEditorInfo + val action = info?.imeOptions?.and(EditorInfo.IME_MASK_ACTION) ?: EditorInfo.IME_ACTION_NONE + val supportsAction = action in setOf( + EditorInfo.IME_ACTION_GO, + EditorInfo.IME_ACTION_SEARCH, + EditorInfo.IME_ACTION_SEND, + EditorInfo.IME_ACTION_NEXT, + ) && info?.imeOptions?.and(EditorInfo.IME_FLAG_NO_ENTER_ACTION) == 0 + if (supportsAction) { + connection.performEditorAction(action) + } else { + connection.sendKeyEvent(KeyEvent(KeyEvent.ACTION_DOWN, KeyEvent.KEYCODE_ENTER)) + connection.sendKeyEvent(KeyEvent(KeyEvent.ACTION_UP, KeyEvent.KEYCODE_ENTER)) + } + } + + private fun switchKeyboard() { + val restored = Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && switchToPreviousInputMethod() + if (VoiceKeyboardSwitching.returnAction(restored) == VoiceKeyboardReturnAction.SHOW_PICKER) { + getSystemService(InputMethodManager::class.java).showInputMethodPicker() + } + } + + private fun openVoiceInboxSetup() { + startActivity( + Intent(this, MainActivity::class.java) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_SINGLE_TOP) + .putExtra(VoiceKeyboardSetup.EXTRA_REQUEST_MICROPHONE_PERMISSION, true), + ) + } + + private fun render(status: Int, showSetup: Boolean = false) { + statusView?.setText(status) + val phase = controller.phase + recordButton?.apply { + val busy = warmUpActive || phase in setOf(VoiceKeyboardPhase.PREPARING, VoiceKeyboardPhase.TRANSCRIBING) + isEnabled = phase != VoiceKeyboardPhase.RESULT_PENDING && !warmUpActive + alpha = if (isEnabled) 1f else 0.6f + background = getDrawable( + when { + warmUpActive -> R.drawable.voice_keyboard_mic_busy + else -> when (phase) { + VoiceKeyboardPhase.RECORDING -> R.drawable.voice_keyboard_mic_recording + VoiceKeyboardPhase.PREPARING, + VoiceKeyboardPhase.TRANSCRIBING, + VoiceKeyboardPhase.RESULT_PENDING, + -> R.drawable.voice_keyboard_mic_busy + VoiceKeyboardPhase.ERROR -> R.drawable.voice_keyboard_mic_error + VoiceKeyboardPhase.IDLE -> R.drawable.voice_keyboard_mic_idle + } + }, + ) + setImageResource( + when { + warmUpActive -> R.drawable.ic_voice_keyboard_mic + else -> when (phase) { + VoiceKeyboardPhase.RECORDING, + VoiceKeyboardPhase.PREPARING, + VoiceKeyboardPhase.TRANSCRIBING, + -> R.drawable.ic_voice_keyboard_stop + else -> R.drawable.ic_voice_keyboard_mic + } + }, + ) + contentDescription = context.getString( + when { + warmUpActive -> R.string.voice_keyboard_preparing + else -> when (phase) { + VoiceKeyboardPhase.RECORDING -> R.string.voice_keyboard_stop + VoiceKeyboardPhase.PREPARING, + VoiceKeyboardPhase.TRANSCRIBING, + -> R.string.voice_keyboard_cancel + VoiceKeyboardPhase.ERROR -> R.string.voice_keyboard_record + else -> R.string.voice_keyboard_record + } + }, + ) + keepScreenOn = phase == VoiceKeyboardPhase.RECORDING + progressView?.visibility = if (busy) View.VISIBLE else View.GONE + } + dismissButton?.visibility = if (phase == VoiceKeyboardPhase.RESULT_PENDING) View.VISIBLE else View.GONE + setupButton?.visibility = if (showSetup) View.VISIBLE else View.GONE + val editorReady = isEditorReady() + if (!editorReady) backspaceRepeater.cancel() + backspaceButton?.isEnabled = editorReady + spaceButton?.isEnabled = editorReady + enterButton?.isEnabled = editorReady + nextKeyboardButton?.isEnabled = phase != VoiceKeyboardPhase.RECORDING + } + + private companion object { + const val MAX_PHRASE_DURATION_MS = 30_000L + } +} diff --git a/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardSetup.kt b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardSetup.kt new file mode 100644 index 0000000..39bc0b1 --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardSetup.kt @@ -0,0 +1,6 @@ +package me.maxistar.voiceinbox + +object VoiceKeyboardSetup { + const val EXTRA_REQUEST_MICROPHONE_PERMISSION = + "me.maxistar.voiceinbox.extra.REQUEST_MICROPHONE_PERMISSION" +} diff --git a/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardSwitching.kt b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardSwitching.kt new file mode 100644 index 0000000..4c75512 --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardSwitching.kt @@ -0,0 +1,15 @@ +package me.maxistar.voiceinbox + +internal enum class VoiceKeyboardReturnAction { + RESTORED_PREVIOUS, + SHOW_PICKER, +} + +internal object VoiceKeyboardSwitching { + fun returnAction(previousKeyboardRestored: Boolean): VoiceKeyboardReturnAction = + if (previousKeyboardRestored) { + VoiceKeyboardReturnAction.RESTORED_PREVIOUS + } else { + VoiceKeyboardReturnAction.SHOW_PICKER + } +} diff --git a/app/src/main/res/drawable/ic_voice_keyboard_delete.xml b/app/src/main/res/drawable/ic_voice_keyboard_delete.xml new file mode 100644 index 0000000..2a7e8db --- /dev/null +++ b/app/src/main/res/drawable/ic_voice_keyboard_delete.xml @@ -0,0 +1,7 @@ + + + diff --git a/app/src/main/res/drawable/ic_voice_keyboard_enter.xml b/app/src/main/res/drawable/ic_voice_keyboard_enter.xml new file mode 100644 index 0000000..f3840f8 --- /dev/null +++ b/app/src/main/res/drawable/ic_voice_keyboard_enter.xml @@ -0,0 +1,7 @@ + + + diff --git a/app/src/main/res/drawable/ic_voice_keyboard_keyboard.xml b/app/src/main/res/drawable/ic_voice_keyboard_keyboard.xml new file mode 100644 index 0000000..fc64afc --- /dev/null +++ b/app/src/main/res/drawable/ic_voice_keyboard_keyboard.xml @@ -0,0 +1,7 @@ + + + diff --git a/app/src/main/res/drawable/ic_voice_keyboard_mic.xml b/app/src/main/res/drawable/ic_voice_keyboard_mic.xml new file mode 100644 index 0000000..9fcc523 --- /dev/null +++ b/app/src/main/res/drawable/ic_voice_keyboard_mic.xml @@ -0,0 +1,7 @@ + + + diff --git a/app/src/main/res/drawable/ic_voice_keyboard_space.xml b/app/src/main/res/drawable/ic_voice_keyboard_space.xml new file mode 100644 index 0000000..4e9308b --- /dev/null +++ b/app/src/main/res/drawable/ic_voice_keyboard_space.xml @@ -0,0 +1,7 @@ + + + diff --git a/app/src/main/res/drawable/ic_voice_keyboard_stop.xml b/app/src/main/res/drawable/ic_voice_keyboard_stop.xml new file mode 100644 index 0000000..02fa7a2 --- /dev/null +++ b/app/src/main/res/drawable/ic_voice_keyboard_stop.xml @@ -0,0 +1,7 @@ + + + diff --git a/app/src/main/res/drawable/voice_keyboard_key.xml b/app/src/main/res/drawable/voice_keyboard_key.xml new file mode 100644 index 0000000..fb02cb2 --- /dev/null +++ b/app/src/main/res/drawable/voice_keyboard_key.xml @@ -0,0 +1,10 @@ + + + + + + + + + diff --git a/app/src/main/res/drawable/voice_keyboard_mic_busy.xml b/app/src/main/res/drawable/voice_keyboard_mic_busy.xml new file mode 100644 index 0000000..49e134e --- /dev/null +++ b/app/src/main/res/drawable/voice_keyboard_mic_busy.xml @@ -0,0 +1,4 @@ + + + + diff --git a/app/src/main/res/drawable/voice_keyboard_mic_error.xml b/app/src/main/res/drawable/voice_keyboard_mic_error.xml new file mode 100644 index 0000000..b8d5d3e --- /dev/null +++ b/app/src/main/res/drawable/voice_keyboard_mic_error.xml @@ -0,0 +1,9 @@ + + + + + + + + diff --git a/app/src/main/res/drawable/voice_keyboard_mic_idle.xml b/app/src/main/res/drawable/voice_keyboard_mic_idle.xml new file mode 100644 index 0000000..af97f26 --- /dev/null +++ b/app/src/main/res/drawable/voice_keyboard_mic_idle.xml @@ -0,0 +1,9 @@ + + + + + + + + diff --git a/app/src/main/res/drawable/voice_keyboard_mic_recording.xml b/app/src/main/res/drawable/voice_keyboard_mic_recording.xml new file mode 100644 index 0000000..718060c --- /dev/null +++ b/app/src/main/res/drawable/voice_keyboard_mic_recording.xml @@ -0,0 +1,9 @@ + + + + + + + + diff --git a/app/src/main/res/drawable/voice_keyboard_secondary_action.xml b/app/src/main/res/drawable/voice_keyboard_secondary_action.xml new file mode 100644 index 0000000..a1ffb5d --- /dev/null +++ b/app/src/main/res/drawable/voice_keyboard_secondary_action.xml @@ -0,0 +1,11 @@ + + + + + + + + + + diff --git a/app/src/main/res/drawable/voice_keyboard_surface.xml b/app/src/main/res/drawable/voice_keyboard_surface.xml new file mode 100644 index 0000000..b94620a --- /dev/null +++ b/app/src/main/res/drawable/voice_keyboard_surface.xml @@ -0,0 +1,4 @@ + + + + diff --git a/app/src/main/res/layout/input_view_voice_keyboard.xml b/app/src/main/res/layout/input_view_voice_keyboard.xml new file mode 100644 index 0000000..a0725fd --- /dev/null +++ b/app/src/main/res/layout/input_view_voice_keyboard.xml @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + +