From fd0bb57338d5b1a6ee79e6c9bb7c44990503cf73 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sun, 14 Sep 2025 20:10:57 +0300 Subject: [PATCH] Background captcha resolving --- app/build.gradle | 1 + .../core/exceptions/resolve/CaptchaHandler.kt | 32 +++++-- .../webview/CaptchaContinuationClient.kt | 30 +++++++ .../ContinuationResumeWebViewClient.kt | 6 +- .../core/network/webview/WebViewExecutor.kt | 88 ++++++++++++++++--- .../core/parser/MangaLoaderContextImpl.kt | 17 +--- .../kotatsu/core/ui/image/FaviconView.kt | 4 +- 7 files changed, 137 insertions(+), 41 deletions(-) create mode 100644 app/src/main/kotlin/org/koitharu/kotatsu/core/network/webview/CaptchaContinuationClient.kt diff --git a/app/build.gradle b/app/build.gradle index ce1427211..c5e532c44 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -87,6 +87,7 @@ android { '-opt-in=coil3.annotation.InternalCoilApi', '-opt-in=kotlinx.serialization.ExperimentalSerializationApi', '-Xjspecify-annotations=strict', + '-Xannotation-default-target=first-only', '-Xtype-enhancement-improvements-strict-mode' ] } diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/core/exceptions/resolve/CaptchaHandler.kt b/app/src/main/kotlin/org/koitharu/kotatsu/core/exceptions/resolve/CaptchaHandler.kt index f73dbb8f3..ceae3d573 100644 --- a/app/src/main/kotlin/org/koitharu/kotatsu/core/exceptions/resolve/CaptchaHandler.kt +++ b/app/src/main/kotlin/org/koitharu/kotatsu/core/exceptions/resolve/CaptchaHandler.kt @@ -7,6 +7,7 @@ import android.content.Context import android.content.Intent import android.os.Build import android.provider.Settings +import androidx.annotation.CheckResult import androidx.annotation.RequiresPermission import androidx.collection.MutableScatterMap import androidx.core.app.NotificationChannelCompat @@ -43,6 +44,7 @@ import org.koitharu.kotatsu.core.model.UnknownMangaSource import org.koitharu.kotatsu.core.model.getTitle import org.koitharu.kotatsu.core.model.isNsfw import org.koitharu.kotatsu.core.nav.AppRouter +import org.koitharu.kotatsu.core.network.webview.WebViewExecutor import org.koitharu.kotatsu.core.parser.favicon.faviconUri import org.koitharu.kotatsu.core.prefs.SourceSettings import org.koitharu.kotatsu.core.util.ext.checkNotificationPermission @@ -65,11 +67,13 @@ class CaptchaHandler @Inject constructor( @LocalizedAppContext private val context: Context, private val databaseProvider: Provider, private val coilProvider: Provider, + private val webViewExecutor: WebViewExecutor, ) : EventListener() { private val exceptionMap = MutableScatterMap() private val mutex = Mutex() + @CheckResult suspend fun handle(exception: CloudFlareException): Boolean = handleException(exception.source, exception, true) suspend fun discard(source: MangaSource) { @@ -79,10 +83,18 @@ class CaptchaHandler @Inject constructor( override fun onError(request: ImageRequest, result: ErrorResult) { super.onError(request, result) val e = result.throwable - if (e is CloudFlareException && request.extras[ignoreCaptchaKey] != true) { + if (e is CloudFlareException) { val scope = request.lifecycle?.coroutineScope ?: processLifecycleScope scope.launch { - handleException(e.source, e, true) + if ( + handleException( + source = e.source, + exception = e, + notify = request.extras[suppressCaptchaKey] != true, + ) + ) { + coilProvider.get().enqueue(request) // TODO check if ok + } } } } @@ -90,11 +102,14 @@ class CaptchaHandler @Inject constructor( private suspend fun handleException( source: MangaSource, exception: CloudFlareException?, - notify: Boolean + notify: Boolean, ): Boolean = withContext(Dispatchers.Default) { if (source == UnknownMangaSource) { return@withContext false } + if (exception != null && webViewExecutor.tryResolveCaptcha(exception, RESOLVE_TIMEOUT)) { + return@withContext true + } mutex.withLock { var removedException: CloudFlareProtectedException? = null if (exception is CloudFlareProtectedException) { @@ -119,7 +134,7 @@ class CaptchaHandler @Inject constructor( notify(exceptions) } } - true + false } @RequiresPermission(Manifest.permission.POST_NOTIFICATIONS) @@ -234,7 +249,7 @@ class CaptchaHandler @Inject constructor( .data(source.faviconUri()) .allowHardware(false) .allowConversionToBitmap(true) - .ignoreCaptchaErrors() + .suppressCaptchaErrors() .mangaSourceExtra(source) .size(context.resources.getNotificationIconSize()) .scale(Scale.FILL) @@ -260,11 +275,11 @@ class CaptchaHandler @Inject constructor( companion object { - fun ImageRequest.Builder.ignoreCaptchaErrors() = apply { - extras[ignoreCaptchaKey] = true + fun ImageRequest.Builder.suppressCaptchaErrors() = apply { + extras[suppressCaptchaKey] = true } - val ignoreCaptchaKey = Extras.Key(false) + private val suppressCaptchaKey = Extras.Key(false) private const val CHANNEL_ID = "captcha" private const val TAG = CHANNEL_ID @@ -272,5 +287,6 @@ class CaptchaHandler @Inject constructor( private const val GROUP_NOTIFICATION_ID = 34 private const val SETTINGS_ACTION_CODE = 3 private const val ACTION_DISCARD = "org.koitharu.kotatsu.CAPTCHA_DISCARD" + private const val RESOLVE_TIMEOUT = 20_000L } } diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/core/network/webview/CaptchaContinuationClient.kt b/app/src/main/kotlin/org/koitharu/kotatsu/core/network/webview/CaptchaContinuationClient.kt new file mode 100644 index 000000000..247de3c07 --- /dev/null +++ b/app/src/main/kotlin/org/koitharu/kotatsu/core/network/webview/CaptchaContinuationClient.kt @@ -0,0 +1,30 @@ +package org.koitharu.kotatsu.core.network.webview + +import android.graphics.Bitmap +import android.webkit.WebView +import org.koitharu.kotatsu.core.network.cookies.MutableCookieJar +import org.koitharu.kotatsu.parsers.network.CloudFlareHelper +import kotlin.coroutines.Continuation + +class CaptchaContinuationClient( + private val cookieJar: MutableCookieJar, + private val targetUrl: String, + continuation: Continuation, +) : ContinuationResumeWebViewClient(continuation) { + + private val oldClearance = CloudFlareHelper.getClearanceCookie(cookieJar, targetUrl) + + override fun onPageFinished(view: WebView?, url: String?) = Unit + + override fun onPageStarted(view: WebView?, url: String?, favicon: Bitmap?) { + super.onPageStarted(view, url, favicon) + checkClearance(view) + } + + private fun checkClearance(view: WebView?) { + val clearance = CloudFlareHelper.getClearanceCookie(cookieJar, targetUrl) + if (clearance != null && clearance != oldClearance) { + resumeContinuation(view) + } + } +} diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/core/network/webview/ContinuationResumeWebViewClient.kt b/app/src/main/kotlin/org/koitharu/kotatsu/core/network/webview/ContinuationResumeWebViewClient.kt index c7d3a2794..4c991cac9 100644 --- a/app/src/main/kotlin/org/koitharu/kotatsu/core/network/webview/ContinuationResumeWebViewClient.kt +++ b/app/src/main/kotlin/org/koitharu/kotatsu/core/network/webview/ContinuationResumeWebViewClient.kt @@ -5,11 +5,15 @@ import android.webkit.WebViewClient import kotlin.coroutines.Continuation import kotlin.coroutines.resume -class ContinuationResumeWebViewClient( +open class ContinuationResumeWebViewClient( private val continuation: Continuation, ) : WebViewClient() { override fun onPageFinished(view: WebView?, url: String?) { + resumeContinuation(view) + } + + protected fun resumeContinuation(view: WebView?) { view?.webViewClient = WebViewClient() // reset to default continuation.resume(Unit) } diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/core/network/webview/WebViewExecutor.kt b/app/src/main/kotlin/org/koitharu/kotatsu/core/network/webview/WebViewExecutor.kt index 4a742d7ba..1fe47735d 100644 --- a/app/src/main/kotlin/org/koitharu/kotatsu/core/network/webview/WebViewExecutor.kt +++ b/app/src/main/kotlin/org/koitharu/kotatsu/core/network/webview/WebViewExecutor.kt @@ -1,31 +1,48 @@ package org.koitharu.kotatsu.core.network.webview import android.content.Context +import android.webkit.WebSettings import android.webkit.WebView -import androidx.annotation.MainThread import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import org.koitharu.kotatsu.core.exceptions.CloudFlareException +import org.koitharu.kotatsu.core.network.CommonHeaders +import org.koitharu.kotatsu.core.network.cookies.MutableCookieJar +import org.koitharu.kotatsu.core.network.proxy.ProxyProvider +import org.koitharu.kotatsu.core.parser.MangaRepository +import org.koitharu.kotatsu.core.parser.ParserMangaRepository import org.koitharu.kotatsu.core.util.ext.configureForParser import org.koitharu.kotatsu.core.util.ext.printStackTraceDebug -import org.koitharu.kotatsu.core.util.ext.sanitizeHeaderValue -import org.koitharu.kotatsu.parsers.util.nullIfEmpty +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.util.runCatchingCancellable import java.lang.ref.WeakReference import javax.inject.Inject +import javax.inject.Provider import javax.inject.Singleton import kotlin.coroutines.resume import kotlin.coroutines.suspendCoroutine @Singleton class WebViewExecutor @Inject constructor( - @ApplicationContext private val context: Context + @ApplicationContext private val context: Context, + private val proxyProvider: ProxyProvider, + private val cookieJar: MutableCookieJar, + private val mangaRepositoryFactoryProvider: Provider, ) { private var webViewCached: WeakReference? = null private val mutex = Mutex() + val defaultUserAgent: String? by lazy { + WebSettings.getDefaultUserAgent(context) + } + suspend fun evaluateJs(baseUrl: String?, script: String): String? = mutex.withLock { withContext(Dispatchers.Main.immediate) { val webView = obtainWebView() @@ -43,16 +60,59 @@ class WebViewExecutor @Inject constructor( } } - @MainThread - fun getDefaultUserAgent() = runCatching { - obtainWebView().settings.userAgentString.sanitizeHeaderValue().trim().nullIfEmpty() - }.onFailure { e -> - e.printStackTraceDebug() - }.getOrNull() + suspend fun tryResolveCaptcha(exception: CloudFlareException, timeout: Long): Boolean = mutex.withLock { + runCatchingCancellable { + withContext(Dispatchers.Main.immediate) { + val webView = obtainWebView() + try { + exception.source.getUserAgent()?.let { + webView.settings.userAgentString = it + } + coroutineScope { + withTimeout(timeout) { + suspendCancellableCoroutine { cont -> + webView.webViewClient = CaptchaContinuationClient( + cookieJar = cookieJar, + targetUrl = exception.url, + continuation = cont, + ) + cont.invokeOnCancellation { + webView.stopLoading() + } + webView.loadUrl(exception.url) + } + } + } + } finally { + webView.settings.userAgentString = defaultUserAgent + } + } + }.onFailure { e -> + exception.addSuppressed(e) + e.printStackTraceDebug() + }.isSuccess + } - @MainThread - private fun obtainWebView(): WebView = webViewCached?.get() ?: WebView(context).also { - it.configureForParser(null) - webViewCached = WeakReference(it) + private suspend fun obtainWebView(): WebView { + webViewCached?.get()?.let { + return it + } + return withContext(Dispatchers.Main.immediate) { + webViewCached?.get()?.let { + return@withContext it + } + WebView(context).also { + it.configureForParser(null) + webViewCached = WeakReference(it) + proxyProvider.applyWebViewConfig() + it.onResume() + it.resumeTimers() + } + } + } + + private fun MangaSource.getUserAgent(): String? { + val repository = mangaRepositoryFactoryProvider.get().create(this) as? ParserMangaRepository + return repository?.getRequestHeaders()?.get(CommonHeaders.USER_AGENT) } } diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt b/app/src/main/kotlin/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt index 49698c0f6..662a1976d 100644 --- a/app/src/main/kotlin/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt +++ b/app/src/main/kotlin/org/koitharu/kotatsu/core/parser/MangaLoaderContextImpl.kt @@ -5,8 +5,6 @@ import android.content.Context import android.util.Base64 import androidx.core.os.LocaleListCompat import dagger.hilt.android.qualifiers.ApplicationContext -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeout import okhttp3.MediaType.Companion.toMediaType import okhttp3.OkHttpClient @@ -33,7 +31,6 @@ import java.util.Locale import java.util.concurrent.TimeUnit import javax.inject.Inject import javax.inject.Singleton -import kotlin.coroutines.EmptyCoroutineContext @Singleton class MangaLoaderContextImpl @Inject constructor( @@ -43,7 +40,6 @@ class MangaLoaderContextImpl @Inject constructor( private val webViewExecutor: WebViewExecutor, ) : MangaLoaderContext() { - private val webViewUserAgent by lazy { obtainWebViewUserAgent() } private val jsTimeout = TimeUnit.SECONDS.toMillis(4) @Deprecated("Provide a base url") @@ -54,7 +50,7 @@ class MangaLoaderContextImpl @Inject constructor( webViewExecutor.evaluateJs(baseUrl, script) } - override fun getDefaultUserAgent(): String = webViewUserAgent + override fun getDefaultUserAgent(): String = webViewExecutor.defaultUserAgent ?: UserAgents.FIREFOX_MOBILE override fun getConfig(source: MangaSource): MangaSourceConfig { return SourceSettings(androidContext, source) @@ -91,15 +87,4 @@ class MangaLoaderContextImpl @Inject constructor( } override fun createBitmap(width: Int, height: Int): Bitmap = BitmapWrapper.create(width, height) - - private fun obtainWebViewUserAgent(): String { - val mainDispatcher = Dispatchers.Main.immediate - return if (!mainDispatcher.isDispatchNeeded(EmptyCoroutineContext)) { - webViewExecutor.getDefaultUserAgent() - } else { - runBlocking(mainDispatcher) { - webViewExecutor.getDefaultUserAgent() - } - } ?: UserAgents.FIREFOX_MOBILE - } } diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/core/ui/image/FaviconView.kt b/app/src/main/kotlin/org/koitharu/kotatsu/core/ui/image/FaviconView.kt index 564c31c38..ed18b792c 100644 --- a/app/src/main/kotlin/org/koitharu/kotatsu/core/ui/image/FaviconView.kt +++ b/app/src/main/kotlin/org/koitharu/kotatsu/core/ui/image/FaviconView.kt @@ -10,7 +10,7 @@ import coil3.asImage import coil3.request.Disposable import coil3.request.ImageRequest import org.koitharu.kotatsu.R -import org.koitharu.kotatsu.core.exceptions.resolve.CaptchaHandler.Companion.ignoreCaptchaErrors +import org.koitharu.kotatsu.core.exceptions.resolve.CaptchaHandler.Companion.suppressCaptchaErrors import org.koitharu.kotatsu.core.image.CoilImageView import org.koitharu.kotatsu.core.parser.favicon.faviconUri import org.koitharu.kotatsu.core.util.ext.isAnimationsEnabled @@ -57,7 +57,7 @@ class FaviconView @JvmOverloads constructor( .fallback(fallbackFactory) .placeholder(placeholderFactory) .mangaSourceExtra(mangaSource) - .ignoreCaptchaErrors() + .suppressCaptchaErrors() .build(), ) }