From 564f052a2f27c9bfdc45f02ca030d36d1d68af22 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Tue, 8 Mar 2022 14:03:12 +0200 Subject: [PATCH] Add Bato.To manga source #77 --- app/build.gradle | 1 + .../kotatsu/base/domain/MangaLoaderContext.kt | 18 +- .../kotatsu/core/model/MangaSource.kt | 1 + .../kotatsu/core/parser/ParserModule.kt | 1 + .../core/parser/site/BatoToRepository.kt | 306 ++++++++++++++++++ .../download/domain/DownloadManager.kt | 2 + .../kotatsu/list/ui/filter/FilterViewModel.kt | 11 +- .../koitharu/kotatsu/utils/ext/StringExt.kt | 15 +- .../core/parser/RemoteMangaRepositoryTest.kt | 1 - .../core/parser/RepositoryTestModule.kt | 10 +- 10 files changed, 359 insertions(+), 7 deletions(-) create mode 100644 app/src/main/java/org/koitharu/kotatsu/core/parser/site/BatoToRepository.kt diff --git a/app/build.gradle b/app/build.gradle index de3d3fa5b..451235f9e 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -108,6 +108,7 @@ dependencies { testImplementation 'junit:junit:4.13.2' testImplementation 'com.google.truth:truth:1.1.3' testImplementation 'org.json:json:20211205' + testImplementation 'io.webfolder:quickjs:1.1.0' testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.6.0' testImplementation 'io.insert-koin:koin-test-junit4:3.1.5' diff --git a/app/src/main/java/org/koitharu/kotatsu/base/domain/MangaLoaderContext.kt b/app/src/main/java/org/koitharu/kotatsu/base/domain/MangaLoaderContext.kt index 328b7be81..eb941a696 100644 --- a/app/src/main/java/org/koitharu/kotatsu/base/domain/MangaLoaderContext.kt +++ b/app/src/main/java/org/koitharu/kotatsu/base/domain/MangaLoaderContext.kt @@ -1,5 +1,9 @@ package org.koitharu.kotatsu.base.domain +import android.annotation.SuppressLint +import android.webkit.WebView +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import okhttp3.* import okhttp3.MediaType.Companion.toMediaType import okhttp3.RequestBody.Companion.toRequestBody @@ -11,7 +15,8 @@ import org.koitharu.kotatsu.core.model.MangaSource import org.koitharu.kotatsu.core.prefs.SourceSettings import org.koitharu.kotatsu.utils.ext.await import org.koitharu.kotatsu.utils.ext.parseJson - +import kotlin.coroutines.resume +import kotlin.coroutines.suspendCoroutine open class MangaLoaderContext( private val okHttp: OkHttpClient, @@ -80,5 +85,16 @@ open class MangaLoaderContext( return json } + @SuppressLint("SetJavaScriptEnabled") + open suspend fun evaluateJs(script: String): String? = withContext(Dispatchers.Main) { + val webView = WebView(get()) + webView.settings.javaScriptEnabled = true + suspendCoroutine { cont -> + webView.evaluateJavascript(script) { result -> + cont.resume(result?.takeUnless { it == "null" }) + } + } + } + open fun getSettings(source: MangaSource) = SourceSettings(get(), source) } \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt b/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt index a9fc84da5..5ec3ff85a 100644 --- a/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt +++ b/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt @@ -35,5 +35,6 @@ enum class MangaSource( EXHENTAI("ExHentai", null), MANGAOWL("MangaOwl", "en"), MANGADEX("MangaDex", null), + BATOTO("Bato.To", null), ; } \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/ParserModule.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/ParserModule.kt index 944bd1ac6..c1e145278 100644 --- a/app/src/main/java/org/koitharu/kotatsu/core/parser/ParserModule.kt +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/ParserModule.kt @@ -32,4 +32,5 @@ val parserModule factory(named(MangaSource.EXHENTAI)) { ExHentaiRepository(get()) } factory(named(MangaSource.MANGAOWL)) { MangaOwlRepository(get()) } factory(named(MangaSource.MANGADEX)) { MangaDexRepository(get()) } + factory(named(MangaSource.BATOTO)) { BatoToRepository(get()) } } \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/BatoToRepository.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/BatoToRepository.kt new file mode 100644 index 000000000..38e65adb3 --- /dev/null +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/BatoToRepository.kt @@ -0,0 +1,306 @@ +package org.koitharu.kotatsu.core.parser.site + +import android.util.Base64 +import androidx.collection.ArraySet +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.base.domain.MangaLoaderContext +import org.koitharu.kotatsu.core.model.* +import org.koitharu.kotatsu.core.parser.RemoteMangaRepository +import org.koitharu.kotatsu.utils.ext.* +import java.nio.charset.StandardCharsets +import java.security.MessageDigest +import java.util.* +import javax.crypto.Cipher +import javax.crypto.spec.IvParameterSpec +import javax.crypto.spec.SecretKeySpec + +private const val PAGE_SIZE = 60 +private const val PAGE_SIZE_SEARCH = 20 + +class BatoToRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepository(loaderContext) { + + override val source = MangaSource.BATOTO + + override val sortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL + ) + + override val defaultDomain: String = "bato.to" + + override suspend fun getList2( + offset: Int, + query: String?, + tags: Set?, + sortOrder: SortOrder? + ): List { + if (!query.isNullOrEmpty()) { + return search(offset, query) + } + val page = (offset / PAGE_SIZE) + 1 + + @Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT") + val url = buildString { + append("https://") + append(getDomain()) + append("/browse?sort=") + when (sortOrder) { + null, + SortOrder.UPDATED -> append("update.za") + SortOrder.POPULARITY -> append("views_a.za") + SortOrder.NEWEST -> append("create.za") + SortOrder.ALPHABETICAL -> append("title.az") + } + if (!tags.isNullOrEmpty()) { + append("&genres=") + appendAll(tags, ",") { it.key } + } + append("&page=") + append(page) + } + val body = loaderContext.httpGet(url).parseHtml().body() + val activePage = getActivePage(body) + if (activePage != page) { + return emptyList() + } + return parseList(body.getElementById("series-list") ?: parseFailed("Cannot find root")) + } + + override suspend fun getDetails(manga: Manga): Manga { + val root = loaderContext.httpGet(manga.url.withDomain()).parseHtml() + .getElementById("mainer") ?: parseFailed("Cannot find root") + val details = root.selectFirst(".detail-set") ?: parseFailed("Cannot find detail-set") + val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate { + it.child(0).text().trim() to it.child(1) + }.orEmpty() + return manga.copy( + title = root.selectFirst("h3.item-title")?.text() ?: manga.title, + isNsfw = !root.selectFirst("alert")?.getElementsContainingOwnText("NSFW").isNullOrEmpty(), + largeCoverUrl = details.selectFirst("img[src]")?.absUrl("src"), + description = details.getElementById("limit-height-body-summary") + ?.selectFirst(".limit-html") + ?.html(), + tags = manga.tags + attrs["Genres:"]?.parseTags().orEmpty(), + state = when (attrs["Release status:"]?.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> manga.state + }, + author = attrs["Authors:"]?.text()?.trim() ?: manga.author, + chapters = root.selectFirst(".episode-list") + ?.selectFirst(".main") + ?.children() + ?.reversed() + ?.mapIndexedNotNull { i, div -> + div.parseChapter(i) + }.orEmpty() + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.withDomain() + val scripts = loaderContext.httpGet(fullUrl).parseHtml().select("script") + for (script in scripts) { + val scriptSrc = script.html() + val p = scriptSrc.indexOf("const images =") + if (p == -1) continue + val start = scriptSrc.indexOf('[', p) + val end = scriptSrc.indexOf(';', start) + if (start == -1 || end == -1) { + continue + } + val images = JSONArray(scriptSrc.substring(start, end)) + val batoJs = scriptSrc.substringBetweenFirst("batojs =", ";")?.trim(' ', '"', '\n') + ?: parseFailed("Cannot find batojs") + val server = scriptSrc.substringBetweenFirst("server =", ";")?.trim(' ', '"', '\n') + ?: parseFailed("Cannot find server") + val password = loaderContext.evaluateJs(batoJs)?.removeSurrounding('"') + ?: parseFailed("Cannot evaluate batojs") + val serverDecrypted = decryptAES(server, password).removeSurrounding('"') + val result = ArrayList(images.length()) + repeat(images.length()) { i -> + val url = images.getString(i) + result += MangaPage( + id = generateUid(url), + url = if (url.startsWith("http")) url else "$serverDecrypted$url", + referer = fullUrl, + preview = null, + source = source, + ) + } + return result + } + parseFailed("Cannot find images list") + } + + override suspend fun getTags(): Set { + val scripts = loaderContext.httpGet( + "https://${getDomain()}/browse" + ).parseHtml().select("script") + for (script in scripts) { + val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue + val jo = JSONObject(genres) + val result = ArraySet(jo.length()) + jo.keys().forEach { key -> + val item = jo.getJSONObject(key) + result += MangaTag( + title = item.getString("text").toTitleCase(), + key = item.getString("file"), + source = source, + ) + } + return result + } + parseFailed("Cannot find gernes list") + } + + override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0" + + private suspend fun search(offset: Int, query: String): List { + val page = (offset / PAGE_SIZE_SEARCH) + 1 + val url = buildString { + append("https://") + append(getDomain()) + append("/search?word=") + append(query.replace(' ', '+')) + append("&page=") + append(page) + } + val body = loaderContext.httpGet(url).parseHtml().body() + val activePage = getActivePage(body) + if (activePage != page) { + return emptyList() + } + return parseList(body.getElementById("series-list") ?: parseFailed("Cannot find root")) + } + + private fun getActivePage(body: Element): Int = body.select("nav ul.pagination > li.page-item.active") + .lastOrNull() + ?.text() + ?.toIntOrNull() ?: parseFailed("Cannot determine current page") + + private fun parseList(root: Element) = root.children().map { div -> + val a = div.selectFirst("a") ?: parseFailed() + val href = a.relUrl("href") + val title = div.selectFirst(".item-title")?.text() ?: parseFailed("Title not found") + Manga( + id = generateUid(href), + title = title, + altTitle = div.selectFirst(".item-alias")?.text()?.takeUnless { it == title }, + url = href, + publicUrl = a.absUrl("href"), + rating = Manga.NO_RATING, + isNsfw = false, + coverUrl = div.selectFirst("img[src]")?.absUrl("src").orEmpty(), + largeCoverUrl = null, + description = null, + tags = div.selectFirst(".item-genre")?.parseTags().orEmpty(), + state = null, + author = null, + source = source, + ) + } + + private fun Element.parseTags() = children().mapToSet { span -> + val text = span.ownText() + MangaTag( + title = text.toTitleCase(), + key = text.lowercase(Locale.ENGLISH).replace(' ', '_'), + source = source, + ) + } + + private fun Element.parseChapter(index: Int): MangaChapter? { + val a = selectFirst("a.chapt") ?: return null + val extra = selectFirst(".extra") + val href = a.relUrl("href") + return MangaChapter( + id = generateUid(href), + name = a.text(), + number = index + 1, + url = href, + scanlator = extra?.getElementsByAttributeValueContaining("href", "/group/")?.text(), + uploadDate = runCatching { + parseChapterDate(extra?.select("i")?.lastOrNull()?.ownText()) + }.getOrDefault(0), + branch = null, + source = source, + ) + } + + private fun parseChapterDate(date: String?): Long { + if (date.isNullOrEmpty()) { + return 0 + } + val value = date.substringBefore(' ').toInt() + val field = when { + "sec" in date -> Calendar.SECOND + "min" in date -> Calendar.MINUTE + "hour" in date -> Calendar.HOUR + "day" in date -> Calendar.DAY_OF_MONTH + "week" in date -> Calendar.WEEK_OF_YEAR + "month" in date -> Calendar.MONTH + "year" in date -> Calendar.YEAR + else -> return 0 + } + val calendar = Calendar.getInstance() + calendar.add(field, -value) + return calendar.timeInMillis + } + + private fun decryptAES(encrypted: String, password: String): String { + val cipherData = Base64.decode(encrypted, Base64.DEFAULT) + val saltData = cipherData.copyOfRange(8, 16) + val (key, iv) = generateKeyAndIV( + keyLength = 32, + ivLength = 16, + iterations = 1, + salt = saltData, + password = password.toByteArray(StandardCharsets.UTF_8), + md = MessageDigest.getInstance("MD5"), + ) + val encryptedData = cipherData.copyOfRange(16, cipherData.size) + val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding") + cipher.init(Cipher.DECRYPT_MODE, key, iv) + return cipher.doFinal(encryptedData).toString(Charsets.UTF_8) + } + + @Suppress("SameParameterValue") + private fun generateKeyAndIV( + keyLength: Int, + ivLength: Int, + iterations: Int, + salt: ByteArray, + password: ByteArray, + md: MessageDigest, + ): Pair { + val digestLength = md.digestLength + val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength + val generatedData = ByteArray(requiredLength) + var generatedLength = 0 + md.reset() + while (generatedLength < keyLength + ivLength) { + if (generatedLength > 0) { + md.update(generatedData, generatedLength - digestLength, digestLength) + } + md.update(password) + md.update(salt, 0, 8) + md.digest(generatedData, generatedLength, digestLength) + repeat(iterations - 1) { + md.update(generatedData, generatedLength, digestLength) + md.digest(generatedData, generatedLength, digestLength) + } + generatedLength += digestLength + } + + return SecretKeySpec(generatedData.copyOfRange(0, keyLength), "AES") to IvParameterSpec( + if (ivLength > 0) { + generatedData.copyOfRange(keyLength, keyLength + ivLength) + } else byteArrayOf() + ) + } +} \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/download/domain/DownloadManager.kt b/app/src/main/java/org/koitharu/kotatsu/download/domain/DownloadManager.kt index 70a8b4db2..1303c0791 100644 --- a/app/src/main/java/org/koitharu/kotatsu/download/domain/DownloadManager.kt +++ b/app/src/main/java/org/koitharu/kotatsu/download/domain/DownloadManager.kt @@ -23,6 +23,7 @@ import org.koitharu.kotatsu.local.data.PagesCache import org.koitharu.kotatsu.local.domain.LocalMangaRepository import org.koitharu.kotatsu.utils.ext.await import org.koitharu.kotatsu.utils.ext.deleteAwait +import org.koitharu.kotatsu.utils.ext.referer import org.koitharu.kotatsu.utils.ext.waitForNetwork import java.io.File @@ -56,6 +57,7 @@ class DownloadManager( imageLoader.execute( ImageRequest.Builder(context) .data(manga.coverUrl) + .referer(manga.publicUrl) .size(coverWidth, coverHeight) .scale(Scale.FILL) .build() diff --git a/app/src/main/java/org/koitharu/kotatsu/list/ui/filter/FilterViewModel.kt b/app/src/main/java/org/koitharu/kotatsu/list/ui/filter/FilterViewModel.kt index 06c4b029e..9ce1579ce 100644 --- a/app/src/main/java/org/koitharu/kotatsu/list/ui/filter/FilterViewModel.kt +++ b/app/src/main/java/org/koitharu/kotatsu/list/ui/filter/FilterViewModel.kt @@ -1,8 +1,10 @@ package org.koitharu.kotatsu.list.ui.filter +import androidx.annotation.AnyThread import androidx.lifecycle.MutableLiveData import androidx.lifecycle.viewModelScope import kotlinx.coroutines.* +import org.koitharu.kotatsu.BuildConfig import org.koitharu.kotatsu.R import org.koitharu.kotatsu.base.domain.MangaDataRepository import org.koitharu.kotatsu.base.ui.BaseViewModel @@ -47,6 +49,7 @@ class FilterViewModel( } } + @AnyThread private fun updateFilters() { val previousJob = job job = launchJob(Dispatchers.Default) { @@ -73,7 +76,7 @@ class FilterViewModel( ensureActive() filter.postValue(list) } - result.value = FilterState(selectedSortOrder, selectedTags) + result.postValue(FilterState(selectedSortOrder, selectedTags)) } private fun showFilter() { @@ -107,8 +110,12 @@ class FilterViewModel( } private fun loadTagsAsync() = viewModelScope.async(Dispatchers.Default) { - kotlin.runCatching { + runCatching { repository.getTags() + }.onFailure { error -> + if (BuildConfig.DEBUG) { + error.printStackTrace() + } }.getOrNull() } } \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/utils/ext/StringExt.kt b/app/src/main/java/org/koitharu/kotatsu/utils/ext/StringExt.kt index 1f7961634..4d60ff637 100644 --- a/app/src/main/java/org/koitharu/kotatsu/utils/ext/StringExt.kt +++ b/app/src/main/java/org/koitharu/kotatsu/utils/ext/StringExt.kt @@ -150,6 +150,19 @@ fun String.substringBetween(from: String, to: String, fallbackValue: String = th } } +fun String.substringBetweenFirst(from: String, to: String): String? { + val fromIndex = indexOf(from) + if (fromIndex == -1) { + return null + } + val toIndex = indexOf(to, fromIndex) + return if (toIndex == -1) { + null + } else { + substring(fromIndex + from.length, toIndex) + } +} + fun String.substringBetweenLast(from: String, to: String, fallbackValue: String = this): String { val fromIndex = lastIndexOf(from) if (fromIndex == -1) { @@ -210,7 +223,7 @@ fun String.levenshteinDistance(other: String): Int { return cost[lhsLength - 1] } -inline fun StringBuilder.appendAll( +inline fun Appendable.appendAll( items: Iterable, separator: CharSequence, transform: (T) -> CharSequence = { it.toString() }, diff --git a/app/src/test/java/org/koitharu/kotatsu/core/parser/RemoteMangaRepositoryTest.kt b/app/src/test/java/org/koitharu/kotatsu/core/parser/RemoteMangaRepositoryTest.kt index 6cf57a268..15dc56d4f 100644 --- a/app/src/test/java/org/koitharu/kotatsu/core/parser/RemoteMangaRepositoryTest.kt +++ b/app/src/test/java/org/koitharu/kotatsu/core/parser/RemoteMangaRepositoryTest.kt @@ -13,7 +13,6 @@ import org.koin.test.KoinTestRule import org.koitharu.kotatsu.core.model.Manga import org.koitharu.kotatsu.core.model.MangaSource import org.koitharu.kotatsu.core.model.SortOrder -import org.koitharu.kotatsu.parsers.repositoryTestModule import org.koitharu.kotatsu.utils.CoroutineTestRule import org.koitharu.kotatsu.utils.TestResponse import org.koitharu.kotatsu.utils.ext.mapToSet diff --git a/app/src/test/java/org/koitharu/kotatsu/core/parser/RepositoryTestModule.kt b/app/src/test/java/org/koitharu/kotatsu/core/parser/RepositoryTestModule.kt index 81d536fea..7a808c8bf 100644 --- a/app/src/test/java/org/koitharu/kotatsu/core/parser/RepositoryTestModule.kt +++ b/app/src/test/java/org/koitharu/kotatsu/core/parser/RepositoryTestModule.kt @@ -1,5 +1,6 @@ -package org.koitharu.kotatsu.parsers +package org.koitharu.kotatsu.core.parser +import com.koushikdutta.quack.QuackContext import okhttp3.CookieJar import okhttp3.OkHttpClient import org.koin.dsl.module @@ -7,7 +8,6 @@ import org.koitharu.kotatsu.base.domain.MangaLoaderContext import org.koitharu.kotatsu.core.model.MangaSource import org.koitharu.kotatsu.core.network.TestCookieJar import org.koitharu.kotatsu.core.network.UserAgentInterceptor -import org.koitharu.kotatsu.core.parser.SourceSettingsStub import org.koitharu.kotatsu.core.prefs.SourceSettings import java.util.concurrent.TimeUnit @@ -28,6 +28,12 @@ val repositoryTestModule override fun getSettings(source: MangaSource): SourceSettings { return SourceSettingsStub() } + + override suspend fun evaluateJs(script: String): String? { + return QuackContext.create().use { + it.evaluate(script)?.toString() + } + } } } } \ No newline at end of file