Add Bato.To manga source #77

2022-03-08 14:03:12 +02:00
parent 8ff4eb2602
commit 564f052a2f
10 changed files with 359 additions and 7 deletions
--- a/app/build.gradle
+++ b/app/build.gradle
@@ -108,6 +108,7 @@ dependencies {
 	testImplementation 'junit:junit:4.13.2'
 	testImplementation 'com.google.truth:truth:1.1.3'
 	testImplementation 'org.json:json:20211205'
+	testImplementation 'io.webfolder:quickjs:1.1.0'
 	testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.6.0'
 	testImplementation 'io.insert-koin:koin-test-junit4:3.1.5'

--- a/app/src/main/java/org/koitharu/kotatsu/base/domain/MangaLoaderContext.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/base/domain/MangaLoaderContext.kt
@@ -1,5 +1,9 @@
 package org.koitharu.kotatsu.base.domain

+import android.annotation.SuppressLint
+import android.webkit.WebView
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
 import okhttp3.*
 import okhttp3.MediaType.Companion.toMediaType
 import okhttp3.RequestBody.Companion.toRequestBody
@@ -11,7 +15,8 @@ import org.koitharu.kotatsu.core.model.MangaSource
 import org.koitharu.kotatsu.core.prefs.SourceSettings
 import org.koitharu.kotatsu.utils.ext.await
 import org.koitharu.kotatsu.utils.ext.parseJson
-
+import kotlin.coroutines.resume
+import kotlin.coroutines.suspendCoroutine

 open class MangaLoaderContext(
 	private val okHttp: OkHttpClient,
@@ -80,5 +85,16 @@ open class MangaLoaderContext(
 		return json
 	}

+	@SuppressLint("SetJavaScriptEnabled")
+	open suspend fun evaluateJs(script: String): String? = withContext(Dispatchers.Main) {
+		val webView = WebView(get())
+		webView.settings.javaScriptEnabled = true
+		suspendCoroutine { cont ->
+			webView.evaluateJavascript(script) { result ->
+				cont.resume(result?.takeUnless { it == "null" })
+			}
+		}
+	}
+
 	open fun getSettings(source: MangaSource) = SourceSettings(get(), source)
 }
--- a/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt
@@ -35,5 +35,6 @@ enum class MangaSource(
 	EXHENTAI("ExHentai", null),
 	MANGAOWL("MangaOwl", "en"),
 	MANGADEX("MangaDex", null),
+	BATOTO("Bato.To", null),
 	;
 }
--- a/app/src/main/java/org/koitharu/kotatsu/core/parser/ParserModule.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/ParserModule.kt
@@ -32,4 +32,5 @@ val parserModule
 		factory<MangaRepository>(named(MangaSource.EXHENTAI)) { ExHentaiRepository(get()) }
 		factory<MangaRepository>(named(MangaSource.MANGAOWL)) { MangaOwlRepository(get()) }
 		factory<MangaRepository>(named(MangaSource.MANGADEX)) { MangaDexRepository(get()) }
+		factory<MangaRepository>(named(MangaSource.BATOTO)) { BatoToRepository(get()) }
 	}
--- a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/BatoToRepository.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/BatoToRepository.kt
@@ -0,0 +1,306 @@
+package org.koitharu.kotatsu.core.parser.site
+
+import android.util.Base64
+import androidx.collection.ArraySet
+import org.json.JSONArray
+import org.json.JSONObject
+import org.jsoup.nodes.Element
+import org.koitharu.kotatsu.base.domain.MangaLoaderContext
+import org.koitharu.kotatsu.core.model.*
+import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
+import org.koitharu.kotatsu.utils.ext.*
+import java.nio.charset.StandardCharsets
+import java.security.MessageDigest
+import java.util.*
+import javax.crypto.Cipher
+import javax.crypto.spec.IvParameterSpec
+import javax.crypto.spec.SecretKeySpec
+
+private const val PAGE_SIZE = 60
+private const val PAGE_SIZE_SEARCH = 20
+
+class BatoToRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepository(loaderContext) {
+
+	override val source = MangaSource.BATOTO
+
+	override val sortOrders: Set<SortOrder> = EnumSet.of(
+		SortOrder.NEWEST,
+		SortOrder.UPDATED,
+		SortOrder.POPULARITY,
+		SortOrder.ALPHABETICAL
+	)
+
+	override val defaultDomain: String = "bato.to"
+
+	override suspend fun getList2(
+		offset: Int,
+		query: String?,
+		tags: Set<MangaTag>?,
+		sortOrder: SortOrder?
+	): List<Manga> {
+		if (!query.isNullOrEmpty()) {
+			return search(offset, query)
+		}
+		val page = (offset / PAGE_SIZE) + 1
+
+		@Suppress("NON_EXHAUSTIVE_WHEN_STATEMENT")
+		val url = buildString {
+			append("https://")
+			append(getDomain())
+			append("/browse?sort=")
+			when (sortOrder) {
+				null,
+				SortOrder.UPDATED -> append("update.za")
+				SortOrder.POPULARITY -> append("views_a.za")
+				SortOrder.NEWEST -> append("create.za")
+				SortOrder.ALPHABETICAL -> append("title.az")
+			}
+			if (!tags.isNullOrEmpty()) {
+				append("&genres=")
+				appendAll(tags, ",") { it.key }
+			}
+			append("&page=")
+			append(page)
+		}
+		val body = loaderContext.httpGet(url).parseHtml().body()
+		val activePage = getActivePage(body)
+		if (activePage != page) {
+			return emptyList()
+		}
+		return parseList(body.getElementById("series-list") ?: parseFailed("Cannot find root"))
+	}
+
+	override suspend fun getDetails(manga: Manga): Manga {
+		val root = loaderContext.httpGet(manga.url.withDomain()).parseHtml()
+			.getElementById("mainer") ?: parseFailed("Cannot find root")
+		val details = root.selectFirst(".detail-set") ?: parseFailed("Cannot find detail-set")
+		val attrs = details.selectFirst(".attr-main")?.select(".attr-item")?.associate {
+			it.child(0).text().trim() to it.child(1)
+		}.orEmpty()
+		return manga.copy(
+			title = root.selectFirst("h3.item-title")?.text() ?: manga.title,
+			isNsfw = !root.selectFirst("alert")?.getElementsContainingOwnText("NSFW").isNullOrEmpty(),
+			largeCoverUrl = details.selectFirst("img[src]")?.absUrl("src"),
+			description = details.getElementById("limit-height-body-summary")
+				?.selectFirst(".limit-html")
+				?.html(),
+			tags = manga.tags + attrs["Genres:"]?.parseTags().orEmpty(),
+			state = when (attrs["Release status:"]?.text()) {
+				"Ongoing" -> MangaState.ONGOING
+				"Completed" -> MangaState.FINISHED
+				else -> manga.state
+			},
+			author = attrs["Authors:"]?.text()?.trim() ?: manga.author,
+			chapters = root.selectFirst(".episode-list")
+				?.selectFirst(".main")
+				?.children()
+				?.reversed()
+				?.mapIndexedNotNull { i, div ->
+					div.parseChapter(i)
+				}.orEmpty()
+		)
+	}
+
+	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+		val fullUrl = chapter.url.withDomain()
+		val scripts = loaderContext.httpGet(fullUrl).parseHtml().select("script")
+		for (script in scripts) {
+			val scriptSrc = script.html()
+			val p = scriptSrc.indexOf("const images =")
+			if (p == -1) continue
+			val start = scriptSrc.indexOf('[', p)
+			val end = scriptSrc.indexOf(';', start)
+			if (start == -1 || end == -1) {
+				continue
+			}
+			val images = JSONArray(scriptSrc.substring(start, end))
+			val batoJs = scriptSrc.substringBetweenFirst("batojs =", ";")?.trim(' ', '"', '\n')
+				?: parseFailed("Cannot find batojs")
+			val server = scriptSrc.substringBetweenFirst("server =", ";")?.trim(' ', '"', '\n')
+				?: parseFailed("Cannot find server")
+			val password = loaderContext.evaluateJs(batoJs)?.removeSurrounding('"')
+				?: parseFailed("Cannot evaluate batojs")
+			val serverDecrypted = decryptAES(server, password).removeSurrounding('"')
+			val result = ArrayList<MangaPage>(images.length())
+			repeat(images.length()) { i ->
+				val url = images.getString(i)
+				result += MangaPage(
+					id = generateUid(url),
+					url = if (url.startsWith("http")) url else "$serverDecrypted$url",
+					referer = fullUrl,
+					preview = null,
+					source = source,
+				)
+			}
+			return result
+		}
+		parseFailed("Cannot find images list")
+	}
+
+	override suspend fun getTags(): Set<MangaTag> {
+		val scripts = loaderContext.httpGet(
+			"https://${getDomain()}/browse"
+		).parseHtml().select("script")
+		for (script in scripts) {
+			val genres = script.html().substringBetweenFirst("const _genres =", ";") ?: continue
+			val jo = JSONObject(genres)
+			val result = ArraySet<MangaTag>(jo.length())
+			jo.keys().forEach { key ->
+				val item = jo.getJSONObject(key)
+				result += MangaTag(
+					title = item.getString("text").toTitleCase(),
+					key = item.getString("file"),
+					source = source,
+				)
+			}
+			return result
+		}
+		parseFailed("Cannot find gernes list")
+	}
+
+	override fun getFaviconUrl(): String = "https://styles.amarkcdn.com/img/batoto/favicon.ico?v0"
+
+	private suspend fun search(offset: Int, query: String): List<Manga> {
+		val page = (offset / PAGE_SIZE_SEARCH) + 1
+		val url = buildString {
+			append("https://")
+			append(getDomain())
+			append("/search?word=")
+			append(query.replace(' ', '+'))
+			append("&page=")
+			append(page)
+		}
+		val body = loaderContext.httpGet(url).parseHtml().body()
+		val activePage = getActivePage(body)
+		if (activePage != page) {
+			return emptyList()
+		}
+		return parseList(body.getElementById("series-list") ?: parseFailed("Cannot find root"))
+	}
+
+	private fun getActivePage(body: Element): Int = body.select("nav ul.pagination > li.page-item.active")
+		.lastOrNull()
+		?.text()
+		?.toIntOrNull() ?: parseFailed("Cannot determine current page")
+
+	private fun parseList(root: Element) = root.children().map { div ->
+		val a = div.selectFirst("a") ?: parseFailed()
+		val href = a.relUrl("href")
+		val title = div.selectFirst(".item-title")?.text() ?: parseFailed("Title not found")
+		Manga(
+			id = generateUid(href),
+			title = title,
+			altTitle = div.selectFirst(".item-alias")?.text()?.takeUnless { it == title },
+			url = href,
+			publicUrl = a.absUrl("href"),
+			rating = Manga.NO_RATING,
+			isNsfw = false,
+			coverUrl = div.selectFirst("img[src]")?.absUrl("src").orEmpty(),
+			largeCoverUrl = null,
+			description = null,
+			tags = div.selectFirst(".item-genre")?.parseTags().orEmpty(),
+			state = null,
+			author = null,
+			source = source,
+		)
+	}
+
+	private fun Element.parseTags() = children().mapToSet { span ->
+		val text = span.ownText()
+		MangaTag(
+			title = text.toTitleCase(),
+			key = text.lowercase(Locale.ENGLISH).replace(' ', '_'),
+			source = source,
+		)
+	}
+
+	private fun Element.parseChapter(index: Int): MangaChapter? {
+		val a = selectFirst("a.chapt") ?: return null
+		val extra = selectFirst(".extra")
+		val href = a.relUrl("href")
+		return MangaChapter(
+			id = generateUid(href),
+			name = a.text(),
+			number = index + 1,
+			url = href,
+			scanlator = extra?.getElementsByAttributeValueContaining("href", "/group/")?.text(),
+			uploadDate = runCatching {
+				parseChapterDate(extra?.select("i")?.lastOrNull()?.ownText())
+			}.getOrDefault(0),
+			branch = null,
+			source = source,
+		)
+	}
+
+	private fun parseChapterDate(date: String?): Long {
+		if (date.isNullOrEmpty()) {
+			return 0
+		}
+		val value = date.substringBefore(' ').toInt()
+		val field = when {
+			"sec" in date -> Calendar.SECOND
+			"min" in date -> Calendar.MINUTE
+			"hour" in date -> Calendar.HOUR
+			"day" in date -> Calendar.DAY_OF_MONTH
+			"week" in date -> Calendar.WEEK_OF_YEAR
+			"month" in date -> Calendar.MONTH
+			"year" in date -> Calendar.YEAR
+			else -> return 0
+		}
+		val calendar = Calendar.getInstance()
+		calendar.add(field, -value)
+		return calendar.timeInMillis
+	}
+
+	private fun decryptAES(encrypted: String, password: String): String {
+		val cipherData = Base64.decode(encrypted, Base64.DEFAULT)
+		val saltData = cipherData.copyOfRange(8, 16)
+		val (key, iv) = generateKeyAndIV(
+			keyLength = 32,
+			ivLength = 16,
+			iterations = 1,
+			salt = saltData,
+			password = password.toByteArray(StandardCharsets.UTF_8),
+			md = MessageDigest.getInstance("MD5"),
+		)
+		val encryptedData = cipherData.copyOfRange(16, cipherData.size)
+		val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding")
+		cipher.init(Cipher.DECRYPT_MODE, key, iv)
+		return cipher.doFinal(encryptedData).toString(Charsets.UTF_8)
+	}
+
+	@Suppress("SameParameterValue")
+	private fun generateKeyAndIV(
+		keyLength: Int,
+		ivLength: Int,
+		iterations: Int,
+		salt: ByteArray,
+		password: ByteArray,
+		md: MessageDigest,
+	): Pair<SecretKeySpec, IvParameterSpec> {
+		val digestLength = md.digestLength
+		val requiredLength = (keyLength + ivLength + digestLength - 1) / digestLength * digestLength
+		val generatedData = ByteArray(requiredLength)
+		var generatedLength = 0
+		md.reset()
+		while (generatedLength < keyLength + ivLength) {
+			if (generatedLength > 0) {
+				md.update(generatedData, generatedLength - digestLength, digestLength)
+			}
+			md.update(password)
+			md.update(salt, 0, 8)
+			md.digest(generatedData, generatedLength, digestLength)
+			repeat(iterations - 1) {
+				md.update(generatedData, generatedLength, digestLength)
+				md.digest(generatedData, generatedLength, digestLength)
+			}
+			generatedLength += digestLength
+		}
+
+		return SecretKeySpec(generatedData.copyOfRange(0, keyLength), "AES") to IvParameterSpec(
+			if (ivLength > 0) {
+				generatedData.copyOfRange(keyLength, keyLength + ivLength)
+			} else byteArrayOf()
+		)
+	}
+}
--- a/app/src/main/java/org/koitharu/kotatsu/download/domain/DownloadManager.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/download/domain/DownloadManager.kt
@@ -23,6 +23,7 @@ import org.koitharu.kotatsu.local.data.PagesCache
 import org.koitharu.kotatsu.local.domain.LocalMangaRepository
 import org.koitharu.kotatsu.utils.ext.await
 import org.koitharu.kotatsu.utils.ext.deleteAwait
+import org.koitharu.kotatsu.utils.ext.referer
 import org.koitharu.kotatsu.utils.ext.waitForNetwork
 import java.io.File

@@ -56,6 +57,7 @@ class DownloadManager(
 				imageLoader.execute(
 					ImageRequest.Builder(context)
 						.data(manga.coverUrl)
+						.referer(manga.publicUrl)
 						.size(coverWidth, coverHeight)
 						.scale(Scale.FILL)
 						.build()
--- a/app/src/main/java/org/koitharu/kotatsu/list/ui/filter/FilterViewModel.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/list/ui/filter/FilterViewModel.kt
@@ -1,8 +1,10 @@
 package org.koitharu.kotatsu.list.ui.filter

+import androidx.annotation.AnyThread
 import androidx.lifecycle.MutableLiveData
 import androidx.lifecycle.viewModelScope
 import kotlinx.coroutines.*
+import org.koitharu.kotatsu.BuildConfig
 import org.koitharu.kotatsu.R
 import org.koitharu.kotatsu.base.domain.MangaDataRepository
 import org.koitharu.kotatsu.base.ui.BaseViewModel
@@ -47,6 +49,7 @@ class FilterViewModel(
 		}
 	}

+	@AnyThread
 	private fun updateFilters() {
 		val previousJob = job
 		job = launchJob(Dispatchers.Default) {
@@ -73,7 +76,7 @@ class FilterViewModel(
 			ensureActive()
 			filter.postValue(list)
 		}
-		result.value = FilterState(selectedSortOrder, selectedTags)
+		result.postValue(FilterState(selectedSortOrder, selectedTags))
 	}

 	private fun showFilter() {
@@ -107,8 +110,12 @@ class FilterViewModel(
 	}

 	private fun loadTagsAsync() = viewModelScope.async(Dispatchers.Default) {
-		kotlin.runCatching {
+		runCatching {
 			repository.getTags()
+		}.onFailure { error ->
+			if (BuildConfig.DEBUG) {
+				error.printStackTrace()
+			}
 		}.getOrNull()
 	}
 }
--- a/app/src/main/java/org/koitharu/kotatsu/utils/ext/StringExt.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/utils/ext/StringExt.kt
@@ -150,6 +150,19 @@ fun String.substringBetween(from: String, to: String, fallbackValue: String = th
 	}
 }

+fun String.substringBetweenFirst(from: String, to: String): String? {
+	val fromIndex = indexOf(from)
+	if (fromIndex == -1) {
+		return null
+	}
+	val toIndex = indexOf(to, fromIndex)
+	return if (toIndex == -1) {
+		null
+	} else {
+		substring(fromIndex + from.length, toIndex)
+	}
+}
+
 fun String.substringBetweenLast(from: String, to: String, fallbackValue: String = this): String {
 	val fromIndex = lastIndexOf(from)
 	if (fromIndex == -1) {
@@ -210,7 +223,7 @@ fun String.levenshteinDistance(other: String): Int {
 	return cost[lhsLength - 1]
 }

-inline fun <T> StringBuilder.appendAll(
+inline fun <T> Appendable.appendAll(
 	items: Iterable<T>,
 	separator: CharSequence,
 	transform: (T) -> CharSequence = { it.toString() },
--- a/app/src/test/java/org/koitharu/kotatsu/core/parser/RemoteMangaRepositoryTest.kt
+++ b/app/src/test/java/org/koitharu/kotatsu/core/parser/RemoteMangaRepositoryTest.kt
@@ -13,7 +13,6 @@ import org.koin.test.KoinTestRule
 import org.koitharu.kotatsu.core.model.Manga
 import org.koitharu.kotatsu.core.model.MangaSource
 import org.koitharu.kotatsu.core.model.SortOrder
-import org.koitharu.kotatsu.parsers.repositoryTestModule
 import org.koitharu.kotatsu.utils.CoroutineTestRule
 import org.koitharu.kotatsu.utils.TestResponse
 import org.koitharu.kotatsu.utils.ext.mapToSet
--- a/app/src/test/java/org/koitharu/kotatsu/core/parser/RepositoryTestModule.kt
+++ b/app/src/test/java/org/koitharu/kotatsu/core/parser/RepositoryTestModule.kt
@@ -1,5 +1,6 @@
-package org.koitharu.kotatsu.parsers
+package org.koitharu.kotatsu.core.parser

+import com.koushikdutta.quack.QuackContext
 import okhttp3.CookieJar
 import okhttp3.OkHttpClient
 import org.koin.dsl.module
@@ -7,7 +8,6 @@ import org.koitharu.kotatsu.base.domain.MangaLoaderContext
 import org.koitharu.kotatsu.core.model.MangaSource
 import org.koitharu.kotatsu.core.network.TestCookieJar
 import org.koitharu.kotatsu.core.network.UserAgentInterceptor
-import org.koitharu.kotatsu.core.parser.SourceSettingsStub
 import org.koitharu.kotatsu.core.prefs.SourceSettings
 import java.util.concurrent.TimeUnit

@@ -28,6 +28,12 @@ val repositoryTestModule
 				override fun getSettings(source: MangaSource): SourceSettings {
 					return SourceSettingsStub()
 				}
+
+				override suspend fun evaluateJs(script: String): String? {
+					return QuackContext.create().use {
+						it.evaluate(script)?.toString()
+					}
+				}
 			}
 		}
 	}