From b43889845656e6ee0d256e6b444b6c89cc3c5e90 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Sun, 10 May 2020 18:52:00 +0300 Subject: [PATCH] Add mangalib source --- .../kotatsu/core/model/MangaSource.kt | 4 +- .../core/parser/site/HentaiLibRepository.kt | 11 + .../core/parser/site/MangaLibRepository.kt | 196 ++++++++++++++++++ .../org/koitharu/kotatsu/utils/ext/JsonExt.kt | 19 +- .../kotatsu/parsers/RemoteRepositoryTest.kt | 8 +- .../kotatsu/parsers/SourceConfigMock.kt | 2 + 6 files changed, 232 insertions(+), 8 deletions(-) create mode 100644 app/src/main/java/org/koitharu/kotatsu/core/parser/site/HentaiLibRepository.kt create mode 100644 app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaLibRepository.kt diff --git a/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt b/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt index daca081c5..0c4b815ff 100644 --- a/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt +++ b/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt @@ -21,5 +21,7 @@ enum class MangaSource( DESUME("Desu.me", "ru", DesuMeRepository::class.java), HENCHAN("Хентай-тян", "ru", HenChanRepository::class.java), YAOICHAN("Яой-тян", "ru", YaoiChanRepository::class.java), - MANGATOWN("MangaTown", "en", MangaTownRepository::class.java) + MANGATOWN("MangaTown", "en", MangaTownRepository::class.java), + MANGALIB("MangaLib", "ru", MangaLibRepository::class.java), + HENTAILIB("HentaiLib", "ru", HentaiLibRepository::class.java) } \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/HentaiLibRepository.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/HentaiLibRepository.kt new file mode 100644 index 000000000..0b3a5c083 --- /dev/null +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/HentaiLibRepository.kt @@ -0,0 +1,11 @@ +package org.koitharu.kotatsu.core.parser.site + +import org.koitharu.kotatsu.core.model.MangaSource + +class HentaiLibRepository : MangaLibRepository() { + + protected override val defaultDomain = "hentailib.me" + + override val source = MangaSource.HENTAILIB + +} \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaLibRepository.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaLibRepository.kt new file mode 100644 index 000000000..0b0b34c6a --- /dev/null +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaLibRepository.kt @@ -0,0 +1,196 @@ +package org.koitharu.kotatsu.core.parser.site + +import org.json.JSONArray +import org.json.JSONObject +import org.koitharu.kotatsu.R +import org.koitharu.kotatsu.core.exceptions.ParseException +import org.koitharu.kotatsu.core.model.* +import org.koitharu.kotatsu.core.parser.RemoteMangaRepository +import org.koitharu.kotatsu.utils.ext.* + +open class MangaLibRepository : RemoteMangaRepository() { + + protected open val defaultDomain = "mangalib.me" + + override val source = MangaSource.MANGALIB + + override val sortOrders = setOf( + SortOrder.RATING, + SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.NEWEST + ) + + override suspend fun getList( + offset: Int, + query: String?, + sortOrder: SortOrder?, + tag: MangaTag? + ): List { + val domain = conf.getDomain(defaultDomain) + val page = (offset / 60f).toIntUp() + val url = buildString { + append("https://") + append(domain) + append("/manga-list?dir=") + append(getSortKey(sortOrder)) + append("&page=") + append(page) + if (tag != null) { + append("&includeGenres[]=") + append(tag.key) + } + } + val doc = loaderContext.httpGet(url).parseHtml() + val root = doc.body().getElementById("manga-list") ?: throw ParseException("Root not found") + val items = root.selectFirst("div.media-cards-grid").select("div.media-card-wrap") + return items.mapNotNull { card -> + val a = card.selectFirst("a.media-card") ?: return@mapNotNull null + val href = a.attr("href").withDomain(domain) + Manga( + id = href.longHashCode(), + title = card.selectFirst("h3").text(), + coverUrl = a.attr("data-src").withDomain(domain), + altTitle = null, + author = null, + rating = Manga.NO_RATING, + url = href, + tags = emptySet(), + state = null, + source = source + ) + } + } + + override fun onCreatePreferences() = setOf(R.string.key_parser_domain) + + override suspend fun getDetails(manga: Manga): Manga { + val doc = loaderContext.httpGet(manga.url + "?section=info").parseHtml() + val root = doc.body().getElementById("main-page") ?: throw ParseException("Root not found") + val title = root.selectFirst("div.media-header__wrap")?.children() + val info = root.selectFirst("div.media-content") + val chaptersDoc = loaderContext.httpGet(manga.url + "?section=chapters").parseHtml() + val scripts = chaptersDoc.body().select("script") + var chapters: ArrayList? = null + scripts@ for (script in scripts) { + val raw = script.html().lines() + for (line in raw) { + if (line.startsWith("window.__CHAPTERS_DATA__")) { + val json = JSONObject(line.substringAfter('=').substringBeforeLast(';')) + val list = json.getJSONArray("list") + val total = list.length() + chapters = ArrayList(total) + for (i in 0 until total) { + val item = list.getJSONObject(i) + val url = buildString { + append(manga.url) + append("/v") + append(item.getInt("chapter_volume")) + append("/c") + append(item.getString("chapter_number")) + append('/') + append(item.getJSONArray("teams").getJSONObject(0).getString("slug")) + } + var name = item.getString("chapter_name") + if (name.isNullOrBlank() || name == "null") { + name = "Том " + item.getInt("chapter_volume") + + " Глава " + item.getString("chapter_number") + } + chapters.add( + MangaChapter( + id = url.longHashCode(), + url = url, + source = source, + number = total - i, + name = name + ) + ) + } + break@scripts + } + } + } + return manga.copy( + title = title?.getOrNull(0)?.text()?.takeUnless(String::isBlank) ?: manga.title, + altTitle = title?.getOrNull(1)?.text()?.substringBefore('/')?.trim(), + rating = root.selectFirst("div.media-stats-item__score") + ?.selectFirst("span") + ?.text()?.toFloatOrNull()?.div(5f) ?: manga.rating, + author = info.getElementsMatchingOwnText("Автор").firstOrNull() + ?.nextElementSibling()?.text() ?: manga.author, + tags = info.getElementsMatchingOwnText("Жанры")?.firstOrNull() + ?.nextElementSibling()?.select("a")?.mapNotNull { a -> + MangaTag( + title = a.text(), + key = a.attr("href").substringAfterLast('='), + source = source + ) + }?.toSet() ?: manga.tags, + description = info.getElementsMatchingOwnText("Описание")?.firstOrNull() + ?.nextElementSibling()?.html(), + chapters = chapters + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = loaderContext.httpGet(chapter.url).parseHtml() + val scripts = doc.head().select("script") + val pg = doc.body().getElementById("pg").html().substringAfter('=').substringBeforeLast(';') + val pages = JSONArray(pg) + for (script in scripts) { + val raw = script.html().trim() + if (raw.startsWith("window.__info")) { + val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';')) + val domain = json.getJSONObject("servers").run { + getStringOrNull("main") ?: getString( + json.getJSONObject("img").getString("server") + ) + } + val url = json.getJSONObject("img").getString("url") + return pages.map { x -> + val pageUrl = "$domain$url${x.getString("u")}" + MangaPage( + id = pageUrl.longHashCode(), + source = source, + url = pageUrl + ) + } + } + } + throw ParseException("Script with info not found") + } + + override suspend fun getTags(): Set { + val domain = conf.getDomain(defaultDomain) + val url = "https://$domain/manga-list" + val doc = loaderContext.httpGet(url).parseHtml() + val scripts = doc.body().select("script") + for (script in scripts) { + val raw = script.html().trim() + if (raw.startsWith("window.__DATA")) { + val json = JSONObject(raw.substringAfter('=').substringBeforeLast(';')) + val genres = json.getJSONObject("filters").getJSONArray("genres") + val result = HashSet(genres.length()) + for (x in genres) { + result += MangaTag( + source = source, + key = x.getInt("id").toString(), + title = x.getString("name") + ) + } + return result + } + } + throw ParseException("Script with genres not found") + } + + private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder) { + SortOrder.RATING -> "desc&sort=rate" + SortOrder.ALPHABETICAL -> "asc&sort=name" + SortOrder.POPULARITY -> "desc&sort=views" + SortOrder.UPDATED -> "desc&sort=last_chapter_at" + SortOrder.NEWEST -> "desc&sort=created_at" + else -> "desc&sort=last_chapter_at" + } +} \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/utils/ext/JsonExt.kt b/app/src/main/java/org/koitharu/kotatsu/utils/ext/JsonExt.kt index 5723b2c10..6ae0ce40a 100644 --- a/app/src/main/java/org/koitharu/kotatsu/utils/ext/JsonExt.kt +++ b/app/src/main/java/org/koitharu/kotatsu/utils/ext/JsonExt.kt @@ -6,7 +6,7 @@ import org.json.JSONObject fun JSONArray.map(block: (JSONObject) -> T): List { val len = length() val result = ArrayList(len) - for(i in 0 until len) { + for (i in 0 until len) { val jo = getJSONObject(i) result.add(block(jo)) } @@ -16,11 +16,24 @@ fun JSONArray.map(block: (JSONObject) -> T): List { fun JSONArray.mapIndexed(block: (Int, JSONObject) -> T): List { val len = length() val result = ArrayList(len) - for(i in 0 until len) { + for (i in 0 until len) { val jo = getJSONObject(i) result.add(block(i, jo)) } return result } -fun JSONObject.getStringOrNull(name: String): String? = opt(name)?.toString() \ No newline at end of file +fun JSONObject.getStringOrNull(name: String): String? = opt(name)?.toString() + +operator fun JSONArray.iterator(): Iterator = JSONIterator(this) + +private class JSONIterator(private val array: JSONArray) : Iterator { + + private val total = array.length() + private var index = 0 + + override fun hasNext() = index < total - 1 + + override fun next(): JSONObject = array.getJSONObject(index++) + +} \ No newline at end of file diff --git a/app/src/test/java/org/koitharu/kotatsu/parsers/RemoteRepositoryTest.kt b/app/src/test/java/org/koitharu/kotatsu/parsers/RemoteRepositoryTest.kt index b010e231c..02320a3de 100644 --- a/app/src/test/java/org/koitharu/kotatsu/parsers/RemoteRepositoryTest.kt +++ b/app/src/test/java/org/koitharu/kotatsu/parsers/RemoteRepositoryTest.kt @@ -22,7 +22,7 @@ class RemoteRepositoryTest(source: MangaSource) { private val repo = MangaProviderFactory.create(source) @Test - fun getList() { + fun list() { val list = runBlocking { repo.getList(60) } Assert.assertFalse(list.isEmpty()) val item = list.random() @@ -42,7 +42,7 @@ class RemoteRepositoryTest(source: MangaSource) { } @Test - fun getTags() { + fun tags() { val tags = runBlocking { repo.getTags() } Assert.assertFalse(tags.isEmpty()) val tag = tags.random() @@ -57,7 +57,7 @@ class RemoteRepositoryTest(source: MangaSource) { } @Test - fun getDetails() { + fun details() { val manga = runBlocking { repo.getList(0) }.random() val details = runBlocking { repo.getDetails(manga) } Assert.assertFalse(details.chapters.isNullOrEmpty()) @@ -68,7 +68,7 @@ class RemoteRepositoryTest(source: MangaSource) { } @Test - fun getPages() { + fun pages() { val manga = runBlocking { repo.getList(0) }.random() val details = runBlocking { repo.getDetails(manga) } val pages = runBlocking { repo.getPages(details.chapters!!.random()) } diff --git a/app/src/test/java/org/koitharu/kotatsu/parsers/SourceConfigMock.kt b/app/src/test/java/org/koitharu/kotatsu/parsers/SourceConfigMock.kt index 834b3a5f1..dd0c41304 100644 --- a/app/src/test/java/org/koitharu/kotatsu/parsers/SourceConfigMock.kt +++ b/app/src/test/java/org/koitharu/kotatsu/parsers/SourceConfigMock.kt @@ -5,4 +5,6 @@ import org.koitharu.kotatsu.core.prefs.SourceConfig class SourceConfigMock : SourceConfig { override fun getDomain(defaultValue: String) = defaultValue + + override fun isUseSsl(defaultValue: Boolean) = defaultValue } \ No newline at end of file