From e0e6f0dab44f2d44d091bf1b1d2880e006a11c47 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Wed, 18 Mar 2020 20:22:16 +0200 Subject: [PATCH] DesuMe parser --- .idea/dictionaries/admin.xml | 1 + app/build.gradle | 1 + .../kotatsu/core/model/MangaSource.kt | 1 + .../core/parser/site/DesuMeRepository.kt | 137 ++++++++++++++++++ .../org/koitharu/kotatsu/utils/ext/JsonExt.kt | 10 ++ .../kotatsu/parsers/RemoteRepositoryTest.kt | 16 +- .../org/koitharu/kotatsu/utils/AssertX.kt | 14 +- 7 files changed, 165 insertions(+), 15 deletions(-) create mode 100644 app/src/main/java/org/koitharu/kotatsu/core/parser/site/DesuMeRepository.kt diff --git a/.idea/dictionaries/admin.xml b/.idea/dictionaries/admin.xml index 445962e31..c84c310d4 100644 --- a/.idea/dictionaries/admin.xml +++ b/.idea/dictionaries/admin.xml @@ -1,6 +1,7 @@ + desu koin kotatsu manga diff --git a/app/build.gradle b/app/build.gradle index ff053e0ad..ce22781e2 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -92,4 +92,5 @@ dependencies { debugImplementation 'com.squareup.leakcanary:leakcanary-android:2.2' testImplementation 'junit:junit:4.13' + testImplementation 'org.json:json:20180813' } \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt b/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt index a8e992914..78dd571ce 100644 --- a/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt +++ b/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt @@ -18,6 +18,7 @@ enum class MangaSource( MINTMANGA("MintManga", "ru", MintMangaRepository::class.java), SELFMANGA("SelfManga", "ru", SelfMangaRepository::class.java), MANGACHAN("Манга-тян", "ru", MangaChanRepository::class.java), + DESUME("Desu.me", "ru", DesuMeRepository::class.java), HENCHAN("Хентай-тян", "ru", HenChanRepository::class.java), YAOICHAN("Яой-тян", "ru", YaoiChanRepository::class.java) } \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/DesuMeRepository.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/DesuMeRepository.kt new file mode 100644 index 000000000..e8532f437 --- /dev/null +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/DesuMeRepository.kt @@ -0,0 +1,137 @@ +package org.koitharu.kotatsu.core.parser.site + +import org.koitharu.kotatsu.R +import org.koitharu.kotatsu.core.exceptions.ParseException +import org.koitharu.kotatsu.core.model.* +import org.koitharu.kotatsu.core.parser.RemoteMangaRepository +import org.koitharu.kotatsu.utils.ext.* + +class DesuMeRepository : RemoteMangaRepository() { + + override val source = MangaSource.DESUME + + override val sortOrders = setOf( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL + ) + + override suspend fun getList( + offset: Int, + query: String?, + sortOrder: SortOrder?, + tag: MangaTag? + ): List { + val domain = conf.getDomain(DOMAIN) + val url = buildString { + append("https://") + append(domain) + append("/manga/api/?limit=20&order=") + append(getSortKey(sortOrder)) + append("&page=") + append((offset / 20) + 1) + if (tag != null) { + append("&genres=") + append(tag.key) + } + if (query != null) { + append("&search=") + append(query) + } + } + val json = loaderContext.httpGet(url).parseJson().getJSONArray("response") + ?: throw ParseException("Invalid response") + val total = json.length() + val list = ArrayList(total) + for (i in 0 until total) { + val jo = json.getJSONObject(i) + val cover = jo.getJSONObject("image") + list += Manga( + url = jo.getString("url"), + source = MangaSource.DESUME, + title = jo.getString("russian"), + altTitle = jo.getString("name"), + coverUrl = cover.getString("preview"), + largeCoverUrl = cover.getString("original"), + state = when { + jo.getInt("ongoing") == 1 -> MangaState.ONGOING + else -> null + }, + rating = jo.getDouble("score").toFloat().coerceIn(0f, 1f), + id = ID_MASK + jo.getLong("id"), + description = jo.getString("description") + ) + } + return list + } + + override suspend fun getDetails(manga: Manga): Manga { + val domain = conf.getDomain(DOMAIN) + val url = "https://$domain/manga/api/${manga.id - ID_MASK}" + val json = loaderContext.httpGet(url).parseJson().getJSONObject("response") + ?: throw ParseException("Invalid response") + return manga.copy( + tags = json.getJSONArray("genres").map { + MangaTag( + key = it.getString("text"), + title = it.getString("russian"), + source = manga.source + ) + }.toSet(), + description = json.getString("description"), + chapters = json.getJSONObject("chapters").getJSONArray("list").mapIndexed { i, it -> + val chid = it.getLong("id") + MangaChapter( + id = ID_MASK + chid, + source = manga.source, + url = "$url/chapter/$chid", + name = it.optString("title", "${manga.title} #${it.getDouble("ch")}"), + number = i + 1 + ) + } + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val json = loaderContext.httpGet(chapter.url).parseJson().getJSONObject("response") + ?: throw ParseException("Invalid response") + return json.getJSONObject("pages").getJSONArray("list").map { + MangaPage( + id = it.getLong("id"), + source = chapter.source, + url = it.getString("img") + ) + } + } + + override suspend fun getTags(): Set { + val domain = conf.getDomain(DOMAIN) + val doc = loaderContext.httpGet("https://$domain/manga/").parseHtml() + val root = doc.body().getElementById("animeFilter").selectFirst(".catalog-genres") + return root.select("li").map { + MangaTag( + source = source, + key = it.selectFirst("input").attr("data-genre"), + title = it.selectFirst("label").text() + ) + }.toSet() + } + + override fun onCreatePreferences() = setOf(R.string.key_parser_domain) + + private fun getSortKey(sortOrder: SortOrder?) = + when (sortOrder) { + SortOrder.ALPHABETICAL -> "name" + SortOrder.POPULARITY -> "popular" + SortOrder.UPDATED -> "updated" + SortOrder.NEWEST -> "id" + else -> "updated" + } + + private companion object { + + private const val ID_MASK = 1000 + private const val DOMAIN = "desu.me" + } +} \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/utils/ext/JsonExt.kt b/app/src/main/java/org/koitharu/kotatsu/utils/ext/JsonExt.kt index 46ba35987..d47daf935 100644 --- a/app/src/main/java/org/koitharu/kotatsu/utils/ext/JsonExt.kt +++ b/app/src/main/java/org/koitharu/kotatsu/utils/ext/JsonExt.kt @@ -11,4 +11,14 @@ fun JSONArray.map(block: (JSONObject) -> T): List { result.add(block(jo)) } return result +} + +fun JSONArray.mapIndexed(block: (Int, JSONObject) -> T): List { + val len = length() + val result = ArrayList(len) + for(i in 0 until len) { + val jo = getJSONObject(i) + result.add(block(i, jo)) + } + return result } \ No newline at end of file diff --git a/app/src/test/java/org/koitharu/kotatsu/parsers/RemoteRepositoryTest.kt b/app/src/test/java/org/koitharu/kotatsu/parsers/RemoteRepositoryTest.kt index 69dee4167..b010e231c 100644 --- a/app/src/test/java/org/koitharu/kotatsu/parsers/RemoteRepositoryTest.kt +++ b/app/src/test/java/org/koitharu/kotatsu/parsers/RemoteRepositoryTest.kt @@ -26,8 +26,8 @@ class RemoteRepositoryTest(source: MangaSource) { val list = runBlocking { repo.getList(60) } Assert.assertFalse(list.isEmpty()) val item = list.random() - AssertX.assertContentType(item.coverUrl, "image") - AssertX.assertContentType(item.url, "text", "html") + AssertX.assertContentType(item.coverUrl, "image/*") + AssertX.assertContentType(item.url, "text/html", "application/json") Assert.assertFalse(item.title.isBlank()) } @@ -36,8 +36,8 @@ class RemoteRepositoryTest(source: MangaSource) { val list = runBlocking { repo.getList(0, query = "tail") } Assert.assertFalse(list.isEmpty()) val item = list.random() - AssertX.assertContentType(item.coverUrl, "image") - AssertX.assertContentType(item.url, "text", "html") + AssertX.assertContentType(item.coverUrl, "image/*") + AssertX.assertContentType(item.url, "text/html", "application/json") Assert.assertFalse(item.title.isBlank()) } @@ -51,8 +51,8 @@ class RemoteRepositoryTest(source: MangaSource) { val list = runBlocking { repo.getList(0, tag = tag) } Assert.assertFalse(list.isEmpty()) val item = list.random() - AssertX.assertContentType(item.coverUrl, "image") - AssertX.assertContentType(item.url, "text", "html") + AssertX.assertContentType(item.coverUrl, "image/*") + AssertX.assertContentType(item.url, "text/html", "application/json") Assert.assertFalse(item.title.isBlank()) } @@ -64,7 +64,7 @@ class RemoteRepositoryTest(source: MangaSource) { Assert.assertFalse(details.description.isNullOrEmpty()) val chapter = details.chapters!!.random() Assert.assertFalse(chapter.name.isBlank()) - AssertX.assertContentType(chapter.url, "text", "html") + AssertX.assertContentType(chapter.url, "text/html", "application/json") } @Test @@ -75,7 +75,7 @@ class RemoteRepositoryTest(source: MangaSource) { Assert.assertFalse(pages.isEmpty()) val page = pages.random() val fullUrl = runBlocking { repo.getPageFullUrl(page) } - AssertX.assertContentType(fullUrl, "image") + AssertX.assertContentType(fullUrl, "image/*") } companion object { diff --git a/app/src/test/java/org/koitharu/kotatsu/utils/AssertX.kt b/app/src/test/java/org/koitharu/kotatsu/utils/AssertX.kt index 7b0880170..bec4a5844 100644 --- a/app/src/test/java/org/koitharu/kotatsu/utils/AssertX.kt +++ b/app/src/test/java/org/koitharu/kotatsu/utils/AssertX.kt @@ -6,22 +6,22 @@ import java.net.URL object AssertX { - fun assertContentType(url: String, type: String, subtype: String? = null) { + fun assertContentType(url: String, vararg types: String) { Assert.assertFalse("URL is empty", url.isEmpty()) val cn = URL(url).openConnection() as HttpURLConnection cn.requestMethod = "HEAD" cn.connect() when (val code = cn.responseCode) { HttpURLConnection.HTTP_MOVED_PERM, - HttpURLConnection.HTTP_MOVED_TEMP -> assertContentType(cn.getHeaderField("Location"), type, subtype) + HttpURLConnection.HTTP_MOVED_TEMP -> assertContentType(cn.getHeaderField("Location"), *types) HttpURLConnection.HTTP_OK -> { val ct = cn.contentType.substringBeforeLast(';').split("/") - Assert.assertEquals(type, ct.first()) - if (subtype != null) { - Assert.assertEquals(subtype, ct.last()) - } + Assert.assertTrue(types.any { + val x = it.split('/') + x[0] == ct[0] && (x[1] == "*" || x[1] == ct[1]) + }) } - else -> Assert.fail("Invalid response code $code") + else -> Assert.fail("Invalid response code $code at $url") } }