From 2135195f2701ce1c691b6f94c40f18708a2df1c2 Mon Sep 17 00:00:00 2001 From: Koitharu Date: Mon, 19 Oct 2020 20:24:58 +0300 Subject: [PATCH] MangaRead parser #17 --- .../kotatsu/core/model/MangaSource.kt | 3 +- .../core/parser/site/ChanRepository.kt | 2 +- .../core/parser/site/GroupleRepository.kt | 16 +- .../core/parser/site/MangaTownRepository.kt | 27 +-- .../core/parser/site/MangareadRepository.kt | 165 ++++++++++++++++++ .../kotatsu/domain/MangaLoaderContext.kt | 23 +++ .../kotatsu/utils/ext/CollectionExt.kt | 4 +- .../kotatsu/parsers/RemoteRepositoryTest.kt | 2 +- .../org/koitharu/kotatsu/utils/AssertX.kt | 4 +- 9 files changed, 222 insertions(+), 24 deletions(-) create mode 100644 app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangareadRepository.kt diff --git a/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt b/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt index 52dea0d97..136bb1722 100644 --- a/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt +++ b/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt @@ -23,6 +23,7 @@ enum class MangaSource( YAOICHAN("Яой-тян", "ru", YaoiChanRepository::class.java), MANGATOWN("MangaTown", "en", MangaTownRepository::class.java), MANGALIB("MangaLib", "ru", MangaLibRepository::class.java), - NUDEMOON("Nude-Moon", "ru", NudeMoonRepository::class.java) + NUDEMOON("Nude-Moon", "ru", NudeMoonRepository::class.java), + MANGAREAD("MangaRead", "en", MangareadRepository::class.java), // HENTAILIB("HentaiLib", "ru", HentaiLibRepository::class.java) } \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/ChanRepository.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/ChanRepository.kt index 7dc4a5cb8..f58b114cd 100644 --- a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/ChanRepository.kt +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/ChanRepository.kt @@ -106,7 +106,7 @@ abstract class ChanRepository(loaderContext: MangaLoaderContext) : RemoteMangaRe val json = data.substring(pos).substringAfter('[').substringBefore(';') .substringBeforeLast(']') return json.split(",").mapNotNull { - it.trim().removeSurrounding('"','\'').takeUnless(String::isBlank) + it.trim().removeSurrounding('"', '\'').takeUnless(String::isBlank) }.map { url -> MangaPage( id = url.longHashCode(), diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/GroupleRepository.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/GroupleRepository.kt index 5a198127b..d7a9af359 100644 --- a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/GroupleRepository.kt +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/GroupleRepository.kt @@ -32,14 +32,18 @@ abstract class GroupleRepository(loaderContext: MangaLoaderContext) : mapOf("q" to query.urlEncoded(), "offset" to offset.toString()) ) tag == null -> loaderContext.httpGet( - "https://$domain/list?sortType=${getSortKey( - sortOrder - )}&offset=$offset" + "https://$domain/list?sortType=${ + getSortKey( + sortOrder + ) + }&offset=$offset" ) else -> loaderContext.httpGet( - "https://$domain/list/genre/${tag.key}?sortType=${getSortKey( - sortOrder - )}&offset=$offset" + "https://$domain/list/genre/${tag.key}?sortType=${ + getSortKey( + sortOrder + ) + }&offset=$offset" ) }.parseHtml() val root = doc.body().getElementById("mangaBox") diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaTownRepository.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaTownRepository.kt index 9afc34b87..ab69986c0 100644 --- a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaTownRepository.kt +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangaTownRepository.kt @@ -10,7 +10,8 @@ import org.koitharu.kotatsu.domain.MangaLoaderContext import org.koitharu.kotatsu.utils.ext.* import java.util.* -class MangaTownRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepository(loaderContext) { +class MangaTownRepository(loaderContext: MangaLoaderContext) : + RemoteMangaRepository(loaderContext) { override val source = MangaSource.MANGATOWN @@ -105,16 +106,17 @@ class MangaTownRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposi }.orEmpty(), description = info.getElementById("show")?.ownText(), chapters = chaptersList?.mapIndexedNotNull { i, li -> - val href = li.selectFirst("a").attr("href").withDomain(domain, ssl) - val name = li.select("span").filter { it.className().isEmpty() }.joinToString(" - ") { it.text() }.trim() - MangaChapter( - id = href.longHashCode(), - url = href, - source = MangaSource.MANGATOWN, - number = i + 1, - name = if (name.isEmpty()) "${manga.title} - ${i + 1}" else name - ) - } + val href = li.selectFirst("a").attr("href").withDomain(domain, ssl) + val name = li.select("span").filter { it.className().isEmpty() } + .joinToString(" - ") { it.text() }.trim() + MangaChapter( + id = href.longHashCode(), + url = href, + source = MangaSource.MANGATOWN, + number = i + 1, + name = if (name.isEmpty()) "${manga.title} - ${i + 1}" else name + ) + } ) } @@ -166,7 +168,8 @@ class MangaTownRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposi } - override fun onCreatePreferences() = arraySetOf(R.string.key_parser_domain, R.string.key_parser_ssl) + override fun onCreatePreferences() = + arraySetOf(R.string.key_parser_domain, R.string.key_parser_ssl) private fun String.parseTagKey() = split('/').findLast { TAG_REGEX matches it } diff --git a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangareadRepository.kt b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangareadRepository.kt new file mode 100644 index 000000000..447910803 --- /dev/null +++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/MangareadRepository.kt @@ -0,0 +1,165 @@ +package org.koitharu.kotatsu.core.parser.site + +import androidx.collection.arraySetOf +import org.koitharu.kotatsu.R +import org.koitharu.kotatsu.core.exceptions.ParseException +import org.koitharu.kotatsu.core.model.* +import org.koitharu.kotatsu.core.parser.RemoteMangaRepository +import org.koitharu.kotatsu.domain.MangaLoaderContext +import org.koitharu.kotatsu.utils.ext.* + +class MangareadRepository( + loaderContext: MangaLoaderContext +) : RemoteMangaRepository(loaderContext) { + + override val source = MangaSource.MANGAREAD + + override val sortOrders = arraySetOf(SortOrder.UPDATED, SortOrder.POPULARITY) + + override suspend fun getList( + offset: Int, + query: String?, + sortOrder: SortOrder?, + tag: MangaTag? + ): List { + if (offset % PAGE_SIZE != 0) { + return emptyList() + } + val domain = conf.getDomain(DOMAIN) + val payload = createRequestTemplate() + payload["page"] = (offset / PAGE_SIZE).toString() + payload["vars[meta_key]"] = when (sortOrder) { + SortOrder.POPULARITY -> "_wp_manga_views" + SortOrder.UPDATED -> "_latest_update" + else -> "_wp_manga_views" + } + payload["vars[wp-manga-genre]"] = tag?.key.orEmpty() + payload["vars[s]"] = query.orEmpty() + val doc = loaderContext.httpPost( + "https://${domain}/wp-admin/admin-ajax.php", + payload + ).parseHtml() + return doc.select("div.row.c-tabs-item__content").map { div -> + val href = div.selectFirst("a").absUrl("href") + val summary = div.selectFirst(".tab-summary") + Manga( + id = href.longHashCode(), + url = href, + coverUrl = div.selectFirst("img").absUrl("src"), + title = summary.selectFirst("h3").text(), + rating = div.selectFirst("span.total_votes")?.ownText() + ?.toFloatOrNull()?.div(5f) ?: -1f, + tags = summary.selectFirst(".mg_genres").select("a").mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text(), + source = MangaSource.MANGAREAD + ) + }, + author = summary.selectFirst(".mg_author")?.selectFirst("a")?.ownText(), + state = when (summary.selectFirst(".mg_status")?.selectFirst(".summary-content") + ?.ownText()?.trim()) { + "OnGoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + }, + source = MangaSource.MANGAREAD + ) + } + } + + override suspend fun getTags(): Set { + val domain = conf.getDomain(DOMAIN) + val doc = loaderContext.httpGet("https://$domain/manga/").parseHtml() + val root = doc.body().getElementById("main-sidebar") + .selectFirst(".genres_wrap") + .selectFirst("ul") + return root.select("li").mapToSet { li -> + val a = li.selectFirst("a") + MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text(), + source = MangaSource.MANGAREAD + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val domain = conf.getDomain(DOMAIN) + val doc = loaderContext.httpGet(manga.url).parseHtml() + val root = doc.body().selectFirst("div.profile-manga") + ?.selectFirst("div.summary_content") + ?.selectFirst("div.post-content") + ?: throw ParseException("Root not found") + val root2 = doc.body().selectFirst("div.content-area") + ?.selectFirst("div.c-page") + ?: throw ParseException("Root2 not found") + val mangaId = doc.getElementsByAttribute("data-postid").firstOrNull() + ?.attr("data-postid")?.toLongOrNull() + ?: throw ParseException("Cannot obtain manga id") + val doc2 = loaderContext.httpPost( + "https://${domain}/wp-admin/admin-ajax.php", + mapOf( + "action" to "manga_get_chapters", + "manga" to mangaId.toString() + ) + ).parseHtml() + return manga.copy( + tags = root.selectFirst("div.genres-content")?.select("a") + ?.mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text(), + source = MangaSource.MANGAREAD + ) + } ?: manga.tags, + description = root2.selectFirst("div.description-summary") + ?.selectFirst("div.summary__content") + ?.select("p")?.drop(1) + ?.joinToString { it.html() }, + chapters = doc2.select("li").asReversed().mapIndexed { i, li -> + val a = li.selectFirst("a") + val href = a.absUrl("href") + MangaChapter( + id = href.longHashCode(), + name = a.ownText(), + number = i + 1, + url = href, + source = MangaSource.MANGAREAD + ) + } + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = loaderContext.httpGet(chapter.url).parseHtml() + val root = doc.body().selectFirst("div.main-col-inner") + ?.selectFirst("div.reading-content") + ?: throw ParseException("Root not found") + return root.select("div.page-break").map { div -> + val img = div.selectFirst("img") + val url = img.absUrl("src") + MangaPage( + id = url.longHashCode(), + url = url, + source = MangaSource.MANGAREAD + ) + } + } + + override fun onCreatePreferences() = arraySetOf(R.string.key_parser_domain) + + private companion object { + + private const val PAGE_SIZE = 12 + private const val DOMAIN = "www.mangaread.org" + + private fun createRequestTemplate() = + "action=madara_load_more&page=1&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5Borderby%5D=meta_value_num&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=OR&vars%5Bpost_type%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmeta_key%5D=_latest_update&vars%5Border%5D=desc&vars%5Bmanga_archives_item_layout%5D=default" + .split('&') + .map { + val pos = it.indexOf('=') + it.substring(0, pos) to it.substring(pos + 1) + }.toMutableMap() + } +} \ No newline at end of file diff --git a/app/src/main/java/org/koitharu/kotatsu/domain/MangaLoaderContext.kt b/app/src/main/java/org/koitharu/kotatsu/domain/MangaLoaderContext.kt index 46da416fb..c41de75b2 100644 --- a/app/src/main/java/org/koitharu/kotatsu/domain/MangaLoaderContext.kt +++ b/app/src/main/java/org/koitharu/kotatsu/domain/MangaLoaderContext.kt @@ -41,6 +41,29 @@ open class MangaLoaderContext : KoinComponent { return okHttp.newCall(request.build()).await() } + suspend fun httpPost( + url: String, + payload: String, + block: (Request.Builder.() -> Unit)? = null + ): Response { + val body = FormBody.Builder() + payload.split('&').forEach { + val pos = it.indexOf('=') + if (pos != -1) { + val k = it.substring(0, pos) + val v = it.substring(pos + 1) + body.addEncoded(k, v) + } + } + val request = Request.Builder() + .post(body.build()) + .url(url) + if (block != null) { + request.block() + } + return okHttp.newCall(request.build()).await() + } + open fun getSettings(source: MangaSource) = SourceConfig(get(), source) fun insertCookies(domain: String, vararg cookies: String) { diff --git a/app/src/main/java/org/koitharu/kotatsu/utils/ext/CollectionExt.kt b/app/src/main/java/org/koitharu/kotatsu/utils/ext/CollectionExt.kt index fe12dbd57..c4ea177d9 100644 --- a/app/src/main/java/org/koitharu/kotatsu/utils/ext/CollectionExt.kt +++ b/app/src/main/java/org/koitharu/kotatsu/utils/ext/CollectionExt.kt @@ -54,4 +54,6 @@ fun LongArray.toArraySet(): Set { } } } -} \ No newline at end of file +} + +fun List>.toMutableMap(): MutableMap = toMap(HashMap(size)) \ No newline at end of file diff --git a/app/src/test/java/org/koitharu/kotatsu/parsers/RemoteRepositoryTest.kt b/app/src/test/java/org/koitharu/kotatsu/parsers/RemoteRepositoryTest.kt index e41122c42..c6e5511d5 100644 --- a/app/src/test/java/org/koitharu/kotatsu/parsers/RemoteRepositoryTest.kt +++ b/app/src/test/java/org/koitharu/kotatsu/parsers/RemoteRepositoryTest.kt @@ -90,7 +90,7 @@ class RemoteRepositoryTest(source: MangaSource) { factory { OkHttpClient.Builder() .cookieJar(TemporaryCookieJar()) - .addInterceptor(UserAgentInterceptor) + .addInterceptor(UserAgentInterceptor()) .connectTimeout(20, TimeUnit.SECONDS) .readTimeout(60, TimeUnit.SECONDS) .writeTimeout(20, TimeUnit.SECONDS) diff --git a/app/src/test/java/org/koitharu/kotatsu/utils/AssertX.kt b/app/src/test/java/org/koitharu/kotatsu/utils/AssertX.kt index 0b7584a75..78c87bd80 100644 --- a/app/src/test/java/org/koitharu/kotatsu/utils/AssertX.kt +++ b/app/src/test/java/org/koitharu/kotatsu/utils/AssertX.kt @@ -3,8 +3,8 @@ package org.koitharu.kotatsu.utils import okhttp3.OkHttpClient import okhttp3.Request import org.junit.Assert -import org.koin.core.KoinComponent -import org.koin.core.inject +import org.koin.core.component.KoinComponent +import org.koin.core.component.inject import java.net.HttpURLConnection object AssertX : KoinComponent {