Add NudeMoon parser (broken)

2020-10-18 15:09:42 +03:00
parent 4dc9df0515
commit ff3ebbf1d9
12 changed files with 226 additions and 32 deletions
--- a/app/src/main/java/org/koitharu/kotatsu/KotatsuApp.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/KotatsuApp.kt
@@ -10,7 +10,9 @@ import coil.ImageLoader
 import coil.util.CoilUtils
 import com.chuckerteam.chucker.api.ChuckerCollector
 import com.chuckerteam.chucker.api.ChuckerInterceptor
+import okhttp3.CookieJar
 import okhttp3.OkHttpClient
+import org.koin.android.ext.android.get
 import org.koin.android.ext.koin.androidContext
 import org.koin.android.ext.koin.androidLogger
 import org.koin.core.context.startKoin
@@ -37,10 +39,6 @@ import java.util.concurrent.TimeUnit

 class KotatsuApp : Application() {

-	private val cookieJar by lazy {
-		PersistentCookieJar(SetCookieCache(), SharedPrefsCookiePersistor(applicationContext))
-	}
-
 	private val chuckerCollector by lazy(LazyThreadSafetyMode.NONE) {
 		ChuckerCollector(applicationContext)
 	}
@@ -48,20 +46,24 @@ class KotatsuApp : Application() {
 	override fun onCreate() {
 		super.onCreate()
 		if (BuildConfig.DEBUG) {
-			StrictMode.setThreadPolicy(StrictMode.ThreadPolicy.Builder()
-				.detectAll()
-				.penaltyLog()
-				.build())
-			StrictMode.setVmPolicy(StrictMode.VmPolicy.Builder()
-				.detectAll()
-				.setClassInstanceLimit(LocalMangaRepository::class.java, 1)
-				.setClassInstanceLimit(PagesCache::class.java, 1)
-				.setClassInstanceLimit(MangaLoaderContext::class.java, 1)
-				.penaltyLog()
-				.build())
+			StrictMode.setThreadPolicy(
+				StrictMode.ThreadPolicy.Builder()
+					.detectAll()
+					.penaltyLog()
+					.build()
+			)
+			StrictMode.setVmPolicy(
+				StrictMode.VmPolicy.Builder()
+					.detectAll()
+					.setClassInstanceLimit(LocalMangaRepository::class.java, 1)
+					.setClassInstanceLimit(PagesCache::class.java, 1)
+					.setClassInstanceLimit(MangaLoaderContext::class.java, 1)
+					.penaltyLog()
+					.build()
+			)
 		}
 		initKoin()
-		initCoil()
+		initCoil(get())
 		Thread.setDefaultUncaughtExceptionHandler(AppCrashHandler(applicationContext))
 		if (BuildConfig.DEBUG) {
 			initErrorHandler()
@@ -78,8 +80,14 @@ class KotatsuApp : Application() {
 			androidContext(applicationContext)
 			modules(
 				module {
+					single<CookieJar> {
+						PersistentCookieJar(
+							SetCookieCache(),
+							SharedPrefsCookiePersistor(applicationContext)
+						)
+					}
 					factory {
-						okHttp()
+						okHttp(get())
 							.cache(CacheUtils.createHttpCache(applicationContext))
 							.build()
 					}
@@ -100,11 +108,11 @@ class KotatsuApp : Application() {
 		}
 	}

-	private fun initCoil() {
+	private fun initCoil(cookieJar: CookieJar) {
 		Coil.setImageLoader(
 			ImageLoader.Builder(applicationContext)
 				.okHttpClient(
-					okHttp()
+					okHttp(cookieJar)
 						.cache(CoilUtils.createDefaultCache(applicationContext))
 						.build()
 				).componentRegistry(
@@ -124,7 +132,7 @@ class KotatsuApp : Application() {
 		}
 	}

-	private fun okHttp() = OkHttpClient.Builder().apply {
+	private fun okHttp(cookieJar: CookieJar) = OkHttpClient.Builder().apply {
 		connectTimeout(20, TimeUnit.SECONDS)
 		readTimeout(60, TimeUnit.SECONDS)
 		writeTimeout(20, TimeUnit.SECONDS)
--- a/app/src/main/java/org/koitharu/kotatsu/core/local/cookies/PersistentCookieJar.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/core/local/cookies/PersistentCookieJar.kt
@@ -15,10 +15,10 @@
 */
 package org.koitharu.kotatsu.core.local.cookies

-import org.koitharu.kotatsu.core.local.cookies.persistence.CookiePersistor
 import okhttp3.Cookie
 import okhttp3.HttpUrl
 import org.koitharu.kotatsu.core.local.cookies.cache.CookieCache
+import org.koitharu.kotatsu.core.local.cookies.persistence.CookiePersistor
 import java.util.*

 class PersistentCookieJar(
--- a/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt
@@ -22,6 +22,7 @@ enum class MangaSource(
 	HENCHAN("Хентай-тян", "ru", HenChanRepository::class.java),
 	YAOICHAN("Яой-тян", "ru", YaoiChanRepository::class.java),
 	MANGATOWN("MangaTown", "en", MangaTownRepository::class.java),
-	MANGALIB("MangaLib", "ru", MangaLibRepository::class.java)
+	MANGALIB("MangaLib", "ru", MangaLibRepository::class.java),
+	NUDEMOON("Nude-Moon", "ru", NudeMoonRepository::class.java)
 	// HENTAILIB("HentaiLib", "ru", HentaiLibRepository::class.java)
 }
--- a/app/src/main/java/org/koitharu/kotatsu/core/parser/site/NudeMoonRepository.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/core/parser/site/NudeMoonRepository.kt
@@ -0,0 +1,151 @@
+package org.koitharu.kotatsu.core.parser.site
+
+import org.koitharu.kotatsu.R
+import org.koitharu.kotatsu.core.exceptions.ParseException
+import org.koitharu.kotatsu.core.model.*
+import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
+import org.koitharu.kotatsu.domain.MangaLoaderContext
+import org.koitharu.kotatsu.utils.ext.*
+import java.util.regex.Pattern
+
+class NudeMoonRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepository(loaderContext) {
+
+	override val source = MangaSource.NUDEMOON
+
+	override val sortOrders = setOf(SortOrder.NEWEST, SortOrder.POPULARITY, SortOrder.RATING)
+
+	init {
+		loaderContext.insertCookies(
+			conf.getDomain(DEFAULT_DOMAIN),
+			"NMfYa=1;",
+			"nm_mobile=0;"
+		)
+	}
+
+	override suspend fun getList(
+		offset: Int,
+		query: String?,
+		sortOrder: SortOrder?,
+		tag: MangaTag?
+	): List<Manga> {
+		val domain = conf.getDomain(DEFAULT_DOMAIN)
+		val url = when {
+			!query.isNullOrEmpty() -> "https://$domain//search?stext=${query.urlEncoded()}&rowstart=$offset"
+			tag != null -> "https://$domain/tags/${tag.key}&rowstart=$offset"
+			else -> "https://$domain/all_manga?${getSortKey(sortOrder)}&rowstart=$offset"
+		}
+		val doc = loaderContext.httpGet(url) {
+			addHeader("Cookie", "NMfYa=1; nm_mobile=0;")
+		}.parseHtml()
+		val root = doc.body().run {
+			selectFirst("td.shoutbox") ?: selectFirst("td.main-bg")
+		} ?: throw ParseException("Cannot find root")
+		return root.select("table.news_pic2").mapNotNull { row ->
+			val a = row.selectFirst("td.bg_style1")?.selectFirst("a")
+				?: return@mapNotNull null
+			val href = a.absUrl("href")
+			val title = a.selectFirst("h2")?.text().orEmpty()
+			val info = row.selectFirst("div.tbl2") ?: return@mapNotNull null
+			Manga(
+				id = href.longHashCode(),
+				url = href,
+				title = title.substringAfter(" / "),
+				altTitle = title.substringBefore(" / ", "")
+					.takeUnless { it.isBlank() },
+				author = info.getElementsContainingOwnText("Автор:")?.firstOrNull()
+					?.nextElementSibling()?.ownText(),
+				coverUrl = row.selectFirst("img.news_pic2")?.absUrl("src")
+					.orEmpty(),
+				tags = row.selectFirst("span.tag-links")?.select("a")
+					?.mapToSet {
+						MangaTag(
+							title = it.text(),
+							key = it.attr("href").substringAfterLast('/').urlEncoded(),
+							source = source
+						)
+					}.orEmpty(),
+				source = source
+			)
+		}
+	}
+
+	override suspend fun getDetails(manga: Manga): Manga {
+		val doc = loaderContext.httpGet(manga.url).parseHtml()
+		val root = doc.body().selectFirst("table.shoutbox")
+			?: throw ParseException("Cannot find root")
+		val info = root.select("div.tbl2")
+		return manga.copy(
+			description = info.select("div.blockquote").lastOrNull()?.html(),
+			tags = info.select("span.tag-links").firstOrNull()?.select("a")?.mapToSet {
+				MangaTag(
+					title = it.text(),
+					key = it.attr("href").substringAfterLast('/').urlEncoded(),
+					source = source
+				)
+			} ?: manga.tags,
+			chapters = listOf(
+				MangaChapter(
+					id = manga.id,
+					url = manga.url,
+					source = source,
+					number = 1,
+					name = manga.title
+				)
+			)
+		)
+	}
+
+	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+		conf.getDomain(DEFAULT_DOMAIN)
+		val doc = loaderContext.httpGet(chapter.url).parseHtml()
+		val root = doc.body().selectFirst("td.main-body")
+			?: throw ParseException("Cannot find root")
+		return root.getElementsByAttributeValueMatching("href", pageUrlPatter).mapNotNull { a ->
+			val url = a.absUrl("href")
+			MangaPage(
+				id = url.longHashCode(),
+				url = url,
+				preview = a.selectFirst("img")?.absUrl("src"),
+				source = source
+			)
+		}
+	}
+
+	override suspend fun getPageFullUrl(page: MangaPage): String {
+		val doc = loaderContext.httpGet(page.url).parseHtml()
+		return doc.body().getElementById("gallery").attr("src").inContextOf(doc)
+	}
+
+	override suspend fun getTags(): Set<MangaTag> {
+		val domain = conf.getDomain(DEFAULT_DOMAIN)
+		val doc = loaderContext.httpGet("https://$domain/all_manga").parseHtml()
+		val root = doc.body().getElementsContainingOwnText("Поиск манги по тегам")
+			.firstOrNull()?.parents()?.find { it.tag().normalName() == "tbody" }
+			?.selectFirst("td.textbox")?.selectFirst("td.small")
+			?: throw ParseException("Tags root not found")
+		return root.select("a").mapToSet {
+			MangaTag(
+				title = it.text(),
+				key = it.attr("href").substringAfterLast('/')
+					.removeSuffix("+").urlEncoded(),
+				source = source
+			)
+		}
+	}
+
+	override fun onCreatePreferences() = setOf(R.string.key_parser_domain)
+
+	private fun getSortKey(sortOrder: SortOrder?) =
+		when (sortOrder ?: sortOrders.minByOrNull { it.ordinal }) {
+			SortOrder.POPULARITY -> "views"
+			SortOrder.NEWEST -> "date"
+			SortOrder.RATING -> "like"
+			else -> "like"
+		}
+
+	private companion object {
+
+		private const val DEFAULT_DOMAIN = "nude-moon.me"
+		private val pageUrlPatter = Pattern.compile(".*\\?page=[0-9]+$")
+	}
+}
--- a/app/src/main/java/org/koitharu/kotatsu/domain/MangaLoaderContext.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/domain/MangaLoaderContext.kt
@@ -1,9 +1,6 @@
 package org.koitharu.kotatsu.domain

-import okhttp3.FormBody
-import okhttp3.OkHttpClient
-import okhttp3.Request
-import okhttp3.Response
+import okhttp3.*
 import org.koin.core.component.KoinComponent
 import org.koin.core.component.get
 import org.koin.core.component.inject
@@ -14,6 +11,7 @@ import org.koitharu.kotatsu.utils.ext.await
 open class MangaLoaderContext : KoinComponent {

 	private val okHttp by inject<OkHttpClient>()
+	private val cookieJar by inject<CookieJar>()

 	suspend fun httpGet(url: String, block: (Request.Builder.() -> Unit)? = null): Response {
 		val request = Request.Builder()
@@ -44,4 +42,19 @@ open class MangaLoaderContext : KoinComponent {
 	}

 	open fun getSettings(source: MangaSource) = SourceConfig(get(), source)
+
+	fun insertCookies(domain: String, vararg cookies: String) {
+		val url = HttpUrl.Builder()
+			.scheme(SCHEME_HTTP)
+			.host(domain)
+			.build()
+		cookieJar.saveFromResponse(url, cookies.mapNotNull {
+			Cookie.parse(url, it)
+		})
+	}
+
+	private companion object {
+
+		private const val SCHEME_HTTP = "http"
+	}
 }
--- a/app/src/main/java/org/koitharu/kotatsu/ui/list/remote/RemoteListPresenter.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/ui/list/remote/RemoteListPresenter.kt
@@ -23,7 +23,7 @@ class RemoteListPresenter : BasePresenter<MangaListView<Unit>>() {
 		presenterScope.launch {
 			viewState.onLoadingStateChanged(true)
 			try {
-				val list = withContext(Dispatchers.IO) {
+				val list = withContext(Dispatchers.Default) {
 					MangaProviderFactory.create(source).getList(
 						offset = offset,
 						sortOrder = filter?.sortOrder,
@@ -64,7 +64,7 @@ class RemoteListPresenter : BasePresenter<MangaListView<Unit>>() {
 		isFilterInitialized = true
 		presenterScope.launch {
 			try {
-				val (sorts, tags) = withContext(Dispatchers.IO) {
+				val (sorts, tags) = withContext(Dispatchers.Default) {
 					val repo = MangaProviderFactory.create(source)
 					repo.sortOrders.sortedBy { it.ordinal } to repo.getTags().sortedBy { it.title }
 				}
--- a/app/src/main/java/org/koitharu/kotatsu/ui/reader/base/PageHolderDelegate.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/ui/reader/base/PageHolderDelegate.kt
@@ -55,9 +55,9 @@ class PageHolderDelegate(
 					state = State.CONVERTED
 					callback.onImageReady(file.toUri())
 				} catch (e2: Throwable) {
-					e2.addSuppressed(e)
+					e.addSuppressed(e2)
 					state = State.ERROR
-					callback.onError(e2)
+					callback.onError(e)
 				}
 			}
 		} else {
@@ -73,6 +73,7 @@ class PageHolderDelegate(
 			try {
 				val file = withContext(Dispatchers.IO) {
 					val pageUrl = MangaProviderFactory.create(data.source).getPageFullUrl(data)
+					check(pageUrl.isNotEmpty()) { "Cannot obtain full image url" }
 					loader.loadFile(pageUrl, force)
 				}
 				this@PageHolderDelegate.file = file
--- a/app/src/main/java/org/koitharu/kotatsu/utils/ext/CollectionExt.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/utils/ext/CollectionExt.kt
@@ -1,5 +1,7 @@
 package org.koitharu.kotatsu.utils.ext

+import androidx.collection.ArraySet
+
 fun <T> MutableCollection<T>.replaceWith(subject: Iterable<T>) {
 	clear()
 	addAll(subject)
@@ -24,4 +26,11 @@ inline fun <T> Iterable<T>.sumByLong(selector: (T) -> Long): Long {
 fun <T> List<T>.medianOrNull(): T? = when {
 	isEmpty() -> null
 	else -> get((size / 2).coerceIn(indices))
+}
+
+inline fun <T, R> Collection<T>.mapToSet(transform: (T) -> R): Set<R> {
+	val destination = ArraySet<R>(size)
+	for (item in this)
+		destination.add(transform(item))
+	return destination
 }
--- a/app/src/main/java/org/koitharu/kotatsu/utils/ext/ParseExt.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/utils/ext/ParseExt.kt
@@ -5,7 +5,9 @@ import okhttp3.internal.closeQuietly
 import org.json.JSONArray
 import org.json.JSONObject
 import org.jsoup.Jsoup
+import org.jsoup.internal.StringUtil
 import org.jsoup.nodes.Document
+import org.jsoup.nodes.Node
 import org.jsoup.select.Elements

 fun Response.parseHtml(): Document {
@@ -59,4 +61,12 @@ inline fun Elements.findText(predicate: (String) -> Boolean): String? {
 		}
 	}
 	return null
+}
+
+fun String.inContextOf(node: Node): String {
+	return if (this.isEmpty()) {
+		""
+	} else {
+		StringUtil.resolve(node.baseUri(), this)
+	}
 }
--- a/app/src/main/java/org/koitharu/kotatsu/utils/ext/StringExt.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/utils/ext/StringExt.kt
@@ -15,6 +15,7 @@ fun String.longHashCode(): Long {
 	return h
 }

+@Deprecated("Use String.inContextOf")
 fun String.withDomain(domain: String, ssl: Boolean = true) = when {
 	this.startsWith("//") -> buildString {
 		append("http")