Fix some manga sources

This commit is contained in:
Koitharu
2021-01-27 18:24:48 +02:00
parent 7f37c1f99e
commit 0e384c134d
22 changed files with 138 additions and 78 deletions

View File

@@ -48,7 +48,7 @@ android {
}
testOptions {
unitTests.includeAndroidResources = true
unitTests.returnDefaultValues = true
unitTests.returnDefaultValues = false
}
}
tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all {

View File

@@ -0,0 +1,13 @@
package org.koitharu.kotatsu.core.exceptions
import androidx.annotation.StringRes
import org.koitharu.kotatsu.R
import org.koitharu.kotatsu.core.exceptions.resolve.ResolvableException
class AuthRequiredException(
val url: String
) : RuntimeException("Authorization required"), ResolvableException {
@StringRes
override val resolveTextId: Int = R.string.sign_in
}

View File

@@ -1,5 +1,6 @@
package org.koitharu.kotatsu.core.exceptions
import androidx.annotation.StringRes
import okio.IOException
import org.koitharu.kotatsu.R
import org.koitharu.kotatsu.core.exceptions.resolve.ResolvableException
@@ -8,5 +9,6 @@ class CloudFlareProtectedException(
val url: String
) : IOException("Protected by CloudFlare"), ResolvableException {
@StringRes
override val resolveTextId: Int = R.string.captcha_solve
}

View File

@@ -5,6 +5,7 @@ import androidx.fragment.app.FragmentManager
import androidx.lifecycle.LifecycleOwner
import kotlinx.coroutines.suspendCancellableCoroutine
import org.koitharu.kotatsu.browser.cloudflare.CloudFlareDialog
import org.koitharu.kotatsu.core.exceptions.AuthRequiredException
import org.koitharu.kotatsu.core.exceptions.CloudFlareProtectedException
import kotlin.coroutines.Continuation
import kotlin.coroutines.resume
@@ -18,6 +19,7 @@ class ExceptionResolver(
suspend fun resolve(e: ResolvableException): Boolean = when (e) {
is CloudFlareProtectedException -> resolveCF(e.url)
is AuthRequiredException -> false //TODO
else -> false
}

View File

@@ -26,7 +26,8 @@ enum class MangaSource(
YAOICHAN("Яой-тян", "ru", YaoiChanRepository::class.java),
MANGATOWN("MangaTown", "en", MangaTownRepository::class.java),
MANGALIB("MangaLib", "ru", MangaLibRepository::class.java),
NUDEMOON("Nude-Moon", "ru", NudeMoonRepository::class.java),
// NUDEMOON("Nude-Moon", "ru", NudeMoonRepository::class.java),
MANGAREAD("MangaRead", "en", MangareadRepository::class.java),
REMANGA("Remanga", "ru", RemangaRepository::class.java),
HENTAILIB("HentaiLib", "ru", HentaiLibRepository::class.java);

View File

@@ -20,7 +20,7 @@ val parserModule
factory<MangaRepository>(named(MangaSource.YAOICHAN)) { YaoiChanRepository(get()) }
factory<MangaRepository>(named(MangaSource.MANGATOWN)) { MangaTownRepository(get()) }
factory<MangaRepository>(named(MangaSource.MANGALIB)) { MangaLibRepository(get()) }
factory<MangaRepository>(named(MangaSource.NUDEMOON)) { NudeMoonRepository(get()) }
// factory<MangaRepository>(named(MangaSource.NUDEMOON)) { NudeMoonRepository(get()) }
factory<MangaRepository>(named(MangaSource.MANGAREAD)) { MangareadRepository(get()) }
factory<MangaRepository>(named(MangaSource.REMANGA)) { RemangaRepository(get()) }
factory<MangaRepository>(named(MangaSource.HENTAILIB)) { HentaiLibRepository(get()) }

View File

@@ -1,6 +1,6 @@
package org.koitharu.kotatsu.core.parser
import android.net.Uri
import okhttp3.HttpUrl.Companion.toHttpUrl
import org.koitharu.kotatsu.base.domain.MangaLoaderContext
import org.koitharu.kotatsu.core.model.MangaPage
import org.koitharu.kotatsu.core.model.MangaSource
@@ -26,10 +26,9 @@ abstract class RemoteMangaRepository(
abstract fun onCreatePreferences(): Set<String>
protected fun generateUid(url: String): Long {
val uri = Uri.parse(url)
val path = uri.path ?: error("Cannot generate uid: bad uri \"$url\"")
val uri = url.toHttpUrl()
val x = source.name.hashCode()
val y = path.hashCode()
val y = "${uri.encodedPath}?${uri.query}".hashCode()
return (x.toLong() shl 32) or (y.toLong() and 0xffffffffL)
}

View File

@@ -16,9 +16,10 @@ abstract class GroupleRepository(loaderContext: MangaLoaderContext) :
protected abstract val defaultDomain: String
override val sortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED, SortOrder.POPULARITY,
SortOrder.NEWEST, SortOrder.RATING
//FIXME SortOrder.ALPHABETICAL
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.NEWEST,
SortOrder.RATING
)
override suspend fun getList(

View File

@@ -5,6 +5,7 @@ import androidx.collection.arraySetOf
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.base.domain.MangaLoaderContext
import org.koitharu.kotatsu.core.exceptions.AuthRequiredException
import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
@@ -144,6 +145,9 @@ open class MangaLibRepository(loaderContext: MangaLoaderContext) :
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = loaderContext.httpGet(chapter.url).parseHtml()
if (doc.location()?.endsWith("/register") == true) {
throw AuthRequiredException("/login".inContextOf(doc))
}
val scripts = doc.head().select("script")
val pg = doc.body().getElementById("pg").html()
.substringAfter('=')

View File

@@ -49,7 +49,8 @@ class MangareadRepository(
Manga(
id = href.longHashCode(),
url = href,
coverUrl = div.selectFirst("img").absUrl("src"),
coverUrl = div.selectFirst("img").attr("data-srcset")
.split(',').firstOrNull()?.substringBeforeLast(' ').orEmpty(),
title = summary.selectFirst("h3").text(),
rating = div.selectFirst("span.total_votes")?.ownText()
?.toFloatOrNull()?.div(5f) ?: -1f,
@@ -75,13 +76,17 @@ class MangareadRepository(
override suspend fun getTags(): Set<MangaTag> {
val domain = conf.getDomain(DOMAIN)
val doc = loaderContext.httpGet("https://$domain/manga/").parseHtml()
val root = doc.body().getElementById("main-sidebar")
.selectFirst(".genres_wrap")
.selectFirst("ul")
return root.select("li").mapToSet { li ->
val root = doc.body().selectFirst("header")
.selectFirst("ul.second-menu")
return root.select("li").mapNotNullToSet { li ->
val a = li.selectFirst("a")
val href = a.attr("href").removeSuffix("/")
.substringAfterLast("genres/", "")
if (href.isEmpty()) {
return@mapNotNullToSet null
}
MangaTag(
key = a.attr("href").removeSuffix("/").substringAfterLast('/'),
key = href,
title = a.text(),
source = MangaSource.MANGAREAD
)
@@ -119,7 +124,8 @@ class MangareadRepository(
} ?: manga.tags,
description = root2.selectFirst("div.description-summary")
?.selectFirst("div.summary__content")
?.select("p")?.drop(1)
?.select("p")
?.filterNot { it.ownText().startsWith("A brief description") }
?.joinToString { it.html() },
chapters = doc2.select("li").asReversed().mapIndexed { i, li ->
val a = li.selectFirst("a")
@@ -142,7 +148,7 @@ class MangareadRepository(
?: throw ParseException("Root not found")
return root.select("div.page-break").map { div ->
val img = div.selectFirst("img")
val url = img.absUrl("src")
val url = img.absUrl("data-src")
MangaPage(
id = url.longHashCode(),
url = url,

View File

@@ -1,15 +1,6 @@
package org.koitharu.kotatsu.core.parser.site
import androidx.collection.arraySetOf
import org.koitharu.kotatsu.base.domain.MangaLoaderContext
import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
import org.koitharu.kotatsu.core.prefs.SourceSettings
import org.koitharu.kotatsu.utils.ext.*
import java.util.*
import java.util.regex.Pattern
/*
class NudeMoonRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepository(loaderContext) {
override val source = MangaSource.NUDEMOON
@@ -153,4 +144,4 @@ class NudeMoonRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposit
private const val DEFAULT_DOMAIN = "nude-moon.me"
private val pageUrlPatter = Pattern.compile(".*\\?page=[0-9]+$")
}
}
}*/

View File

@@ -5,6 +5,6 @@ import org.koitharu.kotatsu.core.model.MangaSource
class SelfMangaRepository(loaderContext: MangaLoaderContext) : GroupleRepository(loaderContext) {
override val defaultDomain = "selfmanga.ru"
override val defaultDomain = "selfmanga.live"
override val source = MangaSource.SELFMANGA
}

View File

@@ -1,6 +1,7 @@
package org.koitharu.kotatsu.list.ui.model
import org.koitharu.kotatsu.R
import org.koitharu.kotatsu.core.exceptions.AuthRequiredException
import org.koitharu.kotatsu.core.exceptions.CloudFlareProtectedException
import org.koitharu.kotatsu.core.exceptions.resolve.ResolvableException
import org.koitharu.kotatsu.core.model.Manga
@@ -58,6 +59,7 @@ fun Throwable.toErrorFooter() = ErrorFooter(
)
private fun getErrorIcon(error: Throwable) = when (error) {
is AuthRequiredException,
is CloudFlareProtectedException -> R.drawable.ic_denied_large
else -> R.drawable.ic_error_large
}

View File

@@ -45,10 +45,7 @@ import org.koitharu.kotatsu.utils.GridTouchHelper
import org.koitharu.kotatsu.utils.ScreenOrientationHelper
import org.koitharu.kotatsu.utils.ShareHelper
import org.koitharu.kotatsu.utils.anim.Motion
import org.koitharu.kotatsu.utils.ext.hasGlobalPoint
import org.koitharu.kotatsu.utils.ext.hideAnimated
import org.koitharu.kotatsu.utils.ext.hitTest
import org.koitharu.kotatsu.utils.ext.showAnimated
import org.koitharu.kotatsu.utils.ext.*
class ReaderActivity : BaseFullscreenActivity<ActivityReaderBinding>(),
ChaptersDialog.OnChapterChangeListener,
@@ -213,7 +210,7 @@ class ReaderActivity : BaseFullscreenActivity<ActivityReaderBinding>(),
private fun onError(e: Throwable) {
val dialog = AlertDialog.Builder(this)
.setTitle(R.string.error_occurred)
.setMessage(e.message)
.setMessage(e.getDisplayMessage(resources))
.setPositiveButton(R.string.close, null)
if (viewModel.content.value?.pages.isNullOrEmpty()) {
dialog.setOnDismissListener {

View File

@@ -85,7 +85,7 @@ class MainSettingsFragment : BasePreferenceFragment(R.string.settings),
findPreference<PreferenceScreen>(AppSettings.KEY_REMOTE_SOURCES)?.run {
val total = MangaSource.values().size - 1
summary = getString(
R.string.enabled_d_from_d, total - settings.hiddenSources.size, total
R.string.enabled_d_of_d, total - settings.hiddenSources.size, total
)
}
}

View File

@@ -4,10 +4,7 @@ import android.content.res.Resources
import android.util.Log
import kotlinx.coroutines.delay
import org.koitharu.kotatsu.R
import org.koitharu.kotatsu.core.exceptions.CloudFlareProtectedException
import org.koitharu.kotatsu.core.exceptions.EmptyHistoryException
import org.koitharu.kotatsu.core.exceptions.UnsupportedFileException
import org.koitharu.kotatsu.core.exceptions.WrongPasswordException
import org.koitharu.kotatsu.core.exceptions.*
import java.io.FileNotFoundException
import java.net.SocketTimeoutException
@@ -28,6 +25,7 @@ suspend inline fun <T, R> T.retryUntilSuccess(maxAttempts: Int, action: T.() ->
}
fun Throwable.getDisplayMessage(resources: Resources) = when (this) {
is AuthRequiredException -> resources.getString(R.string.auth_required)
is CloudFlareProtectedException -> resources.getString(R.string.captcha_required)
is UnsupportedOperationException -> resources.getString(R.string.operation_not_supported)
is UnsupportedFileException -> resources.getString(R.string.text_file_not_supported)

View File

@@ -103,7 +103,7 @@
<string name="large_manga_save_confirm">В этой манге %s. Вы уверены, что хотите сохранить их все?</string>
<string name="save_manga">Сохранить мангу</string>
<string name="notifications">Уведомления</string>
<string name="enabled_d_from_d">Включено %1$d из %2$d</string>
<string name="enabled_d_of_d">Включено %1$d из %2$d</string>
<string name="new_chapters">Новые главы</string>
<string name="show_notification_new_chapters">Уведомлять об обновлении манги, которую Вы читаете</string>
<string name="download">Загрузить</string>
@@ -200,4 +200,6 @@
<string name="power_optimization_already_disabled">Отпимизация батареи уже отключена</string>
<string name="new_chapters_checking">Проверка новых глав</string>
<string name="reverse">В обратном порядке</string>
<string name="sign_in">Войти</string>
<string name="auth_required">Для просмотра этого контента требуется авторизация</string>
</resources>

View File

@@ -104,7 +104,7 @@
<string name="large_manga_save_confirm">This manga has %s. Do you want to save all of it?</string>
<string name="save_manga">Save manga</string>
<string name="notifications">Notifications</string>
<string name="enabled_d_from_d" tools:ignore="PluralsCandidate">Enabled %1$d from %2$d</string>
<string name="enabled_d_of_d" tools:ignore="PluralsCandidate">Enabled %1$d of %2$d</string>
<string name="new_chapters">New chapters</string>
<string name="show_notification_new_chapters">Notify about updates of manga you are reading</string>
<string name="download">Download</string>
@@ -202,4 +202,6 @@
<string name="disable_power_optimization">Disable power optimization</string>
<string name="new_chapters_checking">New chapters checking</string>
<string name="reverse">Reverse</string>
<string name="sign_in">Sign in</string>
<string name="auth_required">You should authorize to view this content</string>
</resources>

View File

@@ -17,6 +17,7 @@ import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.core.network.UserAgentInterceptor
import org.koitharu.kotatsu.core.prefs.SourceSettings
import org.koitharu.kotatsu.utils.AssertX
import org.koitharu.kotatsu.utils.ext.isDistinctBy
import java.util.concurrent.TimeUnit
@RunWith(Parameterized::class)
@@ -32,58 +33,93 @@ class RemoteRepositoryTest(source: MangaSource) : KoinTest {
@Test
fun list() {
val list = runBlocking { repo.getList(60) }
Assert.assertFalse(list.isEmpty())
Assert.assertFalse("List is empty", list.isEmpty())
Assert.assertTrue("Mangas are not distinct", list.isDistinctBy { it.id })
val item = list.random()
AssertX.assertContentType(item.coverUrl, "image/*")
AssertX.assertContentType(item.url, "text/html", "application/json")
Assert.assertFalse(item.title.isBlank())
AssertX.assertContentType("Bad cover at ${item.url}", item.coverUrl, "image/*")
AssertX.assertContentType(
"Wrong content type at ${item.url}",
item.url,
"text/html",
"application/json"
)
Assert.assertFalse("Title is blank at ${item.url}", item.title.isBlank())
}
@Test
fun search() {
val list = runBlocking { repo.getList(0, query = "tail") }
Assert.assertFalse(list.isEmpty())
Assert.assertFalse("List is empty", list.isEmpty())
Assert.assertTrue("Mangas are not distinct", list.isDistinctBy { it.id })
val item = list.random()
AssertX.assertContentType(item.coverUrl, "image/*")
AssertX.assertContentType(item.url, "text/html", "application/json")
Assert.assertFalse(item.title.isBlank())
AssertX.assertContentType("Bad cover at ${item.url}", item.coverUrl, "image/*")
AssertX.assertContentType(
"Wrong content type at ${item.url}",
item.url,
"text/html",
"application/json"
)
Assert.assertFalse("Title is blank at ${item.url}", item.title.isBlank())
}
@Test
fun tags() {
val tags = runBlocking { repo.getTags() }
Assert.assertFalse(tags.isEmpty())
Assert.assertFalse("No tags found", tags.isEmpty())
val tag = tags.random()
Assert.assertFalse(tag.key.isBlank())
Assert.assertFalse(tag.title.isBlank())
Assert.assertFalse("Tag title is blank for ${tag}", tag.key.isBlank())
Assert.assertFalse("Tag title is blank for ${tag}", tag.title.isBlank())
val list = runBlocking { repo.getList(0, tag = tag) }
Assert.assertFalse(list.isEmpty())
Assert.assertFalse("List is empty", list.isEmpty())
val item = list.random()
AssertX.assertContentType(item.coverUrl, "image/*")
AssertX.assertContentType(item.url, "text/html", "application/json")
Assert.assertFalse(item.title.isBlank())
AssertX.assertContentType("Bad cover at ${item.coverUrl}", item.coverUrl, "image/*")
AssertX.assertContentType(
"Wrong response from ${item.url}",
item.url,
"text/html",
"application/json"
)
Assert.assertFalse("Title is blank at ${item.url}", item.title.isBlank())
}
@Test
fun details() {
val manga = runBlocking { repo.getList(0) }.random()
val details = runBlocking { repo.getDetails(manga) }
Assert.assertFalse(details.chapters.isNullOrEmpty())
Assert.assertFalse(details.description.isNullOrEmpty())
val chapter = details.chapters!!.random()
Assert.assertFalse(chapter.name.isBlank())
AssertX.assertContentType(chapter.url, "text/html", "application/json")
Assert.assertFalse("Chapter is empty at ${details.url}", details.chapters.isNullOrEmpty())
Assert.assertFalse(
"Description is empty at ${details.url}",
details.description.isNullOrEmpty()
)
Assert.assertTrue(
"Chapters are not distinct",
details.chapters.orEmpty().isDistinctBy { it.id })
val chapter = details.chapters?.randomOrNull() ?: return
Assert.assertFalse(
"Chapter name missing at ${details.url}:${chapter.number}",
chapter.name.isBlank()
)
AssertX.assertContentType(
"Chapter response wrong at ${chapter.url}",
chapter.url,
"text/html",
"application/json"
)
}
@Test
fun pages() {
val manga = runBlocking { repo.getList(0) }.random()
val details = runBlocking { repo.getDetails(manga) }
val pages = runBlocking { repo.getPages(details.chapters!!.random()) }
Assert.assertFalse(pages.isEmpty())
val page = pages.random()
val chapter = checkNotNull(details.chapters?.randomOrNull()) {
"No chapters at ${details.url}"
}
val pages = runBlocking { repo.getPages(chapter) }
Assert.assertFalse("Cannot find any page at ${chapter.url}", pages.isEmpty())
Assert.assertTrue("Pages are not distinct", pages.isDistinctBy { it.id })
val page = pages.randomOrNull() ?: return
val fullUrl = runBlocking { repo.getPageUrl(page) }
AssertX.assertContentType(fullUrl, "image/*")
AssertX.assertContentType("Wrong page response from $fullUrl", fullUrl, "image/*")
}
companion object {

View File

@@ -3,17 +3,25 @@ package org.koitharu.kotatsu.parsers
import okhttp3.Cookie
import okhttp3.CookieJar
import okhttp3.HttpUrl
import org.koitharu.kotatsu.core.network.cookies.cache.SetCookieCache
class TemporaryCookieJar : CookieJar {
private val cache = SetCookieCache()
private val cache = HashMap<CookieKey, Cookie>()
override fun loadForRequest(url: HttpUrl): List<Cookie> {
return cache.toList()
val time = System.currentTimeMillis()
return cache.values.filter { it.matches(url) && it.expiresAt < time }
}
override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) {
cache.addAll(cookies)
cookies.forEach {
val key = CookieKey(url.host, it.name)
cache[key] = it
}
}
private data class CookieKey(
val host: String,
val name: String
)
}

View File

@@ -11,18 +11,14 @@ object AssertX : KoinComponent {
private val okHttp by inject<OkHttpClient>()
fun assertContentType(url: String, vararg types: String) {
Assert.assertFalse("URL is empty", url.isEmpty())
fun assertContentType(message: String, url: String, vararg types: String) {
Assert.assertFalse("URL is empty: $message", url.isEmpty())
val request = Request.Builder()
.url(url)
.head()
.build()
val response = okHttp.newCall(request).execute()
when (val code = response.code) {
/*HttpURLConnection.HTTP_MOVED_PERM,
HttpURLConnection.HTTP_MOVED_TEMP -> {
assertContentType(cn.getHeaderField("Location"), *types)
}*/
HttpURLConnection.HTTP_OK -> {
val type = response.body!!.contentType()
Assert.assertTrue(types.any {
@@ -30,7 +26,7 @@ object AssertX : KoinComponent {
type?.type == x[0] && (x[1] == "*" || type.subtype == x[1])
})
}
else -> Assert.fail("Invalid response code $code at $url")
else -> Assert.fail("Invalid response code $code at $url: $message")
}
}

View File

@@ -6,7 +6,7 @@ buildscript {
jcenter()
}
dependencies {
classpath 'com.android.tools.build:gradle:4.1.1'
classpath 'com.android.tools.build:gradle:4.1.2'
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
// NOTE: Do not place your application dependencies here; they belong