prifak/Kotatsu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MintManga parser

Browse Source
This commit is contained in:
Koitharu
2020-02-02 15:16:24 +02:00
parent e86f4b625a
commit 6d4a77b023
17 changed files with 226 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app/src/main/java/org/koitharu/kotatsu/core/exceptions/ParseException.kt Normal file
View File

@@ -0,0 +1,3 @@
package org.koitharu.kotatsu.core.exceptions
class ParseException(message: String? = null, cause: Throwable? = null) : RuntimeException(message, cause)

2
app/src/main/java/org/koitharu/kotatsu/core/model/Manga.kt
View File

@@ -13,7 +13,7 @@ data class Manga(
val coverUrl: String,
val largeCoverUrl: String? = null,
val summary: String,
val description: CharSequence? = null,
val description: String? = null, //HTML
val tags: Set<MangaTag> = emptySet(),
val state: MangaState? = null,
val chapters: List<MangaChapter>? = null,

8
app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt
View File

@@ -2,10 +2,12 @@ package org.koitharu.kotatsu.core.model
import android.os.Parcelable
import kotlinx.android.parcel.Parcelize
import org.koitharu.kotatsu.domain.MangaRepository
import org.koitharu.kotatsu.domain.repository.ReadmangaRepository
import org.koitharu.kotatsu.core.parser.MangaRepository
import org.koitharu.kotatsu.core.parser.site.MintMangaRepository
import org.koitharu.kotatsu.core.parser.site.ReadmangaRepository
@Parcelize
enum class MangaSource(val title: String, val cls: Class<out MangaRepository>): Parcelable {
READMANGA_RU("ReadManga", ReadmangaRepository::class.java)
READMANGA_RU("ReadManga", ReadmangaRepository::class.java),
MINTMANGA("MintManga", MintMangaRepository::class.java)
}

2
app/src/main/java/org/koitharu/kotatsu/core/model/SortOrder.kt
View File

@@ -1,5 +1,5 @@
package org.koitharu.kotatsu.core.model
enum class SortOrder {
ALPHABETICAL, POPULARITY, UPDATED, NEWEST
ALPHABETICAL, POPULARITY, UPDATED, NEWEST, RATING
}

9
app/src/main/java/org/koitharu/kotatsu/domain/BaseMangaRepository.kt → app/src/main/java/org/koitharu/kotatsu/core/parser/BaseMangaRepository.kt
View File

@@ -1,13 +1,18 @@
package org.koitharu.kotatsu.domain
package org.koitharu.kotatsu.core.parser
import org.koitharu.kotatsu.core.model.MangaPage
import org.koitharu.kotatsu.core.model.MangaTag
import org.koitharu.kotatsu.core.model.SortOrder
import org.koitharu.kotatsu.domain.MangaLoaderContext
abstract class BaseMangaRepository(protected val loaderContext: MangaLoaderContext) : MangaRepository {
abstract class BaseMangaRepository(protected val loaderContext: MangaLoaderContext) :
MangaRepository {
override val sortOrders: Set<SortOrder> get() = emptySet()
override val isSearchAvailable get() = true
override suspend fun getPageFullUrl(page: MangaPage) : String = page.url
override suspend fun getTags(): Set<MangaTag> = emptySet()
}

11
app/src/main/java/org/koitharu/kotatsu/domain/MangaRepository.kt → app/src/main/java/org/koitharu/kotatsu/core/parser/MangaRepository.kt
View File

@@ -1,9 +1,6 @@
package org.koitharu.kotatsu.domain
package org.koitharu.kotatsu.core.parser
import org.koitharu.kotatsu.core.model.Manga
import org.koitharu.kotatsu.core.model.MangaChapter
import org.koitharu.kotatsu.core.model.MangaPage
import org.koitharu.kotatsu.core.model.SortOrder
import org.koitharu.kotatsu.core.model.*
interface MangaRepository {
@@ -11,11 +8,13 @@ interface MangaRepository {
val isSearchAvailable: Boolean
suspend fun getList(offset: Int, query: String? = null, sortOrder: SortOrder? = null, tags: Set<String>? = null): List<Manga>
suspend fun getList(offset: Int, query: String? = null, sortOrder: SortOrder? = null, tag: MangaTag? = null): List<Manga>
suspend fun getDetails(manga: Manga) : Manga
suspend fun getPages(chapter: MangaChapter) : List<MangaPage>
suspend fun getPageFullUrl(page: MangaPage) : String
suspend fun getTags(): Set<MangaTag>
}

68
app/src/main/java/org/koitharu/kotatsu/domain/repository/ReadmangaRepository.kt → app/src/main/java/org/koitharu/kotatsu/core/parser/site/GroupleRepository.kt
View File

@@ -1,28 +1,44 @@
package org.koitharu.kotatsu.domain.repository
package org.koitharu.kotatsu.core.parser.site
import androidx.core.text.parseAsHtml
import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.domain.BaseMangaRepository
import org.koitharu.kotatsu.core.parser.BaseMangaRepository
import org.koitharu.kotatsu.domain.MangaLoaderContext
import org.koitharu.kotatsu.domain.exceptions.ParseException
import org.koitharu.kotatsu.utils.ext.*
import kotlin.text.removeSurrounding
class ReadmangaRepository(loaderContext: MangaLoaderContext) : BaseMangaRepository(loaderContext) {
abstract class GroupleRepository(
private val source: MangaSource,
loaderContext: MangaLoaderContext
) :
BaseMangaRepository(loaderContext) {
protected abstract val domain: String
override val sortOrders = setOf(
SortOrder.ALPHABETICAL, SortOrder.POPULARITY,
SortOrder.UPDATED, SortOrder.NEWEST, SortOrder.RATING
)
override suspend fun getList(
offset: Int,
query: String?,
sortOrder: SortOrder?,
tags: Set<String>?
tag: MangaTag?
): List<Manga> {
val doc = loaderContext.get("https://readmanga.me/list?sortType=updated&offset=$offset")
val url = if (tag == null) {
"https://$domain/list?sortType=${getSortKey(sortOrder)}&offset=$offset"
} else {
"https://$domain/list/genre/${tag.key}?sortType=${getSortKey(sortOrder)}&offset=$offset"
}
val doc = loaderContext.get(url)
.parseHtml()
val root = doc.body().getElementById("mangaBox")
?.selectFirst("div.tiles.row") ?: throw ParseException("Cannot find root")
return root.select("div.tile").mapNotNull { node ->
val imgDiv = node.selectFirst("div.img") ?: return@mapNotNull null
val descDiv = node.selectFirst("div.desc") ?: return@mapNotNull null
val href = imgDiv.selectFirst("a").attr("href")?.withDomain("readmanga.me")
val href = imgDiv.selectFirst("a").attr("href")?.withDomain(domain)
?: return@mapNotNull null
val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text()
?: return@mapNotNull null
@@ -47,7 +63,7 @@ class ReadmangaRepository(loaderContext: MangaLoaderContext) : BaseMangaReposito
MangaTag(
title = it.text(),
key = it.attr("href").substringAfterLast('/'),
source = MangaSource.READMANGA_RU
source = source
)
}?.toSet()
}.orEmpty(),
@@ -56,7 +72,7 @@ class ReadmangaRepository(loaderContext: MangaLoaderContext) : BaseMangaReposito
?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED
else -> null
},
source = MangaSource.READMANGA_RU
source = source
)
}
}
@@ -65,27 +81,27 @@ class ReadmangaRepository(loaderContext: MangaLoaderContext) : BaseMangaReposito
val doc = loaderContext.get(manga.url).parseHtml()
val root = doc.body().getElementById("mangaBox")
return manga.copy(
description = root.selectFirst("div.manga-description").firstChild()?.html()?.parseAsHtml(),
description = root.selectFirst("div.manga-description").firstChild()?.html(),
largeCoverUrl = root.selectFirst("div.subject-cower")?.selectFirst("img")?.attr(
"data-full"
),
chapters = root.selectFirst("div.chapters-link")?.selectFirst("table")
?.select("a")?.asReversed()?.mapIndexedNotNull { i, a ->
val href =
a.attr("href")?.withDomain("readmanga.me") ?: return@mapIndexedNotNull null
a.attr("href")?.withDomain(domain) ?: return@mapIndexedNotNull null
MangaChapter(
id = href.longHashCode(),
name = a.ownText(),
number = i + 1,
url = href,
source = MangaSource.READMANGA_RU
source = source
)
}
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = loaderContext.get(chapter.url).parseHtml()
val doc = loaderContext.get(chapter.url + "?mtr=1").parseHtml()
val scripts = doc.select("script")
for (script in scripts) {
val data = script.html()
@@ -103,10 +119,32 @@ class ReadmangaRepository(loaderContext: MangaLoaderContext) : BaseMangaReposito
MangaPage(
id = url.longHashCode(),
url = url,
source = MangaSource.READMANGA_RU
source = source
)
}
}
throw ParseException("Pages list not found at ${chapter.url}")
}
override suspend fun getTags(): Set<MangaTag> {
val doc = loaderContext.get("https://$domain/list/genres/sort_name").parseHtml()
val root = doc.body().getElementById("mangaBox").selectFirst("div.leftContent")
.selectFirst("table.table")
return root.select("a.element-link").map { a ->
MangaTag(
title = a.text(),
key = a.attr("href").substringAfterLast('/'),
source = source
)
}.toSet()
}
private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder) {
SortOrder.ALPHABETICAL -> "name"
SortOrder.POPULARITY -> "rate"
SortOrder.UPDATED -> "updated"
SortOrder.NEWEST -> "created"
SortOrder.RATING -> "votes"
null -> "updated"
}
}

10
app/src/main/java/org/koitharu/kotatsu/core/parser/site/MintMangaRepository.kt Normal file
View File

@@ -0,0 +1,10 @@
package org.koitharu.kotatsu.core.parser.site
import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.domain.MangaLoaderContext
class MintMangaRepository(loaderContext: MangaLoaderContext) :
GroupleRepository(MangaSource.MINTMANGA, loaderContext) {
override val domain: String = "mintmanga.live"
}

15
app/src/main/java/org/koitharu/kotatsu/core/parser/site/ReadmangaRepository.kt Normal file
View File

@@ -0,0 +1,15 @@
package org.koitharu.kotatsu.core.parser.site
import androidx.core.text.parseAsHtml
import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.core.parser.BaseMangaRepository
import org.koitharu.kotatsu.domain.MangaLoaderContext
import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.utils.ext.*
class ReadmangaRepository(loaderContext: MangaLoaderContext) :
GroupleRepository(MangaSource.READMANGA_RU, loaderContext) {
override val domain = "readmanga.me"
}

12
app/src/main/java/org/koitharu/kotatsu/domain/HistoryRepository.kt
View File

@@ -7,10 +7,12 @@ import org.koitharu.kotatsu.core.db.entity.HistoryEntity
import org.koitharu.kotatsu.core.db.entity.HistoryWithManga
import org.koitharu.kotatsu.core.db.entity.MangaEntity
import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.core.parser.MangaRepository
import java.io.Closeable
import java.util.*
class HistoryRepository() : KoinComponent, MangaRepository, Closeable {
class HistoryRepository() : KoinComponent,
MangaRepository, Closeable {
private val db: MangaDatabase by inject()
@@ -22,14 +24,14 @@ class HistoryRepository() : KoinComponent, MangaRepository, Closeable {
offset: Int,
query: String?,
sortOrder: SortOrder?,
tags: Set<String>?
): List<Manga> = getHistory(offset, query, sortOrder, tags).map { x -> x.manga }
tag: MangaTag?
): List<Manga> = getHistory(offset, query, sortOrder, tag).map { x -> x.manga }
suspend fun getHistory(
offset: Int,
query: String? = null,
sortOrder: SortOrder? = null,
tags: Set<String>? = null
tag: MangaTag? = null
): List<MangaInfo<MangaHistory>> {
val entities = db.historyDao().getAll(offset, 20, "updated_by")
return entities.map { x -> MangaInfo(x.manga.toManga(), x.history.toMangaHistory()) }
@@ -45,6 +47,8 @@ class HistoryRepository() : KoinComponent, MangaRepository, Closeable {
override suspend fun getPageFullUrl(page: MangaPage) = page.url
override suspend fun getTags() = emptySet<MangaTag>()
suspend fun addOrUpdate(manga: Manga, chapterId: Long, page: Int) {
val dao = db.historyDao()
val entity = HistoryEntity(

1
app/src/main/java/org/koitharu/kotatsu/domain/MangaProviderFactory.kt
View File

@@ -3,6 +3,7 @@ package org.koitharu.kotatsu.domain
import org.koin.core.KoinComponent
import org.koin.core.get
import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.core.parser.MangaRepository
object MangaProviderFactory : KoinComponent {

3
app/src/main/java/org/koitharu/kotatsu/domain/exceptions/ParseException.kt
View File

@@ -1,3 +0,0 @@
package org.koitharu.kotatsu.domain.exceptions
class ParseException(message: String? = null, cause: Throwable? = null) : RuntimeException(message, cause)

3
app/src/main/java/org/koitharu/kotatsu/ui/details/MangaDetailsFragment.kt
View File

@@ -1,5 +1,6 @@
package org.koitharu.kotatsu.ui.details
import androidx.core.text.parseAsHtml
import androidx.core.view.isVisible
import coil.api.load
import kotlinx.android.synthetic.main.fragment_details.*
@@ -22,7 +23,7 @@ class MangaDetailsFragment : BaseFragment(R.layout.fragment_details), MangaDetai
imageView_cover.load(data.manga.largeCoverUrl ?: data.manga.coverUrl)
textView_title.text = data.manga.title
textView_subtitle.text = data.manga.localizedTitle
textView_description.text = data.manga.description
textView_description.text = data.manga.description?.parseAsHtml()
if (data.manga.rating == Manga.NO_RATING) {
ratingBar.isVisible = false
} else {

2
app/src/test/java/org/koitharu/kotatsu/parsers/MangaParserTest.kt
View File

@@ -7,4 +7,6 @@ interface MangaParserTest {
fun testMangaDetails()
fun testMangaPages()
fun testTags()
}

49
app/src/test/java/org/koitharu/kotatsu/parsers/RepositoryTestEnvironment.kt
View File

@@ -1,32 +1,41 @@
package org.koitharu.kotatsu.parsers
import kotlinx.coroutines.runBlocking
import okhttp3.OkHttpClient
import org.junit.BeforeClass
import org.koin.core.context.startKoin
import org.koin.dsl.module
import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.domain.MangaLoaderContext
import org.koitharu.kotatsu.domain.MangaRepository
import org.koitharu.kotatsu.core.parser.MangaRepository
import org.koitharu.kotatsu.parsers.repository.ReadmangaRuTest
abstract class RepositoryTestEnvironment {
lateinit var repository: MangaRepository
lateinit var repository: MangaRepository
@BeforeClass
fun initialize(cls: Class<out MangaRepository>) {
startKoin {
modules(listOf(
module {
factory {
OkHttpClient()
}
}, module {
single {
MangaLoaderContext()
}
}
))
}
val constructor = cls.getConstructor(MangaLoaderContext::class.java)
repository = constructor.newInstance(MangaLoaderContext())
}
@BeforeClass
fun initialize(source: MangaSource) {
startKoin {
modules(listOf(
module {
factory {
OkHttpClient()
}
}, module {
single {
MangaLoaderContext()
}
}
))
}
val constructor = source.cls.getConstructor(MangaLoaderContext::class.java)
repository = constructor.newInstance(MangaLoaderContext())
}
fun getMangaList() = runBlocking { repository.getList(2) }
fun getMangaItem() = runBlocking { repository.getDetails(repository.getList(4).last()) }
fun getTags() = runBlocking { repository.getTags() }
}

66
app/src/test/java/org/koitharu/kotatsu/parsers/repository/MintMangaTest.kt Normal file
View File

@@ -0,0 +1,66 @@
package org.koitharu.kotatsu.parsers.repository
import kotlinx.coroutines.runBlocking
import org.junit.Assert
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.parsers.MangaParserTest
import org.koitharu.kotatsu.parsers.RepositoryTestEnvironment
import org.koitharu.kotatsu.utils.TestUtil
import org.mockito.junit.MockitoJUnitRunner
@RunWith(MockitoJUnitRunner::class)
class MintMangaTest : MangaParserTest {
@Test
override fun testMangaList() {
val list = getMangaList()
Assert.assertTrue(list.size == 70)
val item = list[40]
Assert.assertTrue(item.title.isNotEmpty())
Assert.assertTrue(item.rating in 0f..1f)
TestUtil.assertValidUrl(item.url)
TestUtil.assertValidUrl(item.coverUrl)
Assert.assertEquals(item.source, MangaSource.MINTMANGA)
}
@Test
override fun testMangaDetails() {
val manga = getMangaItem()
Assert.assertNotNull(manga.largeCoverUrl)
TestUtil.assertValidUrl(manga.largeCoverUrl!!)
Assert.assertNotNull(manga.chapters)
val chapter = manga.chapters!!.last()
Assert.assertEquals(chapter.source, MangaSource.MINTMANGA)
TestUtil.assertValidUrl(chapter.url)
}
@Test
override fun testMangaPages() {
val chapter = getMangaItem().chapters!!.first()
val pages = runBlocking { repository.getPages(chapter) }
Assert.assertFalse(pages.isEmpty())
Assert.assertEquals(pages.first().source, MangaSource.MINTMANGA)
TestUtil.assertValidUrl(runBlocking { repository.getPageFullUrl(pages.first()) })
TestUtil.assertValidUrl(runBlocking { repository.getPageFullUrl(pages.last()) })
}
@Test
override fun testTags() {
val tags = getTags()
Assert.assertFalse(tags.isEmpty())
val tag = tags.first()
Assert.assertFalse(tag.title.isBlank())
Assert.assertEquals(tag.source, MangaSource.MINTMANGA)
TestUtil.assertValidUrl("https://mintmanga.live/list/genre/${tag.key}")
}
companion object : RepositoryTestEnvironment() {
@JvmStatic
@BeforeClass
fun setUp() = initialize(MangaSource.MINTMANGA)
}
}

23
app/src/test/java/org/koitharu/kotatsu/parsers/repository/ReadmangaRuTest.kt
View File

@@ -5,7 +5,7 @@ import org.junit.Assert
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
import org.koitharu.kotatsu.domain.repository.ReadmangaRepository
import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.parsers.MangaParserTest
import org.koitharu.kotatsu.parsers.RepositoryTestEnvironment
import org.koitharu.kotatsu.utils.TestUtil
@@ -16,38 +16,51 @@ class ReadmangaRuTest : MangaParserTest {
@Test
override fun testMangaList() {
val list = runBlocking { repository.getList(1) }
val list = getMangaList()
Assert.assertTrue(list.size == 70)
val item = list[40]
Assert.assertTrue(item.title.isNotEmpty())
Assert.assertTrue(item.rating in 0f..1f)
TestUtil.assertValidUrl(item.url)
TestUtil.assertValidUrl(item.coverUrl)
Assert.assertEquals(item.source, MangaSource.READMANGA_RU)
}
@Test
override fun testMangaDetails() {
val manga = runBlocking { repository.getDetails(repository.getList(1).last()) }
val manga = getMangaItem()
Assert.assertNotNull(manga.largeCoverUrl)
TestUtil.assertValidUrl(manga.largeCoverUrl!!)
Assert.assertNotNull(manga.chapters)
val chapter = manga.chapters!!.last()
Assert.assertEquals(chapter.source, MangaSource.READMANGA_RU)
TestUtil.assertValidUrl(chapter.url)
}
@Test
override fun testMangaPages() {
val chapter = runBlocking { repository.getDetails(repository.getList(1).last()).chapters!!.first() }
val chapter = getMangaItem().chapters!!.first()
val pages = runBlocking { repository.getPages(chapter) }
Assert.assertFalse(pages.isEmpty())
Assert.assertEquals(pages.first().source, MangaSource.READMANGA_RU)
TestUtil.assertValidUrl(runBlocking { repository.getPageFullUrl(pages.first()) })
TestUtil.assertValidUrl(runBlocking { repository.getPageFullUrl(pages.last()) })
}
@Test
override fun testTags() {
val tags = getTags()
Assert.assertFalse(tags.isEmpty())
val tag = tags.first()
Assert.assertFalse(tag.title.isBlank())
Assert.assertEquals(tag.source, MangaSource.READMANGA_RU)
TestUtil.assertValidUrl("https://readmanga.me/list/genre/${tag.key}")
}
companion object : RepositoryTestEnvironment() {
@JvmStatic
@BeforeClass
fun setUp() = initialize(ReadmangaRepository::class.java)
fun setUp() = initialize(MangaSource.READMANGA_RU)
}
}