prifak/Kotatsu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MintManga parser

Browse Source
This commit is contained in:
Koitharu
2020-02-02 15:16:24 +02:00
parent e86f4b625a
commit 6d4a77b023
17 changed files with 226 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app/src/main/java/org/koitharu/kotatsu/core/exceptions/ParseException.kt Normal file
View File

@@ -0,0 +1,3 @@
package org.koitharu.kotatsu.core.exceptions
class ParseException(message: String? = null, cause: Throwable? = null) : RuntimeException(message, cause)

2
app/src/main/java/org/koitharu/kotatsu/core/model/Manga.kt
View File

@@ -13,7 +13,7 @@ data class Manga(
val coverUrl: String,
val largeCoverUrl: String? = null,
val summary: String,
val description: CharSequence? = null,
val description: String? = null, //HTML
val tags: Set<MangaTag> = emptySet(),
val state: MangaState? = null,
val chapters: List<MangaChapter>? = null,

8
app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt
View File

@@ -2,10 +2,12 @@ package org.koitharu.kotatsu.core.model
import android.os.Parcelable
import kotlinx.android.parcel.Parcelize
import org.koitharu.kotatsu.domain.MangaRepository
import org.koitharu.kotatsu.domain.repository.ReadmangaRepository
import org.koitharu.kotatsu.core.parser.MangaRepository
import org.koitharu.kotatsu.core.parser.site.MintMangaRepository
import org.koitharu.kotatsu.core.parser.site.ReadmangaRepository
@Parcelize
enum class MangaSource(val title: String, val cls: Class<out MangaRepository>): Parcelable {
READMANGA_RU("ReadManga", ReadmangaRepository::class.java)
READMANGA_RU("ReadManga", ReadmangaRepository::class.java),
MINTMANGA("MintManga", MintMangaRepository::class.java)
}

2
app/src/main/java/org/koitharu/kotatsu/core/model/SortOrder.kt
View File

@@ -1,5 +1,5 @@
package org.koitharu.kotatsu.core.model
enum class SortOrder {
ALPHABETICAL, POPULARITY, UPDATED, NEWEST
ALPHABETICAL, POPULARITY, UPDATED, NEWEST, RATING
}

9
app/src/main/java/org/koitharu/kotatsu/domain/BaseMangaRepository.kt → app/src/main/java/org/koitharu/kotatsu/core/parser/BaseMangaRepository.kt
View File

@@ -1,13 +1,18 @@
package org.koitharu.kotatsu.domain
package org.koitharu.kotatsu.core.parser
import org.koitharu.kotatsu.core.model.MangaPage
import org.koitharu.kotatsu.core.model.MangaTag
import org.koitharu.kotatsu.core.model.SortOrder
import org.koitharu.kotatsu.domain.MangaLoaderContext
abstract class BaseMangaRepository(protected val loaderContext: MangaLoaderContext) : MangaRepository {
abstract class BaseMangaRepository(protected val loaderContext: MangaLoaderContext) :
MangaRepository {
override val sortOrders: Set<SortOrder> get() = emptySet()
override val isSearchAvailable get() = true
override suspend fun getPageFullUrl(page: MangaPage) : String = page.url
override suspend fun getTags(): Set<MangaTag> = emptySet()
}

11
app/src/main/java/org/koitharu/kotatsu/domain/MangaRepository.kt → app/src/main/java/org/koitharu/kotatsu/core/parser/MangaRepository.kt
View File

@@ -1,9 +1,6 @@
package org.koitharu.kotatsu.domain
package org.koitharu.kotatsu.core.parser
import org.koitharu.kotatsu.core.model.Manga
import org.koitharu.kotatsu.core.model.MangaChapter
import org.koitharu.kotatsu.core.model.MangaPage
import org.koitharu.kotatsu.core.model.SortOrder
import org.koitharu.kotatsu.core.model.*
interface MangaRepository {
@@ -11,11 +8,13 @@ interface MangaRepository {
val isSearchAvailable: Boolean
suspend fun getList(offset: Int, query: String? = null, sortOrder: SortOrder? = null, tags: Set<String>? = null): List<Manga>
suspend fun getList(offset: Int, query: String? = null, sortOrder: SortOrder? = null, tag: MangaTag? = null): List<Manga>
suspend fun getDetails(manga: Manga) : Manga
suspend fun getPages(chapter: MangaChapter) : List<MangaPage>
suspend fun getPageFullUrl(page: MangaPage) : String
suspend fun getTags(): Set<MangaTag>
}

68
app/src/main/java/org/koitharu/kotatsu/domain/repository/ReadmangaRepository.kt → app/src/main/java/org/koitharu/kotatsu/core/parser/site/GroupleRepository.kt
View File

@@ -1,28 +1,44 @@
package org.koitharu.kotatsu.domain.repository
package org.koitharu.kotatsu.core.parser.site
import androidx.core.text.parseAsHtml
import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.domain.BaseMangaRepository
import org.koitharu.kotatsu.core.parser.BaseMangaRepository
import org.koitharu.kotatsu.domain.MangaLoaderContext
import org.koitharu.kotatsu.domain.exceptions.ParseException
import org.koitharu.kotatsu.utils.ext.*
import kotlin.text.removeSurrounding
class ReadmangaRepository(loaderContext: MangaLoaderContext) : BaseMangaRepository(loaderContext) {
abstract class GroupleRepository(
private val source: MangaSource,
loaderContext: MangaLoaderContext
) :
BaseMangaRepository(loaderContext) {
protected abstract val domain: String
override val sortOrders = setOf(
SortOrder.ALPHABETICAL, SortOrder.POPULARITY,
SortOrder.UPDATED, SortOrder.NEWEST, SortOrder.RATING
)
override suspend fun getList(
offset: Int,
query: String?,
sortOrder: SortOrder?,
tags: Set<String>?
tag: MangaTag?
): List<Manga> {
val doc = loaderContext.get("https://readmanga.me/list?sortType=updated&offset=$offset")
val url = if (tag == null) {
"https://$domain/list?sortType=${getSortKey(sortOrder)}&offset=$offset"
} else {
"https://$domain/list/genre/${tag.key}?sortType=${getSortKey(sortOrder)}&offset=$offset"
}
val doc = loaderContext.get(url)
.parseHtml()
val root = doc.body().getElementById("mangaBox")
?.selectFirst("div.tiles.row") ?: throw ParseException("Cannot find root")
return root.select("div.tile").mapNotNull { node ->
val imgDiv = node.selectFirst("div.img") ?: return@mapNotNull null
val descDiv = node.selectFirst("div.desc") ?: return@mapNotNull null
val href = imgDiv.selectFirst("a").attr("href")?.withDomain("readmanga.me")
val href = imgDiv.selectFirst("a").attr("href")?.withDomain(domain)
?: return@mapNotNull null
val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text()
?: return@mapNotNull null
@@ -47,7 +63,7 @@ class ReadmangaRepository(loaderContext: MangaLoaderContext) : BaseMangaReposito
MangaTag(
title = it.text(),
key = it.attr("href").substringAfterLast('/'),
source = MangaSource.READMANGA_RU
source = source
)
}?.toSet()
}.orEmpty(),
@@ -56,7 +72,7 @@ class ReadmangaRepository(loaderContext: MangaLoaderContext) : BaseMangaReposito
?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED
else -> null
},
source = MangaSource.READMANGA_RU
source = source
)
}
}
@@ -65,27 +81,27 @@ class ReadmangaRepository(loaderContext: MangaLoaderContext) : BaseMangaReposito
val doc = loaderContext.get(manga.url).parseHtml()
val root = doc.body().getElementById("mangaBox")
return manga.copy(
description = root.selectFirst("div.manga-description").firstChild()?.html()?.parseAsHtml(),
description = root.selectFirst("div.manga-description").firstChild()?.html(),
largeCoverUrl = root.selectFirst("div.subject-cower")?.selectFirst("img")?.attr(
"data-full"
),
chapters = root.selectFirst("div.chapters-link")?.selectFirst("table")
?.select("a")?.asReversed()?.mapIndexedNotNull { i, a ->
val href =
a.attr("href")?.withDomain("readmanga.me") ?: return@mapIndexedNotNull null
a.attr("href")?.withDomain(domain) ?: return@mapIndexedNotNull null
MangaChapter(
id = href.longHashCode(),
name = a.ownText(),
number = i + 1,
url = href,
source = MangaSource.READMANGA_RU
source = source
)
}
)
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
val doc = loaderContext.get(chapter.url).parseHtml()
val doc = loaderContext.get(chapter.url + "?mtr=1").parseHtml()
val scripts = doc.select("script")
for (script in scripts) {
val data = script.html()
@@ -103,10 +119,32 @@ class ReadmangaRepository(loaderContext: MangaLoaderContext) : BaseMangaReposito
MangaPage(
id = url.longHashCode(),
url = url,
source = MangaSource.READMANGA_RU
source = source
)
}
}
throw ParseException("Pages list not found at ${chapter.url}")
}
override suspend fun getTags(): Set<MangaTag> {
val doc = loaderContext.get("https://$domain/list/genres/sort_name").parseHtml()
val root = doc.body().getElementById("mangaBox").selectFirst("div.leftContent")
.selectFirst("table.table")
return root.select("a.element-link").map { a ->
MangaTag(
title = a.text(),
key = a.attr("href").substringAfterLast('/'),
source = source
)
}.toSet()
}
private fun getSortKey(sortOrder: SortOrder?) = when (sortOrder) {
SortOrder.ALPHABETICAL -> "name"
SortOrder.POPULARITY -> "rate"
SortOrder.UPDATED -> "updated"
SortOrder.NEWEST -> "created"
SortOrder.RATING -> "votes"
null -> "updated"
}
}

10
app/src/main/java/org/koitharu/kotatsu/core/parser/site/MintMangaRepository.kt Normal file
View File

@@ -0,0 +1,10 @@
package org.koitharu.kotatsu.core.parser.site
import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.domain.MangaLoaderContext
class MintMangaRepository(loaderContext: MangaLoaderContext) :
GroupleRepository(MangaSource.MINTMANGA, loaderContext) {
override val domain: String = "mintmanga.live"
}

15
app/src/main/java/org/koitharu/kotatsu/core/parser/site/ReadmangaRepository.kt Normal file
View File

@@ -0,0 +1,15 @@
package org.koitharu.kotatsu.core.parser.site
import androidx.core.text.parseAsHtml
import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.core.parser.BaseMangaRepository
import org.koitharu.kotatsu.domain.MangaLoaderContext
import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.utils.ext.*
class ReadmangaRepository(loaderContext: MangaLoaderContext) :
GroupleRepository(MangaSource.READMANGA_RU, loaderContext) {
override val domain = "readmanga.me"
}

12
app/src/main/java/org/koitharu/kotatsu/domain/HistoryRepository.kt
View File

@@ -7,10 +7,12 @@ import org.koitharu.kotatsu.core.db.entity.HistoryEntity
import org.koitharu.kotatsu.core.db.entity.HistoryWithManga
import org.koitharu.kotatsu.core.db.entity.MangaEntity
import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.core.parser.MangaRepository
import java.io.Closeable
import java.util.*
class HistoryRepository() : KoinComponent, MangaRepository, Closeable {
class HistoryRepository() : KoinComponent,
MangaRepository, Closeable {
private val db: MangaDatabase by inject()
@@ -22,14 +24,14 @@ class HistoryRepository() : KoinComponent, MangaRepository, Closeable {
offset: Int,
query: String?,
sortOrder: SortOrder?,
tags: Set<String>?
): List<Manga> = getHistory(offset, query, sortOrder, tags).map { x -> x.manga }
tag: MangaTag?
): List<Manga> = getHistory(offset, query, sortOrder, tag).map { x -> x.manga }
suspend fun getHistory(
offset: Int,
query: String? = null,
sortOrder: SortOrder? = null,
tags: Set<String>? = null
tag: MangaTag? = null
): List<MangaInfo<MangaHistory>> {
val entities = db.historyDao().getAll(offset, 20, "updated_by")
return entities.map { x -> MangaInfo(x.manga.toManga(), x.history.toMangaHistory()) }
@@ -45,6 +47,8 @@ class HistoryRepository() : KoinComponent, MangaRepository, Closeable {
override suspend fun getPageFullUrl(page: MangaPage) = page.url
override suspend fun getTags() = emptySet<MangaTag>()
suspend fun addOrUpdate(manga: Manga, chapterId: Long, page: Int) {
val dao = db.historyDao()
val entity = HistoryEntity(

1
app/src/main/java/org/koitharu/kotatsu/domain/MangaProviderFactory.kt
View File

@@ -3,6 +3,7 @@ package org.koitharu.kotatsu.domain
import org.koin.core.KoinComponent
import org.koin.core.get
import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.core.parser.MangaRepository
object MangaProviderFactory : KoinComponent {

3
app/src/main/java/org/koitharu/kotatsu/domain/exceptions/ParseException.kt
View File

@@ -1,3 +0,0 @@
package org.koitharu.kotatsu.domain.exceptions
class ParseException(message: String? = null, cause: Throwable? = null) : RuntimeException(message, cause)

3
app/src/main/java/org/koitharu/kotatsu/ui/details/MangaDetailsFragment.kt
View File

@@ -1,5 +1,6 @@
package org.koitharu.kotatsu.ui.details
import androidx.core.text.parseAsHtml
import androidx.core.view.isVisible
import coil.api.load
import kotlinx.android.synthetic.main.fragment_details.*
@@ -22,7 +23,7 @@ class MangaDetailsFragment : BaseFragment(R.layout.fragment_details), MangaDetai
imageView_cover.load(data.manga.largeCoverUrl ?: data.manga.coverUrl)
textView_title.text = data.manga.title
textView_subtitle.text = data.manga.localizedTitle
textView_description.text = data.manga.description
textView_description.text = data.manga.description?.parseAsHtml()
if (data.manga.rating == Manga.NO_RATING) {
ratingBar.isVisible = false
} else {