Multiple tags support in (almost) all sources #19

This commit is contained in:
Koitharu
2021-09-11 13:50:47 +03:00
parent 4977464e69
commit c1b6cef362
27 changed files with 187 additions and 117 deletions

View File

@@ -6,5 +6,5 @@ import kotlinx.parcelize.Parcelize
@Parcelize
data class MangaFilter(
val sortOrder: SortOrder?,
val tag: MangaTag?,
val tags: Set<MangaTag>,
) : Parcelable

View File

@@ -9,24 +9,12 @@ interface MangaRepository {
val sortOrders: Set<SortOrder>
suspend fun getList(
suspend fun getList2(
offset: Int,
query: String? = null,
tags: Set<MangaTag>? = null,
sortOrder: SortOrder? = null,
): List<Manga> = if (tags == null || tags.size <= 1) {
getList(offset, query, sortOrder, tags?.singleOrNull())
} else {
throw NotImplementedError("Multiple filter are not supported by this source yet")
}
@Deprecated("Use multiple tag variant")
suspend fun getList(
offset: Int,
query: String? = null,
sortOrder: SortOrder? = null,
tag: MangaTag? = null,
): List<Manga> = throw NotImplementedError("This is fine")
): List<Manga>
suspend fun getDetails(manga: Manga): Manga

View File

@@ -16,7 +16,7 @@ class AnibelRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepositor
SortOrder.NEWEST
)
override suspend fun getList(
override suspend fun getList2(
offset: Int,
query: String?,
tags: Set<MangaTag>?,

View File

@@ -17,11 +17,11 @@ abstract class ChanRepository(loaderContext: MangaLoaderContext) : RemoteMangaRe
SortOrder.ALPHABETICAL
)
override suspend fun getList(
override suspend fun getList2(
offset: Int,
query: String?,
sortOrder: SortOrder?,
tag: MangaTag?
tags: Set<MangaTag>?,
sortOrder: SortOrder?
): List<Manga> {
val domain = getDomain()
val url = when {
@@ -31,7 +31,11 @@ abstract class ChanRepository(loaderContext: MangaLoaderContext) : RemoteMangaRe
}
"https://$domain/?do=search&subaction=search&story=${query.urlEncoded()}"
}
tag != null -> "https://$domain/tags/${tag.key}&n=${getSortKey2(sortOrder)}?offset=$offset"
!tags.isNullOrEmpty() -> tags.joinToString(
prefix = "https://$domain/tags/",
postfix = "&n=${getSortKey2(sortOrder)}?offset=$offset",
separator = "+",
) { tag -> tag.key }
else -> "https://$domain/${getSortKey(sortOrder)}?offset=$offset"
}
val doc = loaderContext.httpGet(url).parseHtml()

View File

@@ -20,11 +20,11 @@ class DesuMeRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepositor
SortOrder.ALPHABETICAL
)
override suspend fun getList(
override suspend fun getList2(
offset: Int,
query: String?,
sortOrder: SortOrder?,
tag: MangaTag?
tags: Set<MangaTag>?,
sortOrder: SortOrder?
): List<Manga> {
if (query != null && offset != 0) {
return emptyList()
@@ -37,9 +37,9 @@ class DesuMeRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepositor
append(getSortKey(sortOrder))
append("&page=")
append((offset / 20) + 1)
if (tag != null) {
if (!tags.isNullOrEmpty()) {
append("&genres=")
append(tag.key)
appendAll(tags, ",") { it.key }
}
if (query != null) {
append("&search=")

View File

@@ -32,14 +32,7 @@ class ExHentaiRepository(
loaderContext.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2")
}
override suspend fun getList(
offset: Int,
query: String?,
sortOrder: SortOrder?,
tag: MangaTag?,
): List<Manga> = getList(offset, query, setOfNotNull(tag), sortOrder)
override suspend fun getList(
override suspend fun getList2(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
@@ -80,7 +73,7 @@ class ExHentaiRepository(
parseFailed("Cannot find root")
} else {
updateDm = true
return getList(offset, query, tags, sortOrder)
return getList2(offset, query, tags, sortOrder)
}
updateDm = false
return root.children().mapNotNull { tr ->

View File

@@ -1,6 +1,7 @@
package org.koitharu.kotatsu.core.parser.site
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Response
import org.koitharu.kotatsu.base.domain.MangaLoaderContext
import org.koitharu.kotatsu.core.exceptions.ParseException
import org.koitharu.kotatsu.core.model.*
@@ -18,11 +19,11 @@ abstract class GroupleRepository(loaderContext: MangaLoaderContext) :
SortOrder.RATING
)
override suspend fun getList(
override suspend fun getList2(
offset: Int,
query: String?,
sortOrder: SortOrder?,
tag: MangaTag?
tags: Set<MangaTag>?,
sortOrder: SortOrder?
): List<Manga> {
val domain = getDomain()
val doc = when {
@@ -33,22 +34,24 @@ abstract class GroupleRepository(loaderContext: MangaLoaderContext) :
"offset" to (offset upBy PAGE_SIZE_SEARCH).toString()
)
)
tag == null -> loaderContext.httpGet(
tags.isNullOrEmpty() -> loaderContext.httpGet(
"https://$domain/list?sortType=${
getSortKey(
sortOrder
)
}&offset=${offset upBy PAGE_SIZE}"
)
else -> loaderContext.httpGet(
"https://$domain/list/genre/${tag.key}?sortType=${
tags.size == 1 -> loaderContext.httpGet(
"https://$domain/list/genre/${tags.first().key}?sortType=${
getSortKey(
sortOrder
)
}&offset=${offset upBy PAGE_SIZE}"
)
}.parseHtml()
val root = doc.body().getElementById("mangaBox")
offset > 0 -> return emptyList()
else -> advancedSearch(domain, tags)
}.parseHtml().body()
val root = (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults"))
?.selectFirst("div.tiles.row") ?: throw ParseException("Cannot find root")
val baseHost = root.baseUri().toHttpUrl().host
return root.select("div.tile").mapNotNull { node ->
@@ -182,6 +185,43 @@ abstract class GroupleRepository(loaderContext: MangaLoaderContext) :
null -> "updated"
}
private suspend fun advancedSearch(domain: String, tags: Set<MangaTag>): Response {
val url = "https://$domain/search/advanced"
// Step 1: map catalog genres names to advanced-search genres ids
val tagsIndex = loaderContext.httpGet(url).parseHtml()
.body().selectFirst("form.search-form")
?.select("div.form-group")
?.get(1) ?: parseFailed("Genres filter element not found")
val tagNames = tags.map { it.title.lowercase() }
val payload = HashMap<String, String>()
var foundGenres = 0
tagsIndex.select("li.property").forEach { li ->
val name = li.text().trim().lowercase()
val id = li.selectFirst("input")?.id()
?: parseFailed("Id for tag $name not found")
payload[id] = if (name in tagNames) {
foundGenres++
"in"
} else ""
}
if (foundGenres != tags.size) {
parseFailed("Some genres are not found")
}
// Step 2: advanced search
payload["q"] = ""
payload["s_high_rate"] = ""
payload["s_single"] = ""
payload["s_mature"] = ""
payload["s_completed"] = ""
payload["s_translated"] = ""
payload["s_many_chapters"] = ""
payload["s_wait_upload"] = ""
payload["s_sale"] = ""
payload["years"] = "1900,2099"
payload["+"] = "Искать".urlEncoded()
return loaderContext.httpPost(url, payload)
}
private companion object {
private const val PAGE_SIZE = 70

View File

@@ -11,13 +11,13 @@ class HenChanRepository(loaderContext: MangaLoaderContext) : ChanRepository(load
override val defaultDomain = "hentaichan.live"
override val source = MangaSource.HENCHAN
override suspend fun getList(
override suspend fun getList2(
offset: Int,
query: String?,
sortOrder: SortOrder?,
tag: MangaTag?
tags: Set<MangaTag>?,
sortOrder: SortOrder?
): List<Manga> {
return super.getList(offset, query, sortOrder, tag).map {
return super.getList2(offset, query, tags, sortOrder).map {
val cover = it.coverUrl
if (cover.contains("_blur")) {
it.copy(coverUrl = cover.replace("_blur", ""))

View File

@@ -26,11 +26,11 @@ open class MangaLibRepository(loaderContext: MangaLoaderContext) :
SortOrder.NEWEST
)
override suspend fun getList(
override suspend fun getList2(
offset: Int,
query: String?,
sortOrder: SortOrder?,
tag: MangaTag?
tags: Set<MangaTag>?,
sortOrder: SortOrder?
): List<Manga> {
if (!query.isNullOrEmpty()) {
return if (offset == 0) search(query) else emptyList()
@@ -43,8 +43,8 @@ open class MangaLibRepository(loaderContext: MangaLoaderContext) :
append(getSortKey(sortOrder))
append("&page=")
append(page)
if (tag != null) {
append("&includeGenres[]=")
tags?.forEach { tag ->
append("&genres[include][]=")
append(tag.key)
}
}

View File

@@ -23,11 +23,11 @@ class MangaTownRepository(loaderContext: MangaLoaderContext) :
SortOrder.UPDATED
)
override suspend fun getList(
override suspend fun getList2(
offset: Int,
query: String?,
sortOrder: SortOrder?,
tag: MangaTag?
tags: Set<MangaTag>?,
sortOrder: SortOrder?
): List<Manga> {
val sortKey = when (sortOrder) {
SortOrder.ALPHABETICAL -> "?name.az"
@@ -43,8 +43,13 @@ class MangaTownRepository(loaderContext: MangaLoaderContext) :
}
"/search?name=${query.urlEncoded()}".withDomain()
}
tag != null -> "/directory/${tag.key}/$page.htm$sortKey".withDomain()
else -> "/directory/$page.htm$sortKey".withDomain()
tags.isNullOrEmpty() -> "/directory/$page.htm$sortKey".withDomain()
tags.size == 1 -> "/directory/${tags.first().key}/$page.htm$sortKey".withDomain()
else -> tags.joinToString(
prefix = "/search?page=$page".withDomain()
) { tag ->
"&genres[${tag.key}]=1"
}
}
val doc = loaderContext.httpGet(url).parseHtml()
val root = doc.body().selectFirst("ul.manga_pic_list")

View File

@@ -20,12 +20,17 @@ class MangareadRepository(
SortOrder.POPULARITY
)
override suspend fun getList(
override suspend fun getList2(
offset: Int,
query: String?,
sortOrder: SortOrder?,
tag: MangaTag?
tags: Set<MangaTag>?,
sortOrder: SortOrder?
): List<Manga> {
val tag = when {
tags.isNullOrEmpty() -> null
tags.size == 1 -> tags.first()
else -> throw NotImplementedError("Multiple genres are not supported by this source")
}
val payload = createRequestTemplate()
payload["page"] = (offset / PAGE_SIZE.toFloat()).toIntUp().toString()
payload["vars[meta_key]"] = when (sortOrder) {

View File

@@ -23,7 +23,7 @@ abstract class NineMangaRepository(
SortOrder.POPULARITY,
)
override suspend fun getList(
override suspend fun getList2(
offset: Int,
query: String?,
tags: Set<MangaTag>?,
@@ -146,7 +146,7 @@ abstract class NineMangaRepository(
val cateId = li.attr("cate_id") ?: return@mapNotNullToSet null
val a = li.selectFirst("a") ?: return@mapNotNullToSet null
MangaTag(
title = a.text(),
title = a.text().toTitleCase(),
key = cateId,
source = source
)

View File

@@ -9,7 +9,6 @@ import org.koitharu.kotatsu.core.model.*
import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
import org.koitharu.kotatsu.utils.ext.*
import java.util.*
import kotlin.collections.ArrayList
class RemangaRepository(loaderContext: MangaLoaderContext) : RemoteMangaRepository(loaderContext) {
@@ -24,11 +23,11 @@ class RemangaRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposito
SortOrder.NEWEST
)
override suspend fun getList(
override suspend fun getList2(
offset: Int,
query: String?,
sortOrder: SortOrder?,
tag: MangaTag?
tags: Set<MangaTag>?,
sortOrder: SortOrder?
): List<Manga> {
val domain = getDomain()
val urlBuilder = StringBuilder()
@@ -40,8 +39,9 @@ class RemangaRepository(loaderContext: MangaLoaderContext) : RemoteMangaReposito
} else {
urlBuilder.append("/api/search/catalog/?ordering=")
.append(getSortKey(sortOrder))
if (tag != null) {
urlBuilder.append("&genres=" + tag.key)
tags?.forEach { tag ->
urlBuilder.append("&genres=")
urlBuilder.append(tag.key)
}
}
urlBuilder

View File

@@ -217,10 +217,7 @@ abstract class MangaListFragment : BaseFragment<FragmentListBinding>(),
activity?.invalidateOptionsMenu()
}
@CallSuper
override fun onFilterChanged(filter: MangaFilter) {
drawer?.closeDrawers()
}
override fun onFilterChanged(filter: MangaFilter) = Unit
override fun onWindowInsetsChanged(insets: Insets) {
val headerHeight = (activity as? AppBarOwner)?.appBar?.measureHeight() ?: insets.top

View File

@@ -6,19 +6,15 @@ import org.koitharu.kotatsu.base.ui.list.BaseViewHolder
import org.koitharu.kotatsu.core.model.MangaFilter
import org.koitharu.kotatsu.core.model.MangaTag
import org.koitharu.kotatsu.core.model.SortOrder
import java.util.*
class FilterAdapter(
sortOrders: List<SortOrder> = emptyList(),
tags: List<MangaTag> = emptyList(),
private val sortOrders: List<SortOrder> = emptyList(),
private val tags: List<MangaTag> = emptyList(),
state: MangaFilter?,
private val listener: OnFilterChangedListener
) : RecyclerView.Adapter<BaseViewHolder<*, Boolean, *>>() {
private val sortOrders = ArrayList<SortOrder>(sortOrders)
private val tags = ArrayList(Collections.singletonList(null) + tags)
private var currentState = state ?: MangaFilter(sortOrders.firstOrNull(), null)
private var currentState = state ?: MangaFilter(sortOrders.firstOrNull(), emptySet())
override fun onCreateViewHolder(parent: ViewGroup, viewType: Int) = when (viewType) {
VIEW_TYPE_SORT -> FilterSortHolder(parent).apply {
@@ -28,7 +24,7 @@ class FilterAdapter(
}
VIEW_TYPE_TAG -> FilterTagHolder(parent).apply {
itemView.setOnClickListener {
setCheckedTag(boundData)
setCheckedTag(boundData ?: return@setOnClickListener, !isChecked)
}
}
else -> throw IllegalArgumentException("Unknown viewType $viewType")
@@ -44,7 +40,7 @@ class FilterAdapter(
}
is FilterTagHolder -> {
val item = tags[position - sortOrders.size]
holder.bind(item, item == currentState.tag)
holder.bind(item, item in currentState.tags)
}
}
}
@@ -54,19 +50,25 @@ class FilterAdapter(
else -> VIEW_TYPE_TAG
}
fun setCheckedTag(tag: MangaTag?) {
if (tag != currentState.tag) {
val oldItemPos = tags.indexOf(currentState.tag)
val newItemPos = tags.indexOf(tag)
currentState = currentState.copy(tag = tag)
if (oldItemPos in tags.indices) {
notifyItemChanged(sortOrders.size + oldItemPos)
fun setCheckedTag(tag: MangaTag, isChecked: Boolean) {
currentState = if (tag in currentState.tags) {
if (!isChecked) {
currentState.copy(tags = currentState.tags - tag)
} else {
return
}
if (newItemPos in tags.indices) {
notifyItemChanged(sortOrders.size + newItemPos)
} else {
if (isChecked) {
currentState.copy(tags = currentState.tags + tag)
} else {
return
}
listener.onFilterChanged(currentState)
}
val index = tags.indexOf(tag)
if (index in tags.indices) {
notifyItemChanged(sortOrders.size + index)
}
listener.onFilterChanged(currentState)
}
fun setCheckedSort(sort: SortOrder) {

View File

@@ -12,7 +12,7 @@ class FilterSortHolder(parent: ViewGroup) :
) {
override fun onBind(data: SortOrder, extra: Boolean) {
binding.radio.setText(data.titleRes)
binding.radio.isChecked = extra
binding.root.setText(data.titleRes)
binding.root.isChecked = extra
}
}

View File

@@ -2,18 +2,20 @@ package org.koitharu.kotatsu.list.ui.filter
import android.view.LayoutInflater
import android.view.ViewGroup
import org.koitharu.kotatsu.R
import org.koitharu.kotatsu.base.ui.list.BaseViewHolder
import org.koitharu.kotatsu.core.model.MangaTag
import org.koitharu.kotatsu.databinding.ItemCheckableSingleBinding
import org.koitharu.kotatsu.databinding.ItemCheckableMultipleBinding
class FilterTagHolder(parent: ViewGroup) :
BaseViewHolder<MangaTag?, Boolean, ItemCheckableSingleBinding>(
ItemCheckableSingleBinding.inflate(LayoutInflater.from(parent.context), parent, false)
BaseViewHolder<MangaTag, Boolean, ItemCheckableMultipleBinding>(
ItemCheckableMultipleBinding.inflate(LayoutInflater.from(parent.context), parent, false)
) {
override fun onBind(data: MangaTag?, extra: Boolean) {
binding.radio.text = data?.title ?: context.getString(R.string.all)
binding.radio.isChecked = extra
val isChecked: Boolean
get() = binding.root.isChecked
override fun onBind(data: MangaTag, extra: Boolean) {
binding.root.text = data.title
binding.root.isChecked = extra
}
}

View File

@@ -25,11 +25,11 @@ class LocalMangaRepository(private val context: Context) : MangaRepository {
private val filenameFilter = CbzFilter()
override suspend fun getList(
override suspend fun getList2(
offset: Int,
query: String?,
sortOrder: SortOrder?,
tag: MangaTag?
tags: Set<MangaTag>?,
sortOrder: SortOrder?
): List<Manga> {
require(offset == 0) {
"LocalMangaRepository does not support pagination"

View File

@@ -61,7 +61,7 @@ class LocalListViewModel(
launchLoadingJob(Dispatchers.Default) {
try {
listError.value = null
mangaList.value = repository.getList(0, tags = null)
mangaList.value = repository.getList2(0)
} catch (e: Throwable) {
listError.value = e
}

View File

@@ -31,7 +31,6 @@ class RemoteListFragment : MangaListFragment() {
override fun onFilterChanged(filter: MangaFilter) {
viewModel.applyFilter(filter)
super.onFilterChanged(filter)
}
override fun onCreateOptionsMenu(menu: Menu, inflater: MenuInflater) {

View File

@@ -78,10 +78,10 @@ class RemoteListViewModel(
loadingJob = launchLoadingJob(Dispatchers.Default) {
try {
listError.value = null
val list = repository.getList(
val list = repository.getList2(
offset = if (append) mangaList.value?.size ?: 0 else 0,
sortOrder = appliedFilter?.sortOrder,
tag = appliedFilter?.tag
tags = appliedFilter?.tags,
)
if (!append) {
mangaList.value = list

View File

@@ -29,7 +29,11 @@ class MangaSearchRepository(
MangaProviderFactory.getSources(settings, includeHidden = false).asFlow()
.flatMapMerge(concurrency) { source ->
runCatching {
source.repository.getList(0, query, SortOrder.POPULARITY)
source.repository.getList2(
offset = 0,
query = query,
sortOrder = SortOrder.POPULARITY
)
}.getOrElse {
emptyList()
}.asFlow()

View File

@@ -71,10 +71,9 @@ class SearchViewModel(
loadingJob = launchLoadingJob(Dispatchers.Default) {
try {
listError.value = null
val list = repository.getList(
val list = repository.getList2(
offset = if (append) mangaList.value?.size ?: 0 else 0,
tags = null,
query = query
query = query,
)
if (!append) {
mangaList.value = list

View File

@@ -48,6 +48,10 @@ fun String.toCamelCase(): String {
return result.toString()
}
fun String.toTitleCase(): String {
return replaceFirstChar { x -> x.uppercase() }
}
fun String.transliterate(skipMissing: Boolean): String {
val cyr = charArrayOf(
'а', 'б', 'в', 'г', 'д', 'е', 'ж', 'з', 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п',
@@ -200,4 +204,20 @@ fun String.levenshteinDistance(other: String): Int {
}
return cost[lhsLength - 1]
}
inline fun <T> StringBuilder.appendAll(
items: Iterable<T>,
separator: CharSequence,
transform: (T) -> CharSequence = { it.toString() },
) {
var isFirst = true
for (item in items) {
if (isFirst) {
isFirst = false
} else {
append(separator)
}
append(transform(item))
}
}

View File

@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="utf-8"?>
<CheckedTextView
xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="?android:listPreferredItemHeightSmall"
android:background="?android:selectableItemBackground"
android:drawableStart="?android:listChoiceIndicatorMultiple"
android:drawablePadding="12dp"
android:gravity="center_vertical|start"
android:paddingStart="?android:listPreferredItemPaddingStart"
android:paddingEnd="?android:listPreferredItemPaddingEnd"
tools:text="@tools:sample/full_names" />

View File

@@ -2,7 +2,6 @@
<CheckedTextView
xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools"
android:id="@+id/radio"
android:layout_width="match_parent"
android:layout_height="?android:listPreferredItemHeightSmall"
android:background="?android:selectableItemBackground"

View File

@@ -38,15 +38,15 @@ class RemoteMangaRepositoryTest(private val source: MangaSource) : KoinTest {
@Test
fun list() = coroutineTestRule.runBlockingTest {
val list = repo.getList(20, query = null, sortOrder = SortOrder.POPULARITY, tag = null)
val list = repo.getList2(20, query = null, sortOrder = SortOrder.POPULARITY, tags = null)
checkMangaList(list)
}
@Test
fun search() = coroutineTestRule.runBlockingTest {
val subject = repo.getList(20, query = null, sortOrder = SortOrder.POPULARITY, tag = null)
val subject = repo.getList2(20, query = null, sortOrder = SortOrder.POPULARITY, tags = null)
.first()
val list = repo.getList(offset = 0, query = subject.title, sortOrder = null, tag = null)
val list = repo.getList2(offset = 0, query = subject.title, sortOrder = null, tags = null)
checkMangaList(list)
Truth.assertThat(list.map { it.url }).contains(subject.url)
}
@@ -63,13 +63,13 @@ class RemoteMangaRepositoryTest(private val source: MangaSource) : KoinTest {
Truth.assertThat(titles).doesNotContain("")
Truth.assertThat(tags.mapToSet { it.source }).containsExactly(source)
val list = repo.getList(offset = 0, tag = tags.last(), query = null, sortOrder = null)
val list = repo.getList2(offset = 0, tags = setOf(tags.last()), query = null, sortOrder = null)
checkMangaList(list)
}
@Test
fun details() = coroutineTestRule.runBlockingTest {
val list = repo.getList(20, query = null, sortOrder = SortOrder.POPULARITY, tag = null)
val list = repo.getList2(20, query = null, sortOrder = SortOrder.POPULARITY, tags = null)
val item = list.first()
val details = repo.getDetails(item)
@@ -87,7 +87,7 @@ class RemoteMangaRepositoryTest(private val source: MangaSource) : KoinTest {
@Test
fun pages() = coroutineTestRule.runBlockingTest {
val list = repo.getList(20, query = null, sortOrder = SortOrder.POPULARITY, tag = null)
val list = repo.getList2(20, query = null, sortOrder = SortOrder.POPULARITY, tags = null)
val chapter =
repo.getDetails(list.first()).chapters?.firstOrNull() ?: error("Chapter is null")
val pages = repo.getPages(chapter)