DesuMe parser

.idea/dictionaries/admin.xml

@@ -1,6 +1,7 @@
 <component name="ProjectDictionaryState">
   <dictionary name="admin">
     <words>
+      <w>desu</w>
       <w>koin</w>
       <w>kotatsu</w>
       <w>manga</w>

app/build.gradle

@@ -92,4 +92,5 @@ dependencies {
 	debugImplementation 'com.squareup.leakcanary:leakcanary-android:2.2'
 
 	testImplementation 'junit:junit:4.13'
+	testImplementation 'org.json:json:20180813'
 }

app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt

@@ -18,6 +18,7 @@ enum class MangaSource(
 	MINTMANGA("MintManga", "ru", MintMangaRepository::class.java),
 	SELFMANGA("SelfManga", "ru", SelfMangaRepository::class.java),
 	MANGACHAN("Манга-тян", "ru", MangaChanRepository::class.java),
+	DESUME("Desu.me", "ru", DesuMeRepository::class.java),
 	HENCHAN("Хентай-тян", "ru", HenChanRepository::class.java),
 	YAOICHAN("Яой-тян", "ru", YaoiChanRepository::class.java)
 }

app/src/main/java/org/koitharu/kotatsu/core/parser/site/DesuMeRepository.kt

@@ -0,0 +1,137 @@
+package org.koitharu.kotatsu.core.parser.site
+
+import org.koitharu.kotatsu.R
+import org.koitharu.kotatsu.core.exceptions.ParseException
+import org.koitharu.kotatsu.core.model.*
+import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
+import org.koitharu.kotatsu.utils.ext.*
+
+class DesuMeRepository : RemoteMangaRepository() {
+
+	override val source = MangaSource.DESUME
+
+	override val sortOrders = setOf(
+		SortOrder.UPDATED,
+		SortOrder.POPULARITY,
+		SortOrder.NEWEST,
+		SortOrder.ALPHABETICAL
+	)
+
+	override suspend fun getList(
+		offset: Int,
+		query: String?,
+		sortOrder: SortOrder?,
+		tag: MangaTag?
+	): List<Manga> {
+		val domain = conf.getDomain(DOMAIN)
+		val url = buildString {
+			append("https://")
+			append(domain)
+			append("/manga/api/?limit=20&order=")
+			append(getSortKey(sortOrder))
+			append("&page=")
+			append((offset / 20) + 1)
+			if (tag != null) {
+				append("&genres=")
+				append(tag.key)
+			}
+			if (query != null) {
+				append("&search=")
+				append(query)
+			}
+		}
+		val json = loaderContext.httpGet(url).parseJson().getJSONArray("response")
+			?: throw ParseException("Invalid response")
+		val total = json.length()
+		val list = ArrayList<Manga>(total)
+		for (i in 0 until total) {
+			val jo = json.getJSONObject(i)
+			val cover = jo.getJSONObject("image")
+			list += Manga(
+				url = jo.getString("url"),
+				source = MangaSource.DESUME,
+				title = jo.getString("russian"),
+				altTitle = jo.getString("name"),
+				coverUrl = cover.getString("preview"),
+				largeCoverUrl = cover.getString("original"),
+				state = when {
+					jo.getInt("ongoing") == 1 -> MangaState.ONGOING
+					else -> null
+				},
+				rating = jo.getDouble("score").toFloat().coerceIn(0f, 1f),
+				id = ID_MASK + jo.getLong("id"),
+				description = jo.getString("description")
+			)
+		}
+		return list
+	}
+
+	override suspend fun getDetails(manga: Manga): Manga {
+		val domain = conf.getDomain(DOMAIN)
+		val url = "https://$domain/manga/api/${manga.id - ID_MASK}"
+		val json = loaderContext.httpGet(url).parseJson().getJSONObject("response")
+			?: throw ParseException("Invalid response")
+		return manga.copy(
+			tags = json.getJSONArray("genres").map {
+				MangaTag(
+					key = it.getString("text"),
+					title = it.getString("russian"),
+					source = manga.source
+				)
+			}.toSet(),
+			description = json.getString("description"),
+			chapters = json.getJSONObject("chapters").getJSONArray("list").mapIndexed { i, it ->
+				val chid = it.getLong("id")
+				MangaChapter(
+					id = ID_MASK + chid,
+					source = manga.source,
+					url = "$url/chapter/$chid",
+					name = it.optString("title", "${manga.title} #${it.getDouble("ch")}"),
+					number = i + 1
+				)
+			}
+		)
+	}
+
+	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+		val json = loaderContext.httpGet(chapter.url).parseJson().getJSONObject("response")
+			?: throw ParseException("Invalid response")
+		return json.getJSONObject("pages").getJSONArray("list").map {
+			MangaPage(
+				id = it.getLong("id"),
+				source = chapter.source,
+				url = it.getString("img")
+			)
+		}
+	}
+
+	override suspend fun getTags(): Set<MangaTag> {
+		val domain = conf.getDomain(DOMAIN)
+		val doc = loaderContext.httpGet("https://$domain/manga/").parseHtml()
+		val root = doc.body().getElementById("animeFilter").selectFirst(".catalog-genres")
+		return root.select("li").map {
+			MangaTag(
+				source = source,
+				key = it.selectFirst("input").attr("data-genre"),
+				title = it.selectFirst("label").text()
+			)
+		}.toSet()
+	}
+
+	override fun onCreatePreferences() = setOf(R.string.key_parser_domain)
+
+	private fun getSortKey(sortOrder: SortOrder?) =
+		when (sortOrder) {
+			SortOrder.ALPHABETICAL -> "name"
+			SortOrder.POPULARITY -> "popular"
+			SortOrder.UPDATED -> "updated"
+			SortOrder.NEWEST -> "id"
+			else -> "updated"
+		}
+
+	private companion object {
+
+		private const val ID_MASK = 1000
+		private const val DOMAIN = "desu.me"
+	}
+}

app/src/main/java/org/koitharu/kotatsu/utils/ext/JsonExt.kt

@@ -11,4 +11,14 @@ fun <T> JSONArray.map(block: (JSONObject) -> T): List<T> {
 		result.add(block(jo))
 	}
 	return result
+}
+
+fun <T> JSONArray.mapIndexed(block: (Int, JSONObject) -> T): List<T> {
+	val len = length()
+	val result = ArrayList<T>(len)
+	for(i in 0 until len) {
+		val jo = getJSONObject(i)
+		result.add(block(i, jo))
+	}
+	return result
 }

app/src/test/java/org/koitharu/kotatsu/parsers/RemoteRepositoryTest.kt

@@ -26,8 +26,8 @@ class RemoteRepositoryTest(source: MangaSource) {
 		val list = runBlocking { repo.getList(60) }
 		Assert.assertFalse(list.isEmpty())
 		val item = list.random()
-		AssertX.assertContentType(item.coverUrl, "image")
+		AssertX.assertContentType(item.coverUrl, "image/*")
-		AssertX.assertContentType(item.url, "text", "html")
+		AssertX.assertContentType(item.url, "text/html", "application/json")
 		Assert.assertFalse(item.title.isBlank())
 	}
 
@@ -36,8 +36,8 @@ class RemoteRepositoryTest(source: MangaSource) {
 		val list = runBlocking { repo.getList(0, query = "tail") }
 		Assert.assertFalse(list.isEmpty())
 		val item = list.random()
-		AssertX.assertContentType(item.coverUrl, "image")
+		AssertX.assertContentType(item.coverUrl, "image/*")
-		AssertX.assertContentType(item.url, "text", "html")
+		AssertX.assertContentType(item.url, "text/html", "application/json")
 		Assert.assertFalse(item.title.isBlank())
 	}
 
@@ -51,8 +51,8 @@ class RemoteRepositoryTest(source: MangaSource) {
 		val list = runBlocking { repo.getList(0, tag = tag) }
 		Assert.assertFalse(list.isEmpty())
 		val item = list.random()
-		AssertX.assertContentType(item.coverUrl, "image")
+		AssertX.assertContentType(item.coverUrl, "image/*")
-		AssertX.assertContentType(item.url, "text", "html")
+		AssertX.assertContentType(item.url, "text/html", "application/json")
 		Assert.assertFalse(item.title.isBlank())
 	}
 
@@ -64,7 +64,7 @@ class RemoteRepositoryTest(source: MangaSource) {
 		Assert.assertFalse(details.description.isNullOrEmpty())
 		val chapter = details.chapters!!.random()
 		Assert.assertFalse(chapter.name.isBlank())
-		AssertX.assertContentType(chapter.url, "text", "html")
+		AssertX.assertContentType(chapter.url, "text/html", "application/json")
 	}
 
 	@Test
@@ -75,7 +75,7 @@ class RemoteRepositoryTest(source: MangaSource) {
 		Assert.assertFalse(pages.isEmpty())
 		val page = pages.random()
 		val fullUrl = runBlocking { repo.getPageFullUrl(page) }
-		AssertX.assertContentType(fullUrl, "image")
+		AssertX.assertContentType(fullUrl, "image/*")
 	}
 
 	companion object {

app/src/test/java/org/koitharu/kotatsu/utils/AssertX.kt

@@ -6,22 +6,22 @@ import java.net.URL
 
 object AssertX {
 
-	fun assertContentType(url: String, type: String, subtype: String? = null) {
+	fun assertContentType(url: String, vararg types: String) {
 		Assert.assertFalse("URL is empty", url.isEmpty())
 		val cn = URL(url).openConnection() as HttpURLConnection
 		cn.requestMethod = "HEAD"
 		cn.connect()
 		when (val code = cn.responseCode) {
 			HttpURLConnection.HTTP_MOVED_PERM,
-			HttpURLConnection.HTTP_MOVED_TEMP -> assertContentType(cn.getHeaderField("Location"), type, subtype)
+			HttpURLConnection.HTTP_MOVED_TEMP -> assertContentType(cn.getHeaderField("Location"), *types)
 			HttpURLConnection.HTTP_OK -> {
 				val ct = cn.contentType.substringBeforeLast(';').split("/")
-				Assert.assertEquals(type, ct.first())
+				Assert.assertTrue(types.any {
-				if (subtype != null) {
+					val x = it.split('/')
-					Assert.assertEquals(subtype, ct.last())
+					x[0] == ct[0] && (x[1] == "*" || x[1] == ct[1])
-				}
+				})
 			}
-			else -> Assert.fail("Invalid response code $code")
+			else -> Assert.fail("Invalid response code $code at $url")
 		}
 	}