NineManga sources #14

.idea/codeStyles/Project.xml

@@ -23,6 +23,7 @@
       </option>
     </AndroidXmlCodeStyleSettings>
     <JetCodeStyleSettings>
+      <option name="ALLOW_TRAILING_COMMA" value="true" />
       <option name="CODE_STYLE_DEFAULTS" value="KOTLIN_OFFICIAL" />
     </JetCodeStyleSettings>
     <codeStyleSettings language="CMake">

app/src/main/java/org/koitharu/kotatsu/base/domain/MangaLoaderContext.kt

@@ -12,10 +12,13 @@ open class MangaLoaderContext(
 	private val cookieJar: CookieJar
 ) : KoinComponent {
 
-	suspend fun httpGet(url: String): Response {
+	suspend fun httpGet(url: String, headers: Headers? = null): Response {
 		val request = Request.Builder()
 			.get()
 			.url(url)
+		if (headers != null) {
+			request.headers(headers)
+		}
 		return okHttp.newCall(request.build()).await()
 	}
 

app/src/main/java/org/koitharu/kotatsu/browser/cloudflare/CloudFlareClient.kt

@@ -3,11 +3,11 @@ package org.koitharu.kotatsu.browser.cloudflare
 import android.graphics.Bitmap
 import android.webkit.WebView
 import okhttp3.HttpUrl.Companion.toHttpUrl
-import org.koitharu.kotatsu.core.network.CookieJar
+import org.koitharu.kotatsu.core.network.AndroidCookieJar
 import org.koitharu.kotatsu.core.network.WebViewClientCompat
 
 class CloudFlareClient(
-	private val cookieJar: CookieJar,
+	private val cookieJar: AndroidCookieJar,
 	private val callback: CloudFlareCallback,
 	private val targetUrl: String
 ) : WebViewClientCompat() {

app/src/main/java/org/koitharu/kotatsu/core/model/MangaSource.kt

@@ -31,7 +31,15 @@ enum class MangaSource(
 	MANGAREAD("MangaRead", "en", MangareadRepository::class.java),
 	REMANGA("Remanga", "ru", RemangaRepository::class.java),
 	HENTAILIB("HentaiLib", "ru", HentaiLibRepository::class.java),
-	ANIBEL("Anibel", "be", AnibelRepository::class.java);
+	ANIBEL("Anibel", "be", AnibelRepository::class.java),
+	NINEMANGA_EN("NineManga English", "en", NineMangaRepository.English::class.java),
+	NINEMANGA_ES("NineManga Español", "es", NineMangaRepository.Spanish::class.java),
+	NINEMANGA_RU("NineManga Русский", "ru", NineMangaRepository.Russian::class.java),
+	NINEMANGA_DE("NineManga Deutsch", "de", NineMangaRepository.Deutsch::class.java),
+	NINEMANGA_IT("NineManga Italiano", "it", NineMangaRepository.Italiano::class.java),
+	NINEMANGA_BR("NineManga Brasil", "br", NineMangaRepository.Brazil::class.java),
+	NINEMANGA_FR("NineManga Français", "fr", NineMangaRepository.Francais::class.java),
+	;
 
 	@get:Throws(NoBeanDefFoundException::class)
 	@Deprecated("")

app/src/main/java/org/koitharu/kotatsu/core/network/AndroidCookieJar.kt

@@ -7,7 +7,7 @@ import okhttp3.HttpUrl
 import kotlin.coroutines.resume
 import kotlin.coroutines.suspendCoroutine
 
-class CookieJar : CookieJar {
+class AndroidCookieJar : CookieJar {
 
 	private val cookieManager = CookieManager.getInstance()
 
@@ -28,10 +28,6 @@ class CookieJar : CookieJar {
 		}
 	}
 
-	fun clearAsync() {
-		cookieManager.removeAllCookies(null)
-	}
-
 	suspend fun clear() = suspendCoroutine<Boolean> { continuation ->
 		cookieManager.removeAllCookies(continuation::resume)
 	}

app/src/main/java/org/koitharu/kotatsu/core/network/CurlLoggingInterceptor.kt

@@ -0,0 +1,59 @@
+package org.koitharu.kotatsu.core.network
+
+import android.util.Log
+import okhttp3.Interceptor
+import okhttp3.Request
+import okhttp3.Response
+import okio.Buffer
+import java.io.IOException
+import java.nio.charset.StandardCharsets
+
+class CurlLoggingInterceptor(
+	private val extraCurlOptions: String? = null,
+) : Interceptor {
+
+	@Throws(IOException::class)
+	override fun intercept(chain: Interceptor.Chain): Response {
+		val request: Request = chain.request()
+		var compressed = false
+		val curlCmd = StringBuilder("curl")
+		if (extraCurlOptions != null) {
+			curlCmd.append(" ").append(extraCurlOptions)
+		}
+		curlCmd.append(" -X ").append(request.method)
+		val headers = request.headers
+		var i = 0
+		val count = headers.size
+		while (i < count) {
+			val name = headers.name(i)
+			val value = headers.value(i)
+			if ("Accept-Encoding".equals(name, ignoreCase = true) && "gzip".equals(value,
+					ignoreCase = true)
+			) {
+				compressed = true
+			}
+			curlCmd.append(" -H " + "\"").append(name).append(": ").append(value).append("\"")
+			i++
+		}
+		val requestBody = request.body
+		if (requestBody != null) {
+			val buffer = Buffer()
+			requestBody.writeTo(buffer)
+			val contentType = requestBody.contentType()
+			val charset = contentType?.charset(StandardCharsets.UTF_8) ?: StandardCharsets.UTF_8
+			curlCmd.append(" --data $'")
+				.append(buffer.readString(charset).replace("\n", "\\n"))
+				.append("'")
+		}
+		curlCmd.append(if (compressed) " --compressed " else " ").append(request.url)
+		Log.d(TAG, "╭--- cURL (" + request.url + ")")
+		Log.d(TAG, curlCmd.toString())
+		Log.d(TAG, "╰--- (copy and paste the above line to a terminal)")
+		return chain.proceed(request)
+	}
+
+	private companion object {
+
+		const val TAG = "CURL"
+	}
+}

app/src/main/java/org/koitharu/kotatsu/core/network/NetworkModule.kt

@@ -6,12 +6,13 @@ import org.koin.android.ext.koin.androidContext
 import org.koin.core.qualifier.named
 import org.koin.dsl.bind
 import org.koin.dsl.module
+import org.koitharu.kotatsu.BuildConfig
 import org.koitharu.kotatsu.utils.CacheUtils
 import java.util.concurrent.TimeUnit
 
 val networkModule
 	get() = module {
-		single { CookieJar() } bind CookieJar::class
+		single { AndroidCookieJar() } bind CookieJar::class
 		single(named(CacheUtils.QUALIFIER_HTTP)) { CacheUtils.createHttpCache(androidContext()) }
 		single {
 			OkHttpClient.Builder().apply {
@@ -22,6 +23,9 @@ val networkModule
 				cache(get(named(CacheUtils.QUALIFIER_HTTP)))
 				addInterceptor(UserAgentInterceptor())
 				addInterceptor(CloudFlareInterceptor())
+				if (BuildConfig.DEBUG) {
+					addNetworkInterceptor(CurlLoggingInterceptor())
+				}
 			}.build()
 		}
 	}

app/src/main/java/org/koitharu/kotatsu/core/parser/ParserModule.kt

@@ -25,4 +25,11 @@ val parserModule
 		factory<MangaRepository>(named(MangaSource.REMANGA)) { RemangaRepository(get()) }
 		factory<MangaRepository>(named(MangaSource.HENTAILIB)) { HentaiLibRepository(get()) }
 		factory<MangaRepository>(named(MangaSource.ANIBEL)) { AnibelRepository(get()) }
+		factory<MangaRepository>(named(MangaSource.NINEMANGA_EN)) { NineMangaRepository.English(get()) }
+		factory<MangaRepository>(named(MangaSource.NINEMANGA_BR)) { NineMangaRepository.Brazil(get()) }
+		factory<MangaRepository>(named(MangaSource.NINEMANGA_DE)) { NineMangaRepository.Deutsch(get()) }
+		factory<MangaRepository>(named(MangaSource.NINEMANGA_ES)) { NineMangaRepository.Spanish(get()) }
+		factory<MangaRepository>(named(MangaSource.NINEMANGA_RU)) { NineMangaRepository.Russian(get()) }
+		factory<MangaRepository>(named(MangaSource.NINEMANGA_IT)) { NineMangaRepository.Italiano(get()) }
+		factory<MangaRepository>(named(MangaSource.NINEMANGA_FR)) { NineMangaRepository.Francais(get()) }
 	}

app/src/main/java/org/koitharu/kotatsu/core/parser/site/NineMangaRepository.kt

@@ -0,0 +1,201 @@
+package org.koitharu.kotatsu.core.parser.site
+
+import okhttp3.Headers
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import org.koitharu.kotatsu.base.domain.MangaLoaderContext
+import org.koitharu.kotatsu.core.exceptions.ParseException
+import org.koitharu.kotatsu.core.model.*
+import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
+import org.koitharu.kotatsu.utils.ext.*
+import java.util.*
+
+abstract class NineMangaRepository(
+	loaderContext: MangaLoaderContext,
+	override val source: MangaSource,
+	override val defaultDomain: String,
+) : RemoteMangaRepository(loaderContext) {
+
+	init {
+		loaderContext.insertCookies(getDomain(), "ninemanga_template_desk=yes")
+	}
+
+	override val sortOrders: Set<SortOrder> = EnumSet.of(
+		SortOrder.POPULARITY,
+	)
+
+	override suspend fun getList(
+		offset: Int,
+		query: String?,
+		sortOrder: SortOrder?,
+		tag: MangaTag?,
+	): List<Manga> {
+		val page = (offset / PAGE_SIZE.toFloat()).toIntUp() + 1
+		val url = buildString {
+			append("https://")
+			append(getDomain())
+			if (query.isNullOrEmpty()) {
+				append("/category/")
+				if (tag != null) {
+					append(tag.key)
+				} else {
+					append("index")
+				}
+				append("_")
+				append(page)
+				append(".html")
+			} else {
+				append("/search/?name_sel=&wd=")
+				append(query.urlEncoded())
+				append("&page=")
+				append(page)
+				append(".html")
+			}
+		}
+		val doc = loaderContext.httpGet(url, PREDEFINED_HEADERS).parseHtml()
+		val root = doc.body().selectFirst("ul.direlist")
+			?: throw ParseException("Cannot find root")
+		val baseHost = root.baseUri().toHttpUrl().host
+		return root.select("li").map { node ->
+			val href = node.selectFirst("a").absUrl("href")
+			val relUrl = href.toRelativeUrl(baseHost)
+			val dd = node.selectFirst("dd")
+			Manga(
+				id = generateUid(relUrl),
+				url = relUrl,
+				publicUrl = href,
+				title = dd.selectFirst("a.bookname").text().toCamelCase(),
+				altTitle = null,
+				coverUrl = node.selectFirst("img").absUrl("src"),
+				rating = Manga.NO_RATING,
+				author = null,
+				tags = emptySet(),
+				state = null,
+				source = source,
+				description = dd.selectFirst("p").html(),
+			)
+		}
+	}
+
+	override suspend fun getDetails(manga: Manga): Manga {
+		val doc = loaderContext.httpGet(
+			manga.url.withDomain() + "?waring=1",
+			PREDEFINED_HEADERS
+		).parseHtml()
+		val root = doc.body().selectFirst("div.manga")
+			?: throw ParseException("Cannot find root")
+		val infoRoot = root.selectFirst("div.bookintro")
+			?: throw ParseException("Cannot find info")
+		return manga.copy(
+			tags = infoRoot.getElementsByAttributeValue("itemprop", "genre")?.first()
+				?.select("a")?.mapToSet { a ->
+					MangaTag(
+						title = a.text(),
+						key = a.attr("href").substringBetween("/", "."),
+						source = source,
+					)
+				}.orEmpty(),
+			author = infoRoot.getElementsByAttributeValue("itemprop", "author")?.first()?.text(),
+			description = infoRoot.getElementsByAttributeValue("itemprop", "description")?.first()
+				?.html()?.substringAfter("</b>"),
+			chapters = root.selectFirst("div.chapterbox")?.selectFirst("ul")
+				?.select("li")?.asReversed()?.mapIndexed { i, li ->
+					val a = li.selectFirst("a")
+					val href = a.relUrl("href")
+					MangaChapter(
+						id = generateUid(href),
+						name = a.text(),
+						number = i + 1,
+						url = href,
+						branch = null,
+						source = source,
+					)
+				}
+		)
+	}
+
+	override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
+		val doc = loaderContext.httpGet(chapter.url.withDomain(), PREDEFINED_HEADERS).parseHtml()
+		return doc.body().getElementById("page")?.select("option")?.map { option ->
+			val url = option.attr("value")
+			MangaPage(
+				id = generateUid(url),
+				url = url,
+				referer = chapter.url.withDomain(),
+				preview = null,
+				source = source,
+			)
+		} ?: throw ParseException("Pages list not found at ${chapter.url}")
+	}
+
+	override suspend fun getPageUrl(page: MangaPage): String {
+		val doc = loaderContext.httpGet(page.url.withDomain(), PREDEFINED_HEADERS).parseHtml()
+		val root = doc.body()
+		return root.selectFirst("a.pic_download")?.absUrl("href")
+			?: throw ParseException("Page image not found")
+	}
+
+	override suspend fun getTags(): Set<MangaTag> {
+		val doc = loaderContext.httpGet("https://${getDomain()}/category/", PREDEFINED_HEADERS)
+			.parseHtml()
+		val root = doc.body().selectFirst("ul.genreidex")
+		return root.select("li").mapToSet { li ->
+			val a = li.selectFirst("a")
+			MangaTag(
+				title = a.text(),
+				key = a.attr("href").substringBetweenLast("/", "."),
+				source = source
+			)
+		}
+	}
+
+	class English(loaderContext: MangaLoaderContext) : NineMangaRepository(
+		loaderContext,
+		MangaSource.NINEMANGA_EN,
+		"www.ninemanga.com",
+	)
+
+	class Spanish(loaderContext: MangaLoaderContext) : NineMangaRepository(
+		loaderContext,
+		MangaSource.NINEMANGA_ES,
+		"es.ninemanga.com",
+	)
+
+	class Russian(loaderContext: MangaLoaderContext) : NineMangaRepository(
+		loaderContext,
+		MangaSource.NINEMANGA_RU,
+		"ru.ninemanga.com",
+	)
+
+	class Deutsch(loaderContext: MangaLoaderContext) : NineMangaRepository(
+		loaderContext,
+		MangaSource.NINEMANGA_DE,
+		"de.ninemanga.com",
+	)
+
+	class Brazil(loaderContext: MangaLoaderContext) : NineMangaRepository(
+		loaderContext,
+		MangaSource.NINEMANGA_BR,
+		"br.ninemanga.com",
+	)
+
+	class Italiano(loaderContext: MangaLoaderContext) : NineMangaRepository(
+		loaderContext,
+		MangaSource.NINEMANGA_IT,
+		"it.ninemanga.com",
+	)
+
+	class Francais(loaderContext: MangaLoaderContext) : NineMangaRepository(
+		loaderContext,
+		MangaSource.NINEMANGA_FR,
+		"fr.ninemanga.com",
+	)
+
+	private companion object {
+
+		const val PAGE_SIZE = 26
+
+		val PREDEFINED_HEADERS = Headers.Builder()
+			.add("Accept-Language", "en-US;q=0.7,en;q=0.3")
+			.build()
+	}
+}

app/src/main/java/org/koitharu/kotatsu/settings/HistorySettingsFragment.kt

@@ -11,7 +11,7 @@ import org.koin.android.ext.android.get
 import org.koin.android.ext.android.inject
 import org.koitharu.kotatsu.R
 import org.koitharu.kotatsu.base.ui.BasePreferenceFragment
-import org.koitharu.kotatsu.core.network.CookieJar
+import org.koitharu.kotatsu.core.network.AndroidCookieJar
 import org.koitharu.kotatsu.core.prefs.AppSettings
 import org.koitharu.kotatsu.local.data.Cache
 import org.koitharu.kotatsu.search.ui.MangaSuggestionsProvider
@@ -75,7 +75,7 @@ class HistorySettingsFragment : BasePreferenceFragment(R.string.history_and_cach
 			}
 			AppSettings.KEY_COOKIES_CLEAR -> {
 				viewLifecycleScope.launch {
-					val cookieJar = get<CookieJar>()
+					val cookieJar = get<AndroidCookieJar>()
 					cookieJar.clear()
 					Snackbar.make(
 						listView ?: return@launch,

app/src/main/java/org/koitharu/kotatsu/utils/ext/StringExt.kt

@@ -29,6 +29,25 @@ fun String.removeSurrounding(vararg chars: Char): String {
 	return this
 }
 
+fun String.toCamelCase(): String {
+	if (isEmpty()) {
+		return this
+	}
+	val result = StringBuilder(length)
+	var capitalize = true
+	for (char in this) {
+		result.append(
+			if (capitalize) {
+				char.uppercase()
+			} else {
+				char.lowercase()
+			}
+		)
+		capitalize = char.isWhitespace()
+	}
+	return result.toString()
+}
+
 fun String.transliterate(skipMissing: Boolean): String {
 	val cyr = charArrayOf(
 		'a', 'б', 'в', 'г', 'д', 'ё', 'ж', 'з', 'и', 'к', 'л', 'м', 'н',
@@ -92,7 +111,7 @@ fun String.md5(): String {
 		.padStart(32, '0')
 }
 
-fun String.substringBetween(from: String, to: String, fallbackValue: String): String {
+fun String.substringBetween(from: String, to: String, fallbackValue: String = this): String {
 	val fromIndex = indexOf(from)
 	if (fromIndex == -1) {
 		return fallbackValue
@@ -105,6 +124,19 @@ fun String.substringBetween(from: String, to: String, fallbackValue: String): St
 	}
 }
 
+fun String.substringBetweenLast(from: String, to: String, fallbackValue: String = this): String {
+	val fromIndex = lastIndexOf(from)
+	if (fromIndex == -1) {
+		return fallbackValue
+	}
+	val toIndex = lastIndexOf(to)
+	return if (toIndex == -1) {
+		fallbackValue
+	} else {
+		substring(fromIndex + from.length, toIndex)
+	}
+}
+
 fun String.find(regex: Regex) = regex.find(this)?.value
 
 fun String.levenshteinDistance(other: String): Int {

app/src/test/java/org/koitharu/kotatsu/parsers/TemporaryCookieJar.kt

@@ -10,7 +10,7 @@ class TemporaryCookieJar : CookieJar {
 
 	override fun loadForRequest(url: HttpUrl): List<Cookie> {
 		val time = System.currentTimeMillis()
-		return cache.values.filter { it.matches(url) && it.expiresAt < time }
+		return cache.values.filter { it.matches(url) && it.expiresAt >= time }
 	}
 
 	override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) {