New headers processing approach

README.md

@@ -16,7 +16,7 @@ Download APK directly from GitHub:
 
 ### Main Features
 
-* Online manga catalogues
+* Online [manga catalogues](https://github.com/KotatsuApp/kotatsu-parsers)
 * Search manga by name and genres
 * Reading history and bookmarks
 * Favourites organized by user-defined categories
@@ -24,7 +24,7 @@ Download APK directly from GitHub:
 * Tablet-optimized Material You UI
 * Standard and Webtoon-optimized reader
 * Notifications about new chapters with updates feed
-* Shikimori integration (manga tracking)
+* Integration with manga tracking services: Shikimori, AniList, MAL (coming soon)
 * Password/fingerprint protect access to the app
 * History and favourites [synchronization](https://github.com/KotatsuApp/kotatsu-syncserver) across devices
 

app/build.gradle

@@ -86,7 +86,7 @@ afterEvaluate {
 	}
 }
 dependencies {
-	implementation('com.github.KotatsuApp:kotatsu-parsers:00abaea324') {
+	implementation('com.github.KotatsuApp:kotatsu-parsers:05d705ac03') {
 		exclude group: 'org.json', module: 'json'
 	}
 

app/src/debug/java/org/koitharu/kotatsu/core/parser/DummyParser.kt

@@ -1,15 +1,20 @@
 package org.koitharu.kotatsu.core.parser
 
-import java.util.*
 import org.koitharu.kotatsu.parsers.MangaLoaderContext
 import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.config.ConfigKey
-import org.koitharu.kotatsu.parsers.model.*
+import org.koitharu.kotatsu.parsers.model.Manga
+import org.koitharu.kotatsu.parsers.model.MangaChapter
+import org.koitharu.kotatsu.parsers.model.MangaPage
+import org.koitharu.kotatsu.parsers.model.MangaSource
+import org.koitharu.kotatsu.parsers.model.MangaTag
+import org.koitharu.kotatsu.parsers.model.SortOrder
+import java.util.EnumSet
 
 /**
  * This parser is just for parser development, it should not be used in releases
  */
-class DummyParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.DUMMY) {
+class DummyParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.DUMMY) {
 
 	override val configKeyDomain: ConfigKey.Domain
 		get() = ConfigKey.Domain("", null)
@@ -37,4 +42,4 @@ class DummyParser(override val context: MangaLoaderContext) : MangaParser(MangaS
 	override suspend fun getTags(): Set<MangaTag> {
 		TODO("Not yet implemented")
 	}
-}
+}

app/src/main/java/org/koitharu/kotatsu/bookmarks/ui/adapter/BookmarkListAD.kt

@@ -11,7 +11,6 @@ import org.koitharu.kotatsu.databinding.ItemBookmarkBinding
 import org.koitharu.kotatsu.utils.ext.disposeImageRequest
 import org.koitharu.kotatsu.utils.ext.enqueueWith
 import org.koitharu.kotatsu.utils.ext.newImageRequest
-import org.koitharu.kotatsu.utils.ext.referer
 
 fun bookmarkListAD(
 	coil: ImageLoader,
@@ -27,7 +26,6 @@ fun bookmarkListAD(
 
 	bind {
 		binding.imageViewThumb.newImageRequest(item.imageUrl, item.manga.source)?.run {
-			referer(item.manga.publicUrl)
 			placeholder(R.drawable.ic_placeholder)
 			fallback(R.drawable.ic_placeholder)
 			error(R.drawable.ic_error_placeholder)

app/src/main/java/org/koitharu/kotatsu/bookmarks/ui/adapter/BookmarksGroupAD.kt

@@ -18,7 +18,6 @@ import org.koitharu.kotatsu.utils.ext.clearItemDecorations
 import org.koitharu.kotatsu.utils.ext.disposeImageRequest
 import org.koitharu.kotatsu.utils.ext.enqueueWith
 import org.koitharu.kotatsu.utils.ext.newImageRequest
-import org.koitharu.kotatsu.utils.ext.referer
 
 fun bookmarksGroupAD(
 	coil: ImageLoader,
@@ -50,7 +49,6 @@ fun bookmarksGroupAD(
 			selectionController.attachToRecyclerView(item.manga, binding.recyclerView)
 		}
 		binding.imageViewCover.newImageRequest(item.manga.coverUrl, item.manga.source)?.run {
-			referer(item.manga.publicUrl)
 			placeholder(R.drawable.ic_placeholder)
 			fallback(R.drawable.ic_placeholder)
 			error(R.drawable.ic_error_placeholder)

app/src/main/java/org/koitharu/kotatsu/browser/BrowserActivity.kt

@@ -8,16 +8,14 @@ import android.net.Uri
 import android.os.Bundle
 import android.view.Menu
 import android.view.MenuItem
-import android.view.ViewGroup
 import androidx.core.graphics.Insets
 import androidx.core.view.isVisible
-import androidx.core.view.updateLayoutParams
 import androidx.core.view.updatePadding
-import com.google.android.material.R as materialR
 import org.koitharu.kotatsu.R
 import org.koitharu.kotatsu.base.ui.BaseActivity
-import org.koitharu.kotatsu.core.network.UserAgentInterceptor
+import org.koitharu.kotatsu.core.network.CommonHeadersInterceptor
 import org.koitharu.kotatsu.databinding.ActivityBrowserBinding
+import com.google.android.material.R as materialR
 
 @SuppressLint("SetJavaScriptEnabled")
 class BrowserActivity : BaseActivity<ActivityBrowserBinding>(), BrowserCallback {
@@ -31,7 +29,7 @@ class BrowserActivity : BaseActivity<ActivityBrowserBinding>(), BrowserCallback
 		}
 		with(binding.webView.settings) {
 			javaScriptEnabled = true
-			userAgentString = UserAgentInterceptor.userAgentChrome
+			userAgentString = CommonHeadersInterceptor.userAgentChrome
 		}
 		binding.webView.webViewClient = BrowserClient(this)
 		binding.webView.webChromeClient = ProgressChromeClient(binding.progressBar)
@@ -72,6 +70,7 @@ class BrowserActivity : BaseActivity<ActivityBrowserBinding>(), BrowserCallback
 			finishAfterTransition()
 			true
 		}
+
 		R.id.action_browser -> {
 			val intent = Intent(Intent.ACTION_VIEW)
 			intent.data = Uri.parse(binding.webView.url)
@@ -81,6 +80,7 @@ class BrowserActivity : BaseActivity<ActivityBrowserBinding>(), BrowserCallback
 			}
 			true
 		}
+
 		else -> super.onOptionsItemSelected(item)
 	}
 

app/src/main/java/org/koitharu/kotatsu/browser/cloudflare/CloudFlareDialog.kt

@@ -15,7 +15,7 @@ import dagger.hilt.android.AndroidEntryPoint
 import okhttp3.Headers
 import org.koitharu.kotatsu.base.ui.AlertDialogFragment
 import org.koitharu.kotatsu.core.network.CommonHeaders
-import org.koitharu.kotatsu.core.network.UserAgentInterceptor
+import org.koitharu.kotatsu.core.network.CommonHeadersInterceptor
 import org.koitharu.kotatsu.core.network.cookies.MutableCookieJar
 import org.koitharu.kotatsu.databinding.FragmentCloudflareBinding
 import org.koitharu.kotatsu.utils.ext.stringArgument
@@ -44,7 +44,7 @@ class CloudFlareDialog : AlertDialogFragment<FragmentCloudflareBinding>(), Cloud
 			cacheMode = WebSettings.LOAD_DEFAULT
 			domStorageEnabled = true
 			databaseEnabled = true
-			userAgentString = arguments?.getString(ARG_UA) ?: UserAgentInterceptor.userAgentChrome
+			userAgentString = arguments?.getString(ARG_UA) ?: CommonHeadersInterceptor.userAgentChrome
 		}
 		binding.webView.webViewClient = CloudFlareClient(cookieJar, this, url.orEmpty())
 		CookieManager.getInstance().setAcceptThirdPartyCookies(binding.webView, true)

app/src/main/java/org/koitharu/kotatsu/core/AppModule.kt

@@ -85,7 +85,7 @@ interface AppModule {
 		@Singleton
 		fun provideOkHttpClient(
 			localStorageManager: LocalStorageManager,
-			userAgentInterceptor: UserAgentInterceptor,
+			commonHeadersInterceptor: CommonHeadersInterceptor,
 			cookieJar: CookieJar,
 			settings: AppSettings,
 		): OkHttpClient {
@@ -98,7 +98,7 @@ interface AppModule {
 				dns(DoHManager(cache, settings))
 				cache(cache)
 				addInterceptor(GZipInterceptor())
-				addInterceptor(userAgentInterceptor)
+				addInterceptor(commonHeadersInterceptor)
 				addInterceptor(CloudFlareInterceptor())
 				if (BuildConfig.DEBUG) {
 					addInterceptor(CurlLoggingInterceptor())

app/src/main/java/org/koitharu/kotatsu/core/network/CommonHeadersInterceptor.kt

@@ -0,0 +1,77 @@
+package org.koitharu.kotatsu.core.network
+
+import android.os.Build
+import android.util.Log
+import dagger.Lazy
+import okhttp3.Interceptor
+import okhttp3.Request
+import okhttp3.Response
+import org.koitharu.kotatsu.BuildConfig
+import org.koitharu.kotatsu.core.parser.MangaRepository
+import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
+import org.koitharu.kotatsu.parsers.model.MangaSource
+import org.koitharu.kotatsu.parsers.util.mergeWith
+import java.util.Locale
+import javax.inject.Inject
+import javax.inject.Singleton
+
+@Singleton
+class CommonHeadersInterceptor @Inject constructor(
+	private val mangaRepositoryFactoryLazy: Lazy<MangaRepository.Factory>,
+) : Interceptor {
+
+	override fun intercept(chain: Interceptor.Chain): Response {
+		val request = chain.request()
+		val source = request.tag(MangaSource::class.java)
+		val repository = if (source != null) {
+			mangaRepositoryFactoryLazy.get().create(source) as? RemoteMangaRepository
+		} else {
+			if (BuildConfig.DEBUG) {
+				Log.w("Http", "Request without source tag: ${request.url}")
+			}
+			null
+		}
+		val headersBuilder = request.headers.newBuilder()
+		repository?.headers?.let {
+			headersBuilder.mergeWith(it, replaceExisting = false)
+		}
+		if (headersBuilder[CommonHeaders.USER_AGENT] == null) {
+			headersBuilder[CommonHeaders.USER_AGENT] = userAgentFallback
+		}
+		if (headersBuilder[CommonHeaders.REFERER] == null && repository != null) {
+			headersBuilder[CommonHeaders.REFERER] = "https://${repository.domain}/"
+		}
+		val newRequest = request.newBuilder().headers(headersBuilder.build()).build()
+		return repository?.intercept(ProxyChain(chain, newRequest)) ?: chain.proceed(newRequest)
+	}
+
+	private class ProxyChain(
+		private val delegate: Interceptor.Chain,
+		private val request: Request,
+	) : Interceptor.Chain by delegate {
+
+		override fun request(): Request = request
+	}
+
+	companion object {
+
+		val userAgentFallback
+			get() = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format(
+				BuildConfig.VERSION_NAME,
+				Build.VERSION.RELEASE,
+				Build.MODEL,
+				Build.BRAND,
+				Build.DEVICE,
+				Locale.getDefault().language,
+			)
+
+		val userAgentChrome
+			get() = (
+				"Mozilla/5.0 (Linux; Android %s; %s) AppleWebKit/537.36 (KHTML, like Gecko) " +
+					"Chrome/100.0.4896.127 Mobile Safari/537.36"
+				).format(
+					Build.VERSION.RELEASE,
+					Build.MODEL,
+				)
+	}
+}

app/src/main/java/org/koitharu/kotatsu/core/network/UserAgentInterceptor.kt

@@ -1,59 +0,0 @@
-package org.koitharu.kotatsu.core.network
-
-import android.os.Build
-import dagger.Lazy
-import okhttp3.Interceptor
-import okhttp3.Request
-import okhttp3.Response
-import org.koitharu.kotatsu.BuildConfig
-import org.koitharu.kotatsu.core.parser.MangaRepository
-import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
-import org.koitharu.kotatsu.parsers.model.MangaSource
-import java.util.Locale
-import javax.inject.Inject
-import javax.inject.Singleton
-
-@Singleton
-class UserAgentInterceptor @Inject constructor(
-	private val mangaRepositoryFactoryLazy: Lazy<MangaRepository.Factory>,
-) : Interceptor {
-
-	override fun intercept(chain: Interceptor.Chain): Response {
-		val request = chain.request()
-		return chain.proceed(
-			if (request.header(CommonHeaders.USER_AGENT) == null) {
-				request.newBuilder()
-					.addHeader(CommonHeaders.USER_AGENT, getUserAgent(request))
-					.build()
-			} else request,
-		)
-	}
-
-	private fun getUserAgent(request: Request): String {
-		val source = request.tag(MangaSource::class.java) ?: return userAgent
-		val repository = mangaRepositoryFactoryLazy.get().create(source) as? RemoteMangaRepository
-		return repository?.userAgent ?: userAgent
-	}
-
-	companion object {
-
-		val userAgent
-			get() = "Kotatsu/%s (Android %s; %s; %s %s; %s)".format(
-				BuildConfig.VERSION_NAME,
-				Build.VERSION.RELEASE,
-				Build.MODEL,
-				Build.BRAND,
-				Build.DEVICE,
-				Locale.getDefault().language,
-			) // TODO Decide what to do with this afterwards
-
-		val userAgentChrome
-			get() = (
-				"Mozilla/5.0 (Linux; Android %s; %s) AppleWebKit/537.36 (KHTML, like Gecko) " +
-					"Chrome/100.0.4896.127 Mobile Safari/537.36"
-				).format(
-					Build.VERSION.RELEASE,
-					Build.MODEL,
-				)
-	}
-}

app/src/main/java/org/koitharu/kotatsu/core/parser/RemoteMangaRepository.kt

@@ -6,9 +6,11 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.MainCoroutineDispatcher
 import kotlinx.coroutines.async
 import kotlinx.coroutines.currentCoroutineContext
+import okhttp3.Headers
+import okhttp3.Interceptor
+import okhttp3.Response
 import org.koitharu.kotatsu.core.cache.ContentCache
 import org.koitharu.kotatsu.core.cache.SafeDeferred
-import org.koitharu.kotatsu.core.network.CommonHeaders
 import org.koitharu.kotatsu.core.prefs.SourceSettings
 import org.koitharu.kotatsu.parsers.MangaParser
 import org.koitharu.kotatsu.parsers.MangaParserAuthProvider
@@ -20,13 +22,14 @@ import org.koitharu.kotatsu.parsers.model.MangaPage
 import org.koitharu.kotatsu.parsers.model.MangaSource
 import org.koitharu.kotatsu.parsers.model.MangaTag
 import org.koitharu.kotatsu.parsers.model.SortOrder
+import org.koitharu.kotatsu.parsers.util.domain
 import org.koitharu.kotatsu.utils.ext.processLifecycleScope
 import org.koitharu.kotatsu.utils.ext.runCatchingCancellable
 
 class RemoteMangaRepository(
 	private val parser: MangaParser,
 	private val cache: ContentCache,
-) : MangaRepository {
+) : MangaRepository, Interceptor {
 
 	override val source: MangaSource
 		get() = parser.source
@@ -40,8 +43,19 @@ class RemoteMangaRepository(
 			getConfig().defaultSortOrder = value
 		}
 
-	val userAgent: String?
+	val domain: String
-		get() = parser.headers?.get(CommonHeaders.USER_AGENT)
+		get() = parser.domain
+
+	val headers: Headers?
+		get() = parser.headers
+
+	override fun intercept(chain: Interceptor.Chain): Response {
+		return if (parser is Interceptor) {
+			parser.intercept(chain)
+		} else {
+			chain.proceed(chain.request())
+		}
+	}
 
 	override suspend fun getList(offset: Int, query: String): List<Manga> {
 		return parser.getList(offset, query)

app/src/main/java/org/koitharu/kotatsu/core/parser/favicon/FaviconFetcher.kt

@@ -20,7 +20,6 @@ import okhttp3.Response
 import okhttp3.ResponseBody
 import okhttp3.internal.closeQuietly
 import org.koitharu.kotatsu.core.model.MangaSource
-import org.koitharu.kotatsu.core.network.CommonHeaders
 import org.koitharu.kotatsu.core.parser.MangaRepository
 import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
 import org.koitharu.kotatsu.local.data.CacheDir
@@ -53,7 +52,7 @@ class FaviconFetcher(
 			options.size.height.pxOrElse { FALLBACK_SIZE },
 		)
 		val icon = checkNotNull(favicons.find(sizePx)) { "No favicons found" }
-		val response = loadIcon(icon.url, repo.userAgent, favicons.referer)
+		val response = loadIcon(icon.url, mangaSource)
 		val responseBody = response.requireBody()
 		val source = writeToDiskCache(responseBody)?.toImageSource() ?: responseBody.toImageSource()
 		return SourceResult(
@@ -63,14 +62,11 @@ class FaviconFetcher(
 		)
 	}
 
-	private suspend fun loadIcon(url: String, userAgent: String?, referer: String): Response {
+	private suspend fun loadIcon(url: String, source: MangaSource): Response {
 		val request = Request.Builder()
 			.url(url)
 			.get()
-			.header(CommonHeaders.REFERER, referer)
+			.tag(MangaSource::class.java, source)
-		if (userAgent != null) {
-			request.header(CommonHeaders.USER_AGENT, userAgent)
-		}
 		@Suppress("UNCHECKED_CAST")
 		options.tags.asMap().forEach { request.tag(it.key as Class<Any>, it.value) }
 		val response = okHttpClient.newCall(request.build()).await()

app/src/main/java/org/koitharu/kotatsu/details/ui/DetailsFragment.kt

@@ -50,7 +50,6 @@ import org.koitharu.kotatsu.utils.ext.drawableTop
 import org.koitharu.kotatsu.utils.ext.enqueueWith
 import org.koitharu.kotatsu.utils.ext.ifNullOrEmpty
 import org.koitharu.kotatsu.utils.ext.measureHeight
-import org.koitharu.kotatsu.utils.ext.referer
 import org.koitharu.kotatsu.utils.ext.resolveDp
 import org.koitharu.kotatsu.utils.ext.scaleUpActivityOptionsOf
 import org.koitharu.kotatsu.utils.ext.textAndVisible
@@ -343,7 +342,6 @@ class DetailsFragment :
 			.data(imageUrl)
 			.tag(manga.source)
 			.crossfade(context)
-			.referer(manga.publicUrl)
 			.lifecycle(viewLifecycleOwner)
 			.placeholderMemoryCacheKey(manga.coverUrl)
 		val previousDrawable = lastResult?.drawable

app/src/main/java/org/koitharu/kotatsu/download/domain/DownloadManager.kt

@@ -38,7 +38,6 @@ import org.koitharu.kotatsu.parsers.util.await
 import org.koitharu.kotatsu.utils.ext.copyToSuspending
 import org.koitharu.kotatsu.utils.ext.deleteAwait
 import org.koitharu.kotatsu.utils.ext.printStackTraceDebug
-import org.koitharu.kotatsu.utils.ext.referer
 import org.koitharu.kotatsu.utils.ext.runCatchingCancellable
 import org.koitharu.kotatsu.utils.progress.PausingProgressJob
 import java.io.File
@@ -220,7 +219,7 @@ class DownloadManager @AssistedInject constructor(
 		val request = Request.Builder()
 			.url(url)
 			.header(CommonHeaders.REFERER, referer)
-			.tag(source)
+			.tag(MangaSource::class.java, source)
 			.cacheControl(CommonHeaders.CACHE_CONTROL_DISABLED)
 			.get()
 			.build()
@@ -250,7 +249,6 @@ class DownloadManager @AssistedInject constructor(
 		imageLoader.execute(
 			ImageRequest.Builder(context)
 				.data(manga.coverUrl)
-				.referer(manga.publicUrl)
 				.tag(manga.source)
 				.size(coverWidth, coverHeight)
 				.scale(Scale.FILL)

app/src/main/java/org/koitharu/kotatsu/download/ui/DownloadItemAD.kt

@@ -17,7 +17,6 @@ import org.koitharu.kotatsu.utils.ext.enqueueWith
 import org.koitharu.kotatsu.utils.ext.getDisplayMessage
 import org.koitharu.kotatsu.utils.ext.newImageRequest
 import org.koitharu.kotatsu.utils.ext.onFirst
-import org.koitharu.kotatsu.utils.ext.referer
 
 fun downloadItemAD(
 	scope: CoroutineScope,
@@ -45,7 +44,6 @@ fun downloadItemAD(
 		job?.cancel()
 		job = item.progressAsFlow().onFirst { state ->
 			binding.imageViewCover.newImageRequest(state.manga.coverUrl, state.manga.source)?.run {
-				referer(state.manga.publicUrl)
 				placeholder(state.cover)
 				fallback(R.drawable.ic_placeholder)
 				error(R.drawable.ic_error_placeholder)

app/src/main/java/org/koitharu/kotatsu/list/ui/adapter/MangaGridItemAD.kt

@@ -15,7 +15,6 @@ import org.koitharu.kotatsu.parsers.model.Manga
 import org.koitharu.kotatsu.utils.ext.disposeImageRequest
 import org.koitharu.kotatsu.utils.ext.enqueueWith
 import org.koitharu.kotatsu.utils.ext.newImageRequest
-import org.koitharu.kotatsu.utils.ext.referer
 import org.koitharu.kotatsu.utils.image.CoverSizeResolver
 
 fun mangaGridItemAD(
@@ -40,7 +39,6 @@ fun mangaGridItemAD(
 		binding.textViewTitle.text = item.title
 		binding.progressView.setPercent(item.progress, MangaListAdapter.PAYLOAD_PROGRESS in payloads)
 		binding.imageViewCover.newImageRequest(item.coverUrl, item.source)?.run {
-			referer(item.manga.publicUrl)
 			size(CoverSizeResolver(binding.imageViewCover))
 			placeholder(R.drawable.ic_placeholder)
 			fallback(R.drawable.ic_placeholder)

app/src/main/java/org/koitharu/kotatsu/list/ui/adapter/MangaListDetailedItemAD.kt

@@ -17,7 +17,6 @@ import org.koitharu.kotatsu.parsers.model.MangaTag
 import org.koitharu.kotatsu.utils.ext.disposeImageRequest
 import org.koitharu.kotatsu.utils.ext.enqueueWith
 import org.koitharu.kotatsu.utils.ext.newImageRequest
-import org.koitharu.kotatsu.utils.ext.referer
 import org.koitharu.kotatsu.utils.ext.textAndVisible
 import org.koitharu.kotatsu.utils.image.CoverSizeResolver
 
@@ -53,7 +52,6 @@ fun mangaListDetailedItemAD(
 		binding.textViewSubtitle.textAndVisible = item.subtitle
 		binding.progressView.setPercent(item.progress, MangaListAdapter.PAYLOAD_PROGRESS in payloads)
 		binding.imageViewCover.newImageRequest(item.coverUrl, item.source)?.run {
-			referer(item.manga.publicUrl)
 			size(CoverSizeResolver(binding.imageViewCover))
 			placeholder(R.drawable.ic_placeholder)
 			fallback(R.drawable.ic_placeholder)

app/src/main/java/org/koitharu/kotatsu/list/ui/adapter/MangaListItemAD.kt

@@ -13,7 +13,6 @@ import org.koitharu.kotatsu.parsers.model.Manga
 import org.koitharu.kotatsu.utils.ext.disposeImageRequest
 import org.koitharu.kotatsu.utils.ext.enqueueWith
 import org.koitharu.kotatsu.utils.ext.newImageRequest
-import org.koitharu.kotatsu.utils.ext.referer
 import org.koitharu.kotatsu.utils.ext.textAndVisible
 
 fun mangaListItemAD(
@@ -36,7 +35,6 @@ fun mangaListItemAD(
 		binding.textViewTitle.text = item.title
 		binding.textViewSubtitle.textAndVisible = item.subtitle
 		binding.imageViewCover.newImageRequest(item.coverUrl, item.source)?.run {
-			referer(item.manga.publicUrl)
 			placeholder(R.drawable.ic_placeholder)
 			fallback(R.drawable.ic_placeholder)
 			error(R.drawable.ic_error_placeholder)

app/src/main/java/org/koitharu/kotatsu/local/ui/ImportService.kt

@@ -26,7 +26,6 @@ import org.koitharu.kotatsu.utils.PendingIntentCompat
 import org.koitharu.kotatsu.utils.ext.asArrayList
 import org.koitharu.kotatsu.utils.ext.getDisplayMessage
 import org.koitharu.kotatsu.utils.ext.printStackTraceDebug
-import org.koitharu.kotatsu.utils.ext.referer
 import org.koitharu.kotatsu.utils.ext.report
 import org.koitharu.kotatsu.utils.ext.toBitmapOrNull
 import javax.inject.Inject
@@ -102,7 +101,6 @@ class ImportService : CoroutineIntentService() {
 					ImageRequest.Builder(applicationContext)
 						.data(manga.coverUrl)
 						.tag(manga.source)
-						.referer(manga.publicUrl)
 						.build(),
 				).toBitmapOrNull(),
 			)

app/src/main/java/org/koitharu/kotatsu/reader/domain/PageLoader.kt

@@ -191,7 +191,7 @@ class PageLoader @Inject constructor(
 				.header(CommonHeaders.REFERER, page.referer)
 				.header(CommonHeaders.ACCEPT, "image/webp,image/png;q=0.9,image/jpeg,*/*;q=0.8")
 				.cacheControl(CommonHeaders.CACHE_CONTROL_DISABLED)
-				.tag(page.source)
+				.tag(MangaSource::class.java, page.source)
 				.build()
 			okHttp.newCall(request).await().use { response ->
 				check(response.isSuccessful) {

app/src/main/java/org/koitharu/kotatsu/reader/ui/colorfilter/ColorFilterConfigActivity.kt

@@ -29,7 +29,6 @@ import org.koitharu.kotatsu.utils.ext.assistedViewModels
 import org.koitharu.kotatsu.utils.ext.decodeRegion
 import org.koitharu.kotatsu.utils.ext.enqueueWith
 import org.koitharu.kotatsu.utils.ext.getParcelableExtraCompat
-import org.koitharu.kotatsu.utils.ext.referer
 import org.koitharu.kotatsu.utils.ext.setValueRounded
 import javax.inject.Inject
 import com.google.android.material.R as materialR
@@ -117,7 +116,6 @@ class ColorFilterConfigActivity :
 		if (preview == null) return
 		ImageRequest.Builder(this@ColorFilterConfigActivity)
 			.data(preview.url)
-			.referer(preview.referer)
 			.scale(Scale.FILL)
 			.decodeRegion()
 			.tag(preview.source)

app/src/main/java/org/koitharu/kotatsu/reader/ui/thumbnails/adapter/PageThumbnailAD.kt

@@ -19,7 +19,6 @@ import org.koitharu.kotatsu.reader.domain.PageLoader
 import org.koitharu.kotatsu.reader.ui.thumbnails.PageThumbnail
 import org.koitharu.kotatsu.utils.ext.decodeRegion
 import org.koitharu.kotatsu.utils.ext.isLowRamDevice
-import org.koitharu.kotatsu.utils.ext.referer
 import org.koitharu.kotatsu.utils.ext.runCatchingCancellable
 import org.koitharu.kotatsu.utils.ext.setTextColorAttr
 import com.google.android.material.R as materialR
@@ -44,7 +43,6 @@ fun pageThumbnailAD(
 			coil.execute(
 				ImageRequest.Builder(context)
 					.data(url)
-					.referer(item.page.referer)
 					.tag(item.page.source)
 					.size(thumbSize)
 					.scale(Scale.FILL)

app/src/main/java/org/koitharu/kotatsu/settings/sources/auth/SourceAuthActivity.kt

@@ -17,7 +17,8 @@ import org.koitharu.kotatsu.base.ui.BaseActivity
 import org.koitharu.kotatsu.browser.BrowserCallback
 import org.koitharu.kotatsu.browser.BrowserClient
 import org.koitharu.kotatsu.browser.ProgressChromeClient
-import org.koitharu.kotatsu.core.network.UserAgentInterceptor
+import org.koitharu.kotatsu.core.network.CommonHeaders
+import org.koitharu.kotatsu.core.network.CommonHeadersInterceptor
 import org.koitharu.kotatsu.core.parser.MangaRepository
 import org.koitharu.kotatsu.core.parser.RemoteMangaRepository
 import org.koitharu.kotatsu.databinding.ActivityBrowserBinding
@@ -60,7 +61,8 @@ class SourceAuthActivity : BaseActivity<ActivityBrowserBinding>(), BrowserCallba
 		}
 		with(binding.webView.settings) {
 			javaScriptEnabled = true
-			userAgentString = repository.userAgent ?: UserAgentInterceptor.userAgent
+			userAgentString = repository.headers?.get(CommonHeaders.USER_AGENT)
+				?: CommonHeadersInterceptor.userAgentFallback
 		}
 		binding.webView.webViewClient = BrowserClient(this)
 		binding.webView.webChromeClient = ProgressChromeClient(binding.progressBar)

app/src/main/java/org/koitharu/kotatsu/tracker/work/TrackWorker.kt

@@ -46,7 +46,6 @@ import org.koitharu.kotatsu.parsers.model.MangaChapter
 import org.koitharu.kotatsu.tracker.domain.Tracker
 import org.koitharu.kotatsu.tracker.domain.model.MangaUpdates
 import org.koitharu.kotatsu.utils.PendingIntentCompat
-import org.koitharu.kotatsu.utils.ext.referer
 import org.koitharu.kotatsu.utils.ext.runCatchingCancellable
 import org.koitharu.kotatsu.utils.ext.toBitmapOrNull
 import org.koitharu.kotatsu.utils.ext.trySetForeground
@@ -157,7 +156,6 @@ class TrackWorker @AssistedInject constructor(
 				coil.execute(
 					ImageRequest.Builder(applicationContext)
 						.data(manga.coverUrl)
-						.referer(manga.publicUrl)
 						.tag(manga.source)
 						.build(),
 				).toBitmapOrNull(),

app/src/main/java/org/koitharu/kotatsu/utils/ext/CoilExt.kt

@@ -10,9 +10,7 @@ import coil.request.ImageResult
 import coil.request.SuccessResult
 import coil.util.CoilUtils
 import com.google.android.material.progressindicator.BaseProgressIndicator
-import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
 import org.koitharu.kotatsu.R
-import org.koitharu.kotatsu.core.network.CommonHeaders
 import org.koitharu.kotatsu.parsers.model.MangaSource
 import org.koitharu.kotatsu.utils.image.RegionBitmapDecoder
 import org.koitharu.kotatsu.utils.progress.ImageRequestIndicatorListener
@@ -51,21 +49,6 @@ fun ImageResult.toBitmapOrNull() = when (this) {
 	is ErrorResult -> null
 }
 
-fun ImageRequest.Builder.referer(referer: String): ImageRequest.Builder {
-	if (referer.isEmpty()) {
-		return this
-	}
-	try {
-		setHeader(CommonHeaders.REFERER, referer)
-	} catch (e: IllegalArgumentException) {
-		val baseUrl = referer.baseUrl()
-		if (baseUrl != null) {
-			setHeader(CommonHeaders.REFERER, baseUrl)
-		}
-	}
-	return this
-}
-
 fun ImageRequest.Builder.indicator(indicator: BaseProgressIndicator<*>): ImageRequest.Builder {
 	return listener(ImageRequestIndicatorListener(indicator))
 }
@@ -83,11 +66,3 @@ fun ImageRequest.Builder.crossfade(context: Context?): ImageRequest.Builder {
 	val duration = context.resources.getInteger(R.integer.config_defaultAnimTime) * context.animatorDurationScale
 	return crossfade(duration.toInt())
 }
-
-private fun String.baseUrl(): String? {
-	return (this.toHttpUrlOrNull()?.newBuilder("/") ?: return null)
-		.username("")
-		.password("")
-		.build()
-		.toString()
-}

app/src/release/java/org/koitharu/kotatsu/core/parser/DummyParser.kt

@@ -12,7 +12,7 @@ import org.koitharu.kotatsu.parsers.model.MangaTag
 import org.koitharu.kotatsu.parsers.model.SortOrder
 import java.util.EnumSet
 
-class DummyParser(override val context: MangaLoaderContext) : MangaParser(MangaSource.DUMMY) {
+class DummyParser(context: MangaLoaderContext) : MangaParser(context, MangaSource.DUMMY) {
 
 	override val configKeyDomain: ConfigKey.Domain
 		get() = ConfigKey.Domain("localhost", null)