diff --git a/app/src/main/java/org/koitharu/kotatsu/base/ui/list/fastscroll/FastScrollRecyclerView.kt b/app/src/main/java/org/koitharu/kotatsu/base/ui/list/fastscroll/FastScrollRecyclerView.kt
index 079391703..5a7c1274e 100644
--- a/app/src/main/java/org/koitharu/kotatsu/base/ui/list/fastscroll/FastScrollRecyclerView.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/base/ui/list/fastscroll/FastScrollRecyclerView.kt
@@ -4,11 +4,8 @@ import android.content.Context
 import android.util.AttributeSet
 import android.view.ViewGroup
 import androidx.annotation.AttrRes
-import androidx.annotation.ColorInt
 import androidx.recyclerview.widget.RecyclerView
-import androidx.swiperefreshlayout.widget.SwipeRefreshLayout
 import org.koitharu.kotatsu.R
-import org.koitharu.kotatsu.utils.ext.parents
 
 class FastScrollRecyclerView @JvmOverloads constructor(
 	context: Context,
@@ -36,44 +33,13 @@ class FastScrollRecyclerView @JvmOverloads constructor(
 		fastScroller.visibility = visibility
 	}
 
-	fun setFastScrollListener(fastScrollListener: FastScroller.FastScrollListener?) =
-		fastScroller.setFastScrollListener(fastScrollListener)
-
-	fun setFastScrollEnabled(enabled: Boolean) {
-		fastScroller.isEnabled = enabled
-	}
-
-	fun setHideScrollbar(hideScrollbar: Boolean) = fastScroller.setHideScrollbar(hideScrollbar)
-
-	fun setTrackVisible(visible: Boolean) = fastScroller.setTrackVisible(visible)
-
-	fun setTrackColor(@ColorInt color: Int) = fastScroller.setTrackColor(color)
-
-	fun setHandleColor(@ColorInt color: Int) = fastScroller.setHandleColor(color)
-
-	@JvmOverloads
-	fun setBubbleVisible(visible: Boolean, always: Boolean = false) = fastScroller.setBubbleVisible(visible, always)
-
-	fun setBubbleColor(@ColorInt color: Int) = fastScroller.setBubbleColor(color)
-
-	fun setBubbleTextColor(@ColorInt color: Int) = fastScroller.setBubbleTextColor(color)
-
-	fun setBubbleTextSize(size: Int) = fastScroller.setBubbleTextSize(size)
-
 	override fun onAttachedToWindow() {
 		super.onAttachedToWindow()
 		fastScroller.attachRecyclerView(this)
-		for (p in parents) {
-			if (p is SwipeRefreshLayout) {
-				fastScroller.setSwipeRefreshLayout(p)
-				return
-			}
-		}
 	}
 
 	override fun onDetachedFromWindow() {
 		fastScroller.detachRecyclerView()
-		fastScroller.setSwipeRefreshLayout(null)
 		super.onDetachedFromWindow()
 	}
 }
\ No newline at end of file
diff --git a/app/src/main/java/org/koitharu/kotatsu/base/ui/list/fastscroll/FastScroller.kt b/app/src/main/java/org/koitharu/kotatsu/base/ui/list/fastscroll/FastScroller.kt
index 8f7b69448..464c071e7 100644
--- a/app/src/main/java/org/koitharu/kotatsu/base/ui/list/fastscroll/FastScroller.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/base/ui/list/fastscroll/FastScroller.kt
@@ -19,7 +19,6 @@ import androidx.core.view.GravityCompat
 import androidx.core.view.isGone
 import androidx.core.view.isVisible
 import androidx.recyclerview.widget.RecyclerView
-import androidx.swiperefreshlayout.widget.SwipeRefreshLayout
 import org.koitharu.kotatsu.R
 import org.koitharu.kotatsu.databinding.FastScrollerBinding
 import org.koitharu.kotatsu.utils.ext.*
@@ -62,7 +61,6 @@ class FastScroller @JvmOverloads constructor(
 	private var handleImage: Drawable? = null
 	private var trackImage: Drawable? = null
 	private var recyclerView: RecyclerView? = null
-	private var swipeRefreshLayout: SwipeRefreshLayout? = null
 	private val scrollbarAnimator = ScrollbarAnimator(binding.scrollbar, scrollbarPaddingEnd)
 	private val bubbleAnimator = BubbleAnimator(binding.bubble)
 
@@ -85,12 +83,6 @@ class FastScroller @JvmOverloads constructor(
 					sectionIndexer?.let { binding.bubble.text = it.getSectionText(recyclerView.context, targetPos) }
 				}
 			}
-
-			swipeRefreshLayout?.let {
-				val firstVisibleItem = recyclerView.layoutManager.firstVisibleItemPosition
-				val topPosition = if (recyclerView.childCount == 0) 0 else recyclerView.getChildAt(0).top
-				it.isEnabled = firstVisibleItem == 0 && topPosition >= 0
-			}
 		}
 
 		override fun onScrollStateChanged(recyclerView: RecyclerView, newState: Int) {
@@ -332,18 +324,6 @@ class FastScroller @JvmOverloads constructor(
 		this.sectionIndexer = sectionIndexer
 	}
 
-	/**
-	 * Set a [SwipeRefreshLayout] to disable when the [RecyclerView] is scrolled away from the top.
-	 *
-	 * Required when SDK target precedes [VERSION_CODES.LOLLIPOP], otherwise use
-	 * [setNestedScrollingEnabled(true)][View.setNestedScrollingEnabled].
-	 *
-	 * @param swipeRefreshLayout The [SwipeRefreshLayout] to set, or null to set none
-	 */
-	fun setSwipeRefreshLayout(swipeRefreshLayout: SwipeRefreshLayout?) {
-		this.swipeRefreshLayout = swipeRefreshLayout
-	}
-
 	/**
 	 * Hide the scrollbar when not scrolling.
 	 *
diff --git a/app/src/main/java/org/koitharu/kotatsu/list/ui/MangaListFragment.kt b/app/src/main/java/org/koitharu/kotatsu/list/ui/MangaListFragment.kt
index bbf088593..1fd7574cc 100644
--- a/app/src/main/java/org/koitharu/kotatsu/list/ui/MangaListFragment.kt
+++ b/app/src/main/java/org/koitharu/kotatsu/list/ui/MangaListFragment.kt
@@ -23,6 +23,7 @@ import org.koitharu.kotatsu.base.ui.list.ListSelectionController
 import org.koitharu.kotatsu.base.ui.list.PaginationScrollListener
 import org.koitharu.kotatsu.base.ui.list.decor.SpacingItemDecoration
 import org.koitharu.kotatsu.base.ui.list.decor.TypedSpacingItemDecoration
+import org.koitharu.kotatsu.base.ui.list.fastscroll.FastScroller
 import org.koitharu.kotatsu.browser.cloudflare.CloudFlareDialog
 import org.koitharu.kotatsu.core.exceptions.CloudFlareProtectedException
 import org.koitharu.kotatsu.core.exceptions.resolve.ExceptionResolver
@@ -48,7 +49,7 @@ abstract class MangaListFragment :
 	PaginationScrollListener.Callback,
 	MangaListListener,
 	SwipeRefreshLayout.OnRefreshListener,
-	ListSelectionController.Callback {
+	ListSelectionController.Callback, FastScroller.FastScrollListener {
 
 	private var listAdapter: MangaListAdapter? = null
 	private var paginationListener: PaginationScrollListener? = null
@@ -88,6 +89,7 @@ abstract class MangaListFragment :
 			adapter = listAdapter
 			checkNotNull(selectionController).attachToRecyclerView(binding.recyclerView)
 			addOnScrollListener(paginationListener!!)
+			fastScroller.setFastScrollListener(this@MangaListFragment)
 		}
 		with(binding.swipeRefreshLayout) {
 			setProgressBackgroundColorSchemeColor(context.getThemeColor(com.google.android.material.R.attr.colorPrimary))
@@ -289,6 +291,14 @@ abstract class MangaListFragment :
 		binding.recyclerView.invalidateItemDecorations()
 	}
 
+	override fun onFastScrollStart(fastScroller: FastScroller) {
+		binding.swipeRefreshLayout.isEnabled = false
+	}
+
+	override fun onFastScrollStop(fastScroller: FastScroller) {
+		binding.swipeRefreshLayout.isEnabled = isSwipeRefreshEnabled
+	}
+
 	private fun collectSelectedItems(): Set<Manga> {
 		val checkedIds = selectionController?.peekCheckedIds() ?: return emptySet()
 		val items = listAdapter?.items ?: return emptySet()