Improve tracker part 2

This commit is contained in:
Koitharu
2024-04-06 17:12:27 +03:00
parent 54ff63dbc7
commit 1bf01ca240
9 changed files with 146 additions and 166 deletions

View File

@@ -18,6 +18,7 @@ fun calculateTimeAgo(instant: Instant, showMonths: Boolean = false): DateTimeAgo
if (instant.until(Instant.now(), ChronoUnit.MINUTES) < 3) DateTimeAgo.JustNow
else DateTimeAgo.Today
}
diffDays == 1L -> DateTimeAgo.Yesterday
diffDays < 6 -> DateTimeAgo.DaysAgo(diffDays.toInt())
else -> {
@@ -30,3 +31,5 @@ fun calculateTimeAgo(instant: Instant, showMonths: Boolean = false): DateTimeAgo
}
}
}
fun Long.toInstantOrNull() = if (this == 0L) null else Instant.ofEpochMilli(this)

View File

@@ -50,6 +50,9 @@ abstract class FavouritesDao {
@Query("SELECT * FROM favourites WHERE deleted_at = 0 ORDER BY created_at DESC LIMIT :limit OFFSET :offset")
abstract suspend fun findAllRaw(offset: Int, limit: Int): List<FavouriteManga>
@Query("SELECT DISTINCT manga_id FROM favourites WHERE deleted_at = 0 AND category_id IN (SELECT category_id FROM favourite_categories WHERE track = 1)")
abstract suspend fun findIdsWithTrack(): LongArray
@Transaction
@Query(
"SELECT * FROM favourites WHERE category_id = :categoryId AND deleted_at = 0 " +
@@ -135,6 +138,9 @@ abstract class FavouritesDao {
@Query("SELECT DISTINCT category_id FROM favourites WHERE manga_id IN (:mangaIds) AND deleted_at = 0 ORDER BY favourites.created_at ASC")
abstract suspend fun findCategoriesIds(mangaIds: Collection<Long>): List<Long>
@Query("SELECT DISTINCT favourite_categories.category_id FROM favourites LEFT JOIN favourite_categories ON favourites.category_id = favourite_categories.category_id WHERE manga_id = :mangaId AND favourites.deleted_at = 0 AND favourite_categories.deleted_at = 0 AND favourite_categories.track = 1")
abstract suspend fun findCategoriesIdsWithTrack(mangaId: Long): List<Long>
/** INSERT **/
@Insert(onConflict = OnConflictStrategy.REPLACE)

View File

@@ -58,6 +58,9 @@ abstract class HistoryDao {
@Query("SELECT * FROM manga WHERE manga_id IN (SELECT manga_id FROM history WHERE deleted_at = 0)")
abstract suspend fun findAllManga(): List<MangaEntity>
@Query("SELECT manga_id FROM history WHERE deleted_at = 0")
abstract suspend fun findAllIds(): LongArray
@Query(
"""SELECT tags.* FROM tags
LEFT JOIN manga_tags ON tags.tag_id = manga_tags.tag_id

View File

@@ -33,5 +33,14 @@ class TrackEntity(
const val RESULT_HAS_UPDATE = 1
const val RESULT_NO_UPDATE = 2
const val RESULT_FAILED = 3
fun create(mangaId: Long) = TrackEntity(
mangaId = mangaId,
lastChapterId = 0L,
newChapters = 0,
lastCheckTime = 0L,
lastChapterDate = 0,
lastResult = RESULT_NONE,
)
}
}

View File

@@ -0,0 +1,14 @@
package org.koitharu.kotatsu.tracker.data
import androidx.room.Embedded
import androidx.room.Relation
import org.koitharu.kotatsu.core.db.entity.MangaEntity
class TrackWithManga(
@Embedded val track: TrackEntity,
@Relation(
parentColumn = "manga_id",
entityColumn = "manga_id",
)
val manga: MangaEntity,
)

View File

@@ -14,6 +14,13 @@ abstract class TracksDao {
@Query("SELECT * FROM tracks")
abstract suspend fun findAll(): List<TrackEntity>
@Transaction
@Query("SELECT * FROM tracks ORDER BY last_check_time ASC LIMIT :limit OFFSET :offset")
abstract suspend fun findAll(offset: Int, limit: Int): List<TrackWithManga>
@Query("SELECT manga_id FROM tracks")
abstract suspend fun findAllIds(): LongArray
@Query("SELECT * FROM tracks WHERE manga_id IN (:ids)")
abstract suspend fun findAll(ids: Collection<Long>): List<TrackEntity>

View File

@@ -1,7 +1,6 @@
package org.koitharu.kotatsu.tracker.domain
import androidx.annotation.VisibleForTesting
import androidx.collection.MutableLongSet
import coil.request.CachePolicy
import org.koitharu.kotatsu.core.model.getPreferredBranch
import org.koitharu.kotatsu.core.parser.MangaRepository
@@ -10,6 +9,7 @@ import org.koitharu.kotatsu.core.prefs.AppSettings
import org.koitharu.kotatsu.core.util.CompositeMutex2
import org.koitharu.kotatsu.history.data.HistoryRepository
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
import org.koitharu.kotatsu.tracker.domain.model.MangaTracking
import org.koitharu.kotatsu.tracker.domain.model.MangaUpdates
import org.koitharu.kotatsu.tracker.work.TrackerNotificationChannels
@@ -26,56 +26,19 @@ class Tracker @Inject constructor(
private val mangaRepositoryFactory: MangaRepository.Factory,
) {
suspend fun getAllTracks(): List<TrackingItem> {
val sources = settings.trackSources
if (sources.isEmpty()) {
return emptyList()
}
val knownManga = MutableLongSet()
val result = ArrayList<TrackingItem>()
// Favourites
if (AppSettings.TRACK_FAVOURITES in sources) {
val favourites = repository.getAllFavouritesManga()
channels.updateChannels(favourites.keys)
for ((category, mangaList) in favourites) {
if (!category.isTrackingEnabled || mangaList.isEmpty()) {
continue
}
val categoryTracks = repository.getTracks(mangaList)
val channelId = if (channels.isFavouriteNotificationsEnabled(category)) {
channels.getFavouritesChannelId(category.id)
} else {
null
}
for (track in categoryTracks) {
if (knownManga.add(track.manga.id)) {
result.add(TrackingItem(track, channelId))
}
}
}
}
// History
if (AppSettings.TRACK_HISTORY in sources) {
for (i in 0 until historyRepository.getCount() step 20) {
val history = historyRepository.getList(i, 20)
val historyTracks = repository.getTracks(history)
val channelId = if (channels.isHistoryNotificationsEnabled()) {
suspend fun getTracks(limit: Int): List<TrackingItem> {
repository.updateTracks()
return repository.getTracks(0, limit).map {
val categoryId = repository.getCategoryId(it.manga.id)
TrackingItem(
tracking = it,
channelId = if (categoryId == 0L) {
channels.getHistoryChannelId()
} else {
null
}
for (track in historyTracks) {
if (knownManga.add(track.manga.id)) {
result.add(TrackingItem(track, channelId))
}
}
}
channels.getFavouritesChannelId(categoryId)
},
)
}
return result
}
suspend fun getTracks(ids: Set<Long>): List<TrackingItem> {
return getAllTracks().filterTo(ArrayList(ids.size)) { x -> x.tracking.manga.id in ids }
}
suspend fun gc() {
@@ -85,11 +48,18 @@ class Tracker @Inject constructor(
suspend fun fetchUpdates(
track: MangaTracking,
commit: Boolean
): MangaUpdates.Success = withMangaLock(track.manga.id) {
val repo = mangaRepositoryFactory.create(track.manga.source)
require(repo is RemoteMangaRepository) { "Repository ${repo.javaClass.simpleName} is not supported" }
val manga = repo.getDetails(track.manga, CachePolicy.WRITE_ONLY)
val updates = compare(track, manga, getBranch(manga))
): MangaUpdates = withMangaLock(track.manga.id) {
val updates = runCatchingCancellable {
val repo = mangaRepositoryFactory.create(track.manga.source)
require(repo is RemoteMangaRepository) { "Repository ${repo.javaClass.simpleName} is not supported" }
val manga = repo.getDetails(track.manga, CachePolicy.WRITE_ONLY)
compare(track, manga, getBranch(manga))
}.getOrElse { error ->
MangaUpdates.Failure(
manga = track.manga,
error = error,
)
}
if (commit) {
repository.saveUpdates(updates)
}

View File

@@ -1,7 +1,6 @@
package org.koitharu.kotatsu.tracker.domain
import androidx.annotation.VisibleForTesting
import androidx.collection.MutableLongSet
import androidx.room.withTransaction
import dagger.Reusable
import kotlinx.coroutines.flow.Flow
@@ -14,20 +13,19 @@ import org.koitharu.kotatsu.core.db.MangaDatabase
import org.koitharu.kotatsu.core.db.entity.MangaEntity
import org.koitharu.kotatsu.core.db.entity.toManga
import org.koitharu.kotatsu.core.model.FavouriteCategory
import org.koitharu.kotatsu.core.model.isLocal
import org.koitharu.kotatsu.core.prefs.AppSettings
import org.koitharu.kotatsu.core.util.ext.ifZero
import org.koitharu.kotatsu.core.util.ext.mapItems
import org.koitharu.kotatsu.core.util.ext.toInstantOrNull
import org.koitharu.kotatsu.favourites.data.toFavouriteCategory
import org.koitharu.kotatsu.local.data.LocalMangaRepository
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.util.mapToSet
import org.koitharu.kotatsu.tracker.data.TrackEntity
import org.koitharu.kotatsu.tracker.data.TrackLogEntity
import org.koitharu.kotatsu.tracker.data.toTrackingLogItem
import org.koitharu.kotatsu.tracker.domain.model.MangaTracking
import org.koitharu.kotatsu.tracker.domain.model.MangaUpdates
import org.koitharu.kotatsu.tracker.domain.model.TrackingLogItem
import java.time.Instant
import java.util.concurrent.atomic.AtomicBoolean
import javax.inject.Inject
import javax.inject.Provider
@@ -42,6 +40,7 @@ private const val MAX_LOG_SIZE = 120
@Reusable
class TrackingRepository @Inject constructor(
private val db: MangaDatabase,
private val settings: AppSettings,
private val localMangaRepositoryProvider: Provider<LocalMangaRepository>,
) {
@@ -70,36 +69,18 @@ class TrackingRepository @Inject constructor(
.onStart { gcIfNotCalled() }
}
@Deprecated("")
suspend fun getTracks(mangaList: Collection<Manga>): List<MangaTracking> {
val ids = mangaList.mapToSet { it.id }
val dao = db.getTracksDao()
val tracks = if (ids.size <= MAX_QUERY_IDS) {
dao.findAll(ids)
} else {
// TODO split tracks in the worker
ids.windowed(MAX_QUERY_IDS, MAX_QUERY_IDS, true)
.flatMap { dao.findAll(it) }
}.groupBy { it.mangaId }
val idSet = MutableLongSet(mangaList.size)
val result = ArrayList<MangaTracking>(mangaList.size)
for (item in mangaList) {
val manga = if (item.isLocal) {
localMangaRepositoryProvider.get().getRemoteManga(item) ?: continue
} else {
item
}
if (!idSet.add(manga.id)) {
continue
}
val track = tracks[manga.id]?.lastOrNull()
result += MangaTracking(
manga = manga,
lastChapterId = track?.lastChapterId ?: NO_ID,
lastCheck = track?.lastCheckTime?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli),
suspend fun getCategoryId(mangaId: Long): Long {
return db.getFavouritesDao().findCategoriesIdsWithTrack(mangaId).firstOrNull() ?: NO_ID
}
suspend fun getTracks(offset: Int, limit: Int): List<MangaTracking> {
return db.getTracksDao().findAll(offset, limit).map {
MangaTracking(
manga = it.manga.toManga(emptySet()),
lastChapterId = it.track.lastChapterId,
lastCheck = it.track.lastCheckTime.toInstantOrNull(),
)
}
return result
}
@VisibleForTesting
@@ -108,7 +89,7 @@ class TrackingRepository @Inject constructor(
return MangaTracking(
manga = manga,
lastChapterId = track?.lastChapterId ?: NO_ID,
lastCheck = track?.lastCheckTime?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli),
lastCheck = track?.lastCheckTime?.toInstantOrNull(),
)
}
@@ -145,11 +126,11 @@ class TrackingRepository @Inject constructor(
}
}
suspend fun saveUpdates(updates: MangaUpdates.Success) {
suspend fun saveUpdates(updates: MangaUpdates) {
db.withTransaction {
val track = getOrCreateTrack(updates.manga.id).mergeWith(updates)
db.getTracksDao().upsert(track)
if (updates.isValid && updates.newChapters.isNotEmpty()) {
if (updates is MangaUpdates.Success && updates.isValid && updates.newChapters.isNotEmpty()) {
updatePercent(updates)
val logEntity = TrackLogEntity(
mangaId = updates.manga.id,
@@ -211,15 +192,38 @@ class TrackingRepository @Inject constructor(
}
}
suspend fun updateTracks() = db.withTransaction {
val dao = db.getTracksDao()
dao.gc()
val ids = dao.findAllIds().toMutableSet()
val size = ids.size
// history
if (AppSettings.TRACK_HISTORY in settings.trackSources) {
val historyIds = db.getHistoryDao().findAllIds()
for (mangaId in historyIds) {
if (!ids.remove(mangaId)) {
dao.upsert(TrackEntity.create(mangaId))
}
}
}
// favorites
if (AppSettings.TRACK_FAVOURITES in settings.trackSources) {
val favoritesIds = db.getFavouritesDao().findIdsWithTrack()
for (mangaId in favoritesIds) {
if (!ids.remove(mangaId)) {
dao.upsert(TrackEntity.create(mangaId))
}
}
}
// remove unused
for (mangaId in ids) {
dao.delete(mangaId)
}
size - ids.size
}
private suspend fun getOrCreateTrack(mangaId: Long): TrackEntity {
return db.getTracksDao().find(mangaId) ?: TrackEntity(
mangaId = mangaId,
lastChapterId = 0L,
newChapters = 0,
lastCheckTime = 0L,
lastChapterDate = 0,
lastResult = TrackEntity.RESULT_NONE,
)
return db.getTracksDao().find(mangaId) ?: TrackEntity.create(mangaId)
}
private suspend fun updatePercent(updates: MangaUpdates.Success) {
@@ -237,16 +241,27 @@ class TrackingRepository @Inject constructor(
db.getHistoryDao().update(history.copy(percent = newPercent))
}
private fun TrackEntity.mergeWith(updates: MangaUpdates.Success): TrackEntity {
private fun TrackEntity.mergeWith(updates: MangaUpdates): TrackEntity {
val chapters = updates.manga.chapters.orEmpty()
return TrackEntity(
mangaId = mangaId,
lastChapterId = chapters.lastOrNull()?.id ?: NO_ID,
newChapters = if (updates.isValid) newChapters + updates.newChapters.size else 0,
lastCheckTime = System.currentTimeMillis(),
lastChapterDate = updates.lastChapterDate().ifZero { lastChapterDate },
lastResult = if (updates.isNotEmpty()) TrackEntity.RESULT_HAS_UPDATE else TrackEntity.RESULT_NO_UPDATE,
)
return when (updates) {
is MangaUpdates.Failure -> TrackEntity(
mangaId = mangaId,
lastChapterId = lastChapterId,
newChapters = newChapters,
lastCheckTime = System.currentTimeMillis(),
lastChapterDate = lastChapterDate,
lastResult = TrackEntity.RESULT_FAILED,
)
is MangaUpdates.Success -> TrackEntity(
mangaId = mangaId,
lastChapterId = chapters.lastOrNull()?.id ?: NO_ID,
newChapters = if (updates.isValid) newChapters + updates.newChapters.size else 0,
lastCheckTime = System.currentTimeMillis(),
lastChapterDate = updates.lastChapterDate().ifZero { lastChapterDate },
lastResult = if (updates.isNotEmpty()) TrackEntity.RESULT_HAS_UPDATE else TrackEntity.RESULT_NO_UPDATE,
)
}
}
private suspend fun gcIfNotCalled() {

View File

@@ -11,7 +11,6 @@ import androidx.core.app.NotificationCompat.VISIBILITY_SECRET
import androidx.core.app.NotificationManagerCompat
import androidx.core.app.PendingIntentCompat
import androidx.core.content.ContextCompat
import androidx.core.content.edit
import androidx.hilt.work.HiltWorker
import androidx.work.BackoffPolicy
import androidx.work.Constraints
@@ -34,14 +33,12 @@ import dagger.Reusable
import dagger.assisted.Assisted
import dagger.assisted.AssistedInject
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.NonCancellable
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.channelFlow
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.toList
import kotlinx.coroutines.launch
import kotlinx.coroutines.runInterruptible
import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withPermit
import kotlinx.coroutines.withContext
@@ -59,7 +56,6 @@ import org.koitharu.kotatsu.core.util.ext.trySetForeground
import org.koitharu.kotatsu.details.ui.DetailsActivity
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.util.mapToSet
import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
import org.koitharu.kotatsu.settings.SettingsActivity
import org.koitharu.kotatsu.settings.work.PeriodicWorkScheduler
@@ -86,7 +82,7 @@ class TrackWorker @AssistedInject constructor(
trySetForeground()
logger.log("doWork(): attempt $runAttemptCount")
return try {
doWorkImpl()
doWorkImpl(isFullRun = TAG_ONESHOT in tags)
} catch (e: CancellationException) {
throw e
} catch (e: Throwable) {
@@ -100,49 +96,18 @@ class TrackWorker @AssistedInject constructor(
}
}
private suspend fun doWorkImpl(): Result {
private suspend fun doWorkImpl(isFullRun: Boolean): Result {
if (!settings.isTrackerEnabled) {
return Result.success(workDataOf(0, 0))
}
val retryIds = getRetryIds()
val tracks = if (retryIds.isNotEmpty()) {
tracker.getTracks(retryIds)
} else {
tracker.getAllTracks()
}
val tracks = tracker.getTracks(if (isFullRun) Int.MAX_VALUE else BATCH_SIZE)
logger.log("Total ${tracks.size} tracks")
if (tracks.isEmpty()) {
return Result.success(workDataOf(0, 0))
}
val results = checkUpdatesAsync(tracks)
tracker.gc()
var success = 0
var failed = 0
val retry = HashSet<Long>()
results.forEach { x ->
when (x) {
is MangaUpdates.Success -> success++
is MangaUpdates.Failure -> {
failed++
if (x.shouldRetry()) {
retry += x.manga.id
}
}
}
}
if (runAttemptCount > MAX_ATTEMPTS) {
retry.clear()
}
setRetryIds(retry)
logger.log("Result: success: $success, failed: $failed, retry: ${retry.size}")
val resultData = workDataOf(success, failed)
return when {
retry.isNotEmpty() -> Result.retry()
success == 0 && failed != 0 -> Result.failure(resultData)
else -> Result.success(resultData)
}
checkUpdatesAsync(tracks)
return Result.success()
}
private suspend fun checkUpdatesAsync(tracks: List<TrackingItem>): List<MangaUpdates> {
@@ -153,10 +118,13 @@ class TrackWorker @AssistedInject constructor(
semaphore.withPermit {
send(
runCatchingCancellable {
tracker.fetchUpdates(track, commit = true)
.copy(channelId = channelId)
}.onFailure { e ->
logger.log("checkUpdatesAsync", e)
tracker.fetchUpdates(track, commit = true).let {
if (it is MangaUpdates.Success) {
it.copy(channelId = channelId)
} else {
it
}
}
}.getOrElse { error ->
MangaUpdates.Failure(
manga = track.manga,
@@ -174,6 +142,7 @@ class TrackWorker @AssistedInject constructor(
when (it) {
is MangaUpdates.Failure -> {
val e = it.error
logger.log("checkUpdatesAsync", e)
if (e is CloudFlareProtectedException) {
CaptchaNotifier(applicationContext).notify(e)
}
@@ -323,22 +292,6 @@ class TrackWorker @AssistedInject constructor(
)
}.build()
private suspend fun setRetryIds(ids: Set<Long>) = runInterruptible(Dispatchers.IO) {
val prefs = applicationContext.getSharedPreferences(TAG, Context.MODE_PRIVATE)
prefs.edit(commit = true) {
if (ids.isEmpty()) {
remove(KEY_RETRY_IDS)
} else {
putStringSet(KEY_RETRY_IDS, ids.mapToSet { it.toString() })
}
}
}
private fun getRetryIds(): Set<Long> {
val prefs = applicationContext.getSharedPreferences(TAG, Context.MODE_PRIVATE)
return prefs.getStringSet(KEY_RETRY_IDS, null)?.mapToSet { it.toLong() }.orEmpty()
}
private fun workDataOf(success: Int, failed: Int): Data {
return Data.Builder()
.putInt(DATA_KEY_SUCCESS, success)
@@ -410,6 +363,6 @@ class TrackWorker @AssistedInject constructor(
const val MAX_ATTEMPTS = 3
const val DATA_KEY_SUCCESS = "success"
const val DATA_KEY_FAILED = "failed"
const val KEY_RETRY_IDS = "retry"
const val BATCH_SIZE = 20
}
}