diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/core/db/MangaDatabase.kt b/app/src/main/kotlin/org/koitharu/kotatsu/core/db/MangaDatabase.kt index 8d6571aaa..c0b3d6efa 100644 --- a/app/src/main/kotlin/org/koitharu/kotatsu/core/db/MangaDatabase.kt +++ b/app/src/main/kotlin/org/koitharu/kotatsu/core/db/MangaDatabase.kt @@ -31,6 +31,7 @@ import org.koitharu.kotatsu.core.db.migrations.Migration15To16 import org.koitharu.kotatsu.core.db.migrations.Migration16To17 import org.koitharu.kotatsu.core.db.migrations.Migration17To18 import org.koitharu.kotatsu.core.db.migrations.Migration18To19 +import org.koitharu.kotatsu.core.db.migrations.Migration19To20 import org.koitharu.kotatsu.core.db.migrations.Migration1To2 import org.koitharu.kotatsu.core.db.migrations.Migration2To3 import org.koitharu.kotatsu.core.db.migrations.Migration3To4 @@ -57,7 +58,7 @@ import org.koitharu.kotatsu.tracker.data.TrackEntity import org.koitharu.kotatsu.tracker.data.TrackLogEntity import org.koitharu.kotatsu.tracker.data.TracksDao -const val DATABASE_VERSION = 19 +const val DATABASE_VERSION = 20 @Database( entities = [ @@ -116,6 +117,7 @@ fun getDatabaseMigrations(context: Context): Array = arrayOf( Migration16To17(context), Migration17To18(), Migration18To19(), + Migration19To20(), ) fun MangaDatabase(context: Context): MangaDatabase = Room diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/core/db/dao/TrackLogsDao.kt b/app/src/main/kotlin/org/koitharu/kotatsu/core/db/dao/TrackLogsDao.kt index ee0da6165..1d0b2bc10 100644 --- a/app/src/main/kotlin/org/koitharu/kotatsu/core/db/dao/TrackLogsDao.kt +++ b/app/src/main/kotlin/org/koitharu/kotatsu/core/db/dao/TrackLogsDao.kt @@ -1,6 +1,10 @@ package org.koitharu.kotatsu.core.db.dao -import androidx.room.* +import androidx.room.Dao +import androidx.room.Insert +import androidx.room.OnConflictStrategy +import androidx.room.Query +import androidx.room.Transaction import kotlinx.coroutines.flow.Flow import org.koitharu.kotatsu.tracker.data.TrackLogEntity import org.koitharu.kotatsu.tracker.data.TrackLogWithManga @@ -24,6 +28,9 @@ interface TrackLogsDao { @Query("DELETE FROM track_logs WHERE manga_id NOT IN (SELECT manga_id FROM tracks)") suspend fun gc() + @Query("DELETE FROM track_logs WHERE id IN (SELECT id FROM track_logs ORDER BY created_at DESC LIMIT 0 OFFSET :size)") + suspend fun trim(size: Int) + @Query("SELECT COUNT(*) FROM track_logs") suspend fun count(): Int } diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/core/db/migrations/Migration19To20.kt b/app/src/main/kotlin/org/koitharu/kotatsu/core/db/migrations/Migration19To20.kt new file mode 100644 index 000000000..0bd5d3c8e --- /dev/null +++ b/app/src/main/kotlin/org/koitharu/kotatsu/core/db/migrations/Migration19To20.kt @@ -0,0 +1,16 @@ +package org.koitharu.kotatsu.core.db.migrations + +import androidx.room.migration.Migration +import androidx.sqlite.db.SupportSQLiteDatabase + +class Migration19To20 : Migration(19, 20) { + + override fun migrate(db: SupportSQLiteDatabase) { + db.execSQL("CREATE TABLE tracks_bk (manga_id INTEGER NOT NULL, chapters_total INTEGER NOT NULL, last_chapter_id INTEGER NOT NULL, chapters_new INTEGER NOT NULL, last_check INTEGER NOT NULL, last_notified_id INTEGER NOT NULL, PRIMARY KEY(manga_id))") + db.execSQL("INSERT INTO tracks_bk SELECT manga_id, chapters_total, last_chapter_id, chapters_new, last_check, last_notified_id FROM tracks") + db.execSQL("DROP TABLE tracks") + db.execSQL("CREATE TABLE tracks (`manga_id` INTEGER NOT NULL, `last_chapter_id` INTEGER NOT NULL, `chapters_new` INTEGER NOT NULL, `last_check_time` INTEGER NOT NULL, `last_chapter_date` INTEGER NOT NULL, `last_result` INTEGER NOT NULL, PRIMARY KEY(`manga_id`), FOREIGN KEY(`manga_id`) REFERENCES `manga`(`manga_id`) ON UPDATE NO ACTION ON DELETE CASCADE )") + db.execSQL("INSERT INTO tracks SELECT manga_id, last_chapter_id, chapters_new, last_check AS last_check_time, 0 AS last_chapter_date, 0 AS last_result FROM tracks_bk") + db.execSQL("DROP TABLE tracks_bk") + } +} diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/core/util/ext/Primitive.kt b/app/src/main/kotlin/org/koitharu/kotatsu/core/util/ext/Primitive.kt index 85fe52e38..4068968a4 100644 --- a/app/src/main/kotlin/org/koitharu/kotatsu/core/util/ext/Primitive.kt +++ b/app/src/main/kotlin/org/koitharu/kotatsu/core/util/ext/Primitive.kt @@ -2,6 +2,8 @@ package org.koitharu.kotatsu.core.util.ext inline fun Int.ifZero(defaultValue: () -> Int): Int = if (this == 0) defaultValue() else this +inline fun Long.ifZero(defaultValue: () -> Long): Long = if (this == 0L) defaultValue() else this + fun longOf(a: Int, b: Int): Long { return a.toLong() shl 32 or (b.toLong() and 0xffffffffL) } diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/history/data/HistoryDao.kt b/app/src/main/kotlin/org/koitharu/kotatsu/history/data/HistoryDao.kt index e374ace60..b53d86cd7 100644 --- a/app/src/main/kotlin/org/koitharu/kotatsu/history/data/HistoryDao.kt +++ b/app/src/main/kotlin/org/koitharu/kotatsu/history/data/HistoryDao.kt @@ -78,6 +78,9 @@ abstract class HistoryDao { @Query("SELECT COUNT(*) FROM history WHERE deleted_at = 0") abstract fun observeCount(): Flow + @Query("SELECT COUNT(*) FROM history WHERE deleted_at = 0") + abstract suspend fun getCount(): Int + @Query("SELECT percent FROM history WHERE manga_id = :id AND deleted_at = 0") abstract suspend fun findProgress(id: Long): Float? diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/history/data/HistoryRepository.kt b/app/src/main/kotlin/org/koitharu/kotatsu/history/data/HistoryRepository.kt index 516d37e72..44b61d4c5 100644 --- a/app/src/main/kotlin/org/koitharu/kotatsu/history/data/HistoryRepository.kt +++ b/app/src/main/kotlin/org/koitharu/kotatsu/history/data/HistoryRepository.kt @@ -45,6 +45,10 @@ class HistoryRepository @Inject constructor( return entities.map { it.manga.toManga(it.tags.toMangaTags()) } } + suspend fun getCount(): Int { + return db.getHistoryDao().getCount() + } + suspend fun getLastOrNull(): Manga? { val entity = db.getHistoryDao().findAll(0, 1).firstOrNull() ?: return null return entity.manga.toManga(entity.tags.toMangaTags()) diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/tracker/data/TrackEntity.kt b/app/src/main/kotlin/org/koitharu/kotatsu/tracker/data/TrackEntity.kt index eb8228d13..536c60f5f 100644 --- a/app/src/main/kotlin/org/koitharu/kotatsu/tracker/data/TrackEntity.kt +++ b/app/src/main/kotlin/org/koitharu/kotatsu/tracker/data/TrackEntity.kt @@ -20,11 +20,18 @@ import org.koitharu.kotatsu.core.db.entity.MangaEntity class TrackEntity( @PrimaryKey(autoGenerate = false) @ColumnInfo(name = "manga_id") val mangaId: Long, - @get:Deprecated(message = "Should not be used", level = DeprecationLevel.WARNING) - @ColumnInfo(name = "chapters_total") val totalChapters: Int, @ColumnInfo(name = "last_chapter_id") val lastChapterId: Long, @ColumnInfo(name = "chapters_new") val newChapters: Int, - @ColumnInfo(name = "last_check") val lastCheck: Long, - @get:Deprecated(message = "Should not be used", level = DeprecationLevel.WARNING) - @ColumnInfo(name = "last_notified_id") val lastNotifiedChapterId: Long -) + @ColumnInfo(name = "last_check_time") val lastCheckTime: Long, + @ColumnInfo(name = "last_chapter_date") val lastChapterDate: Long, + @ColumnInfo(name = "last_result") val lastResult: Int, +) { + + companion object { + + const val RESULT_NONE = 0 + const val RESULT_HAS_UPDATE = 1 + const val RESULT_NO_UPDATE = 2 + const val RESULT_FAILED = 3 + } +} diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/tracker/data/TrackLogEntity.kt b/app/src/main/kotlin/org/koitharu/kotatsu/tracker/data/TrackLogEntity.kt index 1fedc4663..5f2dd4896 100644 --- a/app/src/main/kotlin/org/koitharu/kotatsu/tracker/data/TrackLogEntity.kt +++ b/app/src/main/kotlin/org/koitharu/kotatsu/tracker/data/TrackLogEntity.kt @@ -13,14 +13,14 @@ import org.koitharu.kotatsu.core.db.entity.MangaEntity entity = MangaEntity::class, parentColumns = ["manga_id"], childColumns = ["manga_id"], - onDelete = ForeignKey.CASCADE - ) - ] + onDelete = ForeignKey.CASCADE, + ), + ], ) class TrackLogEntity( @PrimaryKey(autoGenerate = true) @ColumnInfo(name = "id") val id: Long = 0L, @ColumnInfo(name = "manga_id", index = true) val mangaId: Long, @ColumnInfo(name = "chapters") val chapters: String, - @ColumnInfo(name = "created_at") val createdAt: Long = System.currentTimeMillis(), -) \ No newline at end of file + @ColumnInfo(name = "created_at") val createdAt: Long, +) diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/Tracker.kt b/app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/Tracker.kt index 689bd2ee4..912cdaa6e 100644 --- a/app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/Tracker.kt +++ b/app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/Tracker.kt @@ -56,20 +56,21 @@ class Tracker @Inject constructor( } // History if (AppSettings.TRACK_HISTORY in sources) { - val history = repository.getAllHistoryManga() - val historyTracks = repository.getTracks(history) - val channelId = if (channels.isHistoryNotificationsEnabled()) { - channels.getHistoryChannelId() - } else { - null - } - for (track in historyTracks) { - if (knownManga.add(track.manga.id)) { - result.add(TrackingItem(track, channelId)) + for (i in 0 until historyRepository.getCount() step 20) { + val history = historyRepository.getList(i, 20) + val historyTracks = repository.getTracks(history) + val channelId = if (channels.isHistoryNotificationsEnabled()) { + channels.getHistoryChannelId() + } else { + null + } + for (track in historyTracks) { + if (knownManga.add(track.manga.id)) { + result.add(TrackingItem(track, channelId)) + } } } } - result.trimToSize() return result } diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/TrackingRepository.kt b/app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/TrackingRepository.kt index f9d67f097..37ae91e16 100644 --- a/app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/TrackingRepository.kt +++ b/app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/TrackingRepository.kt @@ -15,6 +15,7 @@ import org.koitharu.kotatsu.core.db.entity.MangaEntity import org.koitharu.kotatsu.core.db.entity.toManga import org.koitharu.kotatsu.core.model.FavouriteCategory import org.koitharu.kotatsu.core.model.isLocal +import org.koitharu.kotatsu.core.util.ext.ifZero import org.koitharu.kotatsu.core.util.ext.mapItems import org.koitharu.kotatsu.favourites.data.toFavouriteCategory import org.koitharu.kotatsu.local.data.LocalMangaRepository @@ -32,7 +33,11 @@ import javax.inject.Inject import javax.inject.Provider private const val NO_ID = 0L + +@Deprecated("Use buckets") private const val MAX_QUERY_IDS = 100 +private const val MAX_BUCKET_SIZE = 20 +private const val MAX_LOG_SIZE = 120 @Reusable class TrackingRepository @Inject constructor( @@ -65,6 +70,7 @@ class TrackingRepository @Inject constructor( .onStart { gcIfNotCalled() } } + @Deprecated("") suspend fun getTracks(mangaList: Collection): List { val ids = mangaList.mapToSet { it.id } val dao = db.getTracksDao() @@ -90,7 +96,7 @@ class TrackingRepository @Inject constructor( result += MangaTracking( manga = manga, lastChapterId = track?.lastChapterId ?: NO_ID, - lastCheck = track?.lastCheck?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli), + lastCheck = track?.lastCheckTime?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli), ) } return result @@ -102,7 +108,7 @@ class TrackingRepository @Inject constructor( return MangaTracking( manga = manga, lastChapterId = track?.lastChapterId ?: NO_ID, - lastCheck = track?.lastCheck?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli), + lastCheck = track?.lastCheckTime?.takeUnless { it == 0L }?.let(Instant::ofEpochMilli), ) } @@ -131,9 +137,12 @@ class TrackingRepository @Inject constructor( suspend fun clearCounters() = db.getTracksDao().clearCounters() - suspend fun gc() { + suspend fun gc() = db.withTransaction { db.getTracksDao().gc() - db.getTrackLogsDao().gc() + db.getTrackLogsDao().run { + gc() + trim(MAX_LOG_SIZE) + } } suspend fun saveUpdates(updates: MangaUpdates.Success) { @@ -172,7 +181,6 @@ class TrackingRepository @Inject constructor( val lastChapterId = chapters.lastOrNull()?.id ?: NO_ID val entity = TrackEntity( mangaId = manga.id, - totalChapters = chapters.size, lastChapterId = lastChapterId, newChapters = when { track.newChapters == 0 -> 0 @@ -180,8 +188,9 @@ class TrackingRepository @Inject constructor( chapterIndex >= lastNewChapterIndex -> chapters.lastIndex - chapterIndex else -> track.newChapters }, - lastCheck = System.currentTimeMillis(), - lastNotifiedChapterId = lastChapterId, + lastCheckTime = System.currentTimeMillis(), + lastChapterDate = maxOf(track.lastChapterDate, chapters.lastOrNull()?.uploadDate ?: 0L), + lastResult = track.lastResult, ) db.getTracksDao().upsert(entity) } @@ -202,18 +211,14 @@ class TrackingRepository @Inject constructor( } } - suspend fun getAllHistoryManga(): List { - return db.getHistoryDao().findAllManga().toMangaList() - } - private suspend fun getOrCreateTrack(mangaId: Long): TrackEntity { return db.getTracksDao().find(mangaId) ?: TrackEntity( mangaId = mangaId, - totalChapters = 0, lastChapterId = 0L, newChapters = 0, - lastCheck = 0L, - lastNotifiedChapterId = 0L, + lastCheckTime = 0L, + lastChapterDate = 0, + lastResult = TrackEntity.RESULT_NONE, ) } @@ -236,11 +241,11 @@ class TrackingRepository @Inject constructor( val chapters = updates.manga.chapters.orEmpty() return TrackEntity( mangaId = mangaId, - totalChapters = chapters.size, lastChapterId = chapters.lastOrNull()?.id ?: NO_ID, newChapters = if (updates.isValid) newChapters + updates.newChapters.size else 0, - lastCheck = System.currentTimeMillis(), - lastNotifiedChapterId = NO_ID, + lastCheckTime = System.currentTimeMillis(), + lastChapterDate = updates.lastChapterDate().ifZero { lastChapterDate }, + lastResult = if (updates.isNotEmpty()) TrackEntity.RESULT_HAS_UPDATE else TrackEntity.RESULT_NO_UPDATE, ) } diff --git a/app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/model/MangaUpdates.kt b/app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/model/MangaUpdates.kt index 6dc13dd60..f81bd6ab9 100644 --- a/app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/model/MangaUpdates.kt +++ b/app/src/main/kotlin/org/koitharu/kotatsu/tracker/domain/model/MangaUpdates.kt @@ -1,6 +1,7 @@ package org.koitharu.kotatsu.tracker.domain.model import org.koitharu.kotatsu.core.exceptions.TooManyRequestExceptions +import org.koitharu.kotatsu.core.util.ext.ifZero import org.koitharu.kotatsu.parsers.model.Manga import org.koitharu.kotatsu.parsers.model.MangaChapter @@ -16,6 +17,15 @@ sealed interface MangaUpdates { ) : MangaUpdates { fun isNotEmpty() = newChapters.isNotEmpty() + + fun lastChapterDate(): Long { + val lastChapter = newChapters.lastOrNull() + return if (lastChapter == null) { + manga.chapters?.lastOrNull()?.uploadDate ?: 0L + } else { + lastChapter.uploadDate.ifZero { System.currentTimeMillis() } + } + } } data class Failure(