Migrate LocalMangaInfo to Okio

This commit is contained in:
Koitharu
2024-10-26 08:37:40 +03:00
parent ad0452486f
commit 9425d29596
19 changed files with 408 additions and 474 deletions

View File

@@ -82,7 +82,7 @@ afterEvaluate {
}
}
dependencies {
implementation('com.github.KotatsuApp:kotatsu-parsers:3d5cc5ceff') {
implementation('com.github.KotatsuApp:kotatsu-parsers:1.4') {
exclude group: 'org.json', module: 'json'
}

View File

@@ -19,6 +19,7 @@ import coil3.toAndroidUri
import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.runInterruptible
import okio.IOException
import org.koitharu.kotatsu.R
import org.koitharu.kotatsu.core.exceptions.CloudFlareProtectedException
import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.core.parser.EmptyMangaRepository
@@ -26,6 +27,7 @@ import org.koitharu.kotatsu.core.parser.MangaRepository
import org.koitharu.kotatsu.core.parser.ParserMangaRepository
import org.koitharu.kotatsu.core.parser.external.ExternalMangaRepository
import org.koitharu.kotatsu.core.util.ext.fetch
import org.koitharu.kotatsu.local.data.LocalMangaRepository
import kotlin.coroutines.coroutineContext
import coil3.Uri as CoilUri
@@ -36,7 +38,7 @@ class FaviconFetcher(
private val mangaRepositoryFactory: MangaRepository.Factory,
) : Fetcher {
override suspend fun fetch(): FetchResult {
override suspend fun fetch(): FetchResult? {
val mangaSource = MangaSource(uri.schemeSpecificPart)
return when (val repo = mangaRepositoryFactory.create(mangaSource)) {
@@ -48,7 +50,9 @@ class FaviconFetcher(
dataSource = DataSource.MEMORY,
)
else -> throw IllegalArgumentException("")
is LocalMangaRepository -> imageLoader.fetch(R.drawable.ic_storage, options)
else -> throw IllegalArgumentException("Unsupported repo ${repo.javaClass.simpleName}")
}
}

View File

@@ -7,6 +7,9 @@ import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.withContext
import okhttp3.ResponseBody
import okio.BufferedSink
import okio.FileSystem
import okio.IOException
import okio.Path
import okio.Source
import org.koitharu.kotatsu.core.util.CancellableSource
import org.koitharu.kotatsu.core.util.progress.ProgressResponseBody
@@ -33,3 +36,15 @@ fun InputStream.toByteBuffer(): ByteBuffer {
val bytes = outStream.toByteArray()
return ByteBuffer.allocateDirect(bytes.size).put(bytes).position(0) as ByteBuffer
}
fun FileSystem.isDirectory(path: Path) = try {
metadataOrNull(path)?.isDirectory == true
} catch (_: IOException) {
false
}
fun FileSystem.isRegularFile(path: Path) = try {
metadataOrNull(path)?.isRegularFile == true
} catch (_: IOException) {
false
}

View File

@@ -1,6 +1,8 @@
package org.koitharu.kotatsu.core.util.ext
import android.net.Uri
import androidx.core.net.toUri
import okio.Path
import java.io.File
const val URI_SCHEME_ZIP = "file+zip"
@@ -20,6 +22,17 @@ fun Uri.isNetworkUri() = scheme.let {
it == URI_SCHEME_HTTP || it == URI_SCHEME_HTTPS
}
fun File.toZipUri(entryName: String): Uri = Uri.parse("$URI_SCHEME_ZIP://$absolutePath#$entryName")
fun File.toZipUri(entryPath: String): Uri = Uri.parse("$URI_SCHEME_ZIP://$absolutePath#$entryPath")
fun File.toZipUri(entryPath: Path?): Uri =
toZipUri(entryPath?.toString()?.removePrefix(Path.DIRECTORY_SEPARATOR).orEmpty())
fun String.toUriOrNull() = if (isEmpty()) null else Uri.parse(this)
fun File.toUri(fragment: String?): Uri = toUri().run {
if (fragment != null) {
buildUpon().fragment(fragment).build()
} else {
this
}
}

View File

@@ -71,7 +71,7 @@ import org.koitharu.kotatsu.local.data.LocalMangaRepository
import org.koitharu.kotatsu.local.data.LocalStorageChanges
import org.koitharu.kotatsu.local.data.PagesCache
import org.koitharu.kotatsu.local.data.TempFileFilter
import org.koitharu.kotatsu.local.data.input.LocalMangaInput
import org.koitharu.kotatsu.local.data.input.LocalMangaParser
import org.koitharu.kotatsu.local.data.output.LocalMangaOutput
import org.koitharu.kotatsu.local.domain.MangaLock
import org.koitharu.kotatsu.local.domain.model.LocalManga
@@ -262,7 +262,7 @@ class DownloadWorker @AssistedInject constructor(
}
if (output.flushChapter(chapter.value)) {
runCatchingCancellable {
localStorageChanges.emit(LocalMangaInput.of(output.rootFile).getManga())
localStorageChanges.emit(LocalMangaParser(output.rootFile).getManga(withDetails = false))
}.onFailure(Throwable::printStackTraceDebug)
}
publishState(currentState.copy(downloadedChapters = currentState.downloadedChapters + 1))
@@ -270,7 +270,7 @@ class DownloadWorker @AssistedInject constructor(
publishState(currentState.copy(isIndeterminate = true, eta = -1L, isStuck = false))
output.mergeWithExisting()
output.finish()
val localManga = LocalMangaInput.of(output.rootFile).getManga()
val localManga = LocalMangaParser(output.rootFile).getManga(withDetails = false)
localStorageChanges.emit(localManga)
publishState(currentState.copy(localManga = localManga, eta = -1L, isStuck = false))
} catch (e: Exception) {

View File

@@ -2,13 +2,14 @@ package org.koitharu.kotatsu.local.data
import java.io.File
private fun isCbzExtension(ext: String?): Boolean {
private fun isZipExtension(ext: String?): Boolean {
return ext.equals("cbz", ignoreCase = true) || ext.equals("zip", ignoreCase = true)
}
fun hasCbzExtension(string: String): Boolean {
fun hasZipExtension(string: String): Boolean {
val ext = string.substringAfterLast('.', "")
return isCbzExtension(ext)
return isZipExtension(ext)
}
fun File.hasCbzExtension() = isCbzExtension(extension)
val File.isZipArchive: Boolean
get() = isFile && isZipExtension(extension)

View File

@@ -2,6 +2,7 @@ package org.koitharu.kotatsu.local.data
import android.net.Uri
import androidx.core.net.toFile
import androidx.core.net.toUri
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.MutableSharedFlow
@@ -19,7 +20,7 @@ import org.koitharu.kotatsu.core.util.ext.deleteAwait
import org.koitharu.kotatsu.core.util.ext.printStackTraceDebug
import org.koitharu.kotatsu.core.util.ext.withChildren
import org.koitharu.kotatsu.local.data.index.LocalMangaIndex
import org.koitharu.kotatsu.local.data.input.LocalMangaInput
import org.koitharu.kotatsu.local.data.input.LocalMangaParser
import org.koitharu.kotatsu.local.data.output.LocalMangaOutput
import org.koitharu.kotatsu.local.data.output.LocalMangaUtil
import org.koitharu.kotatsu.local.domain.MangaLock
@@ -125,15 +126,15 @@ class LocalMangaRepository @Inject constructor(
}
override suspend fun getDetails(manga: Manga): Manga = when {
!manga.isLocal -> requireNotNull(findSavedManga(manga)?.manga) {
!manga.isLocal -> requireNotNull(findSavedManga(manga, withDetails = true)?.manga) {
"Manga is not local or saved"
}
else -> LocalMangaInput.of(manga).getManga().manga
else -> LocalMangaParser(manga.url.toUri()).getManga(withDetails = true).manga
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
return LocalMangaInput.of(chapter).getPages(chapter)
return LocalMangaParser(chapter.url.toUri()).getPages(chapter)
}
suspend fun delete(manga: Manga): Boolean {
@@ -147,7 +148,7 @@ class LocalMangaRepository @Inject constructor(
}
suspend fun deleteChapters(manga: Manga, ids: Set<Long>) = lock.withLock(manga) {
val subject = if (manga.isLocal) manga else checkNotNull(findSavedManga(manga)) {
val subject = if (manga.isLocal) manga else checkNotNull(findSavedManga(manga, withDetails = false)) {
"Manga is not stored on local storage"
}.manga
LocalMangaUtil(subject).deleteChapters(ids)
@@ -156,27 +157,27 @@ class LocalMangaRepository @Inject constructor(
suspend fun getRemoteManga(localManga: Manga): Manga? {
return runCatchingCancellable {
LocalMangaInput.of(localManga).getMangaInfo()?.takeUnless { it.isLocal }
LocalMangaParser(localManga.url.toUri()).getMangaInfo()?.takeUnless { it.isLocal }
}.onFailure {
it.printStackTraceDebug()
}.getOrNull()
}
suspend fun findSavedManga(remoteManga: Manga): LocalManga? = runCatchingCancellable {
suspend fun findSavedManga(remoteManga: Manga, withDetails: Boolean = true): LocalManga? = runCatchingCancellable {
// very fast path
localMangaIndex.get(remoteManga.id)?.let {
return@runCatchingCancellable it
localMangaIndex.get(remoteManga.id, withDetails)?.let { cached ->
return@runCatchingCancellable cached
}
// fast path
LocalMangaInput.find(storageManager.getReadableDirs(), remoteManga)?.let {
return it.getManga()
LocalMangaParser.find(storageManager.getReadableDirs(), remoteManga)?.let {
return it.getManga(withDetails)
}
// slow path
val files = getAllFiles()
return channelFlow {
for (file in files) {
launch {
val mangaInput = LocalMangaInput.ofOrNull(file)
val mangaInput = LocalMangaParser.getOrNull(file)
runCatchingCancellable {
val mangaInfo = mangaInput?.getMangaInfo()
if (mangaInfo != null && mangaInfo.id == remoteManga.id) {
@@ -187,7 +188,7 @@ class LocalMangaRepository @Inject constructor(
}
}
}
}.firstOrNull()?.getManga()
}.firstOrNull()?.getManga(withDetails)
}.onSuccess { x: LocalManga? ->
if (x != null) {
localMangaIndex.put(x)
@@ -237,7 +238,7 @@ class LocalMangaRepository @Inject constructor(
for (file in files) {
launch(dispatcher) {
runCatchingCancellable {
LocalMangaInput.ofOrNull(file)?.getManga()
LocalMangaParser.getOrNull(file)?.getManga(withDetails = false)
}.onFailure { e ->
e.printStackTraceDebug()
}.onSuccess { m ->

View File

@@ -1,11 +1,17 @@
package org.koitharu.kotatsu.local.data
import androidx.annotation.WorkerThread
import okio.FileSystem
import okio.Path
import okio.Path.Companion.toOkioPath
import okio.buffer
import org.jetbrains.annotations.Blocking
import org.json.JSONArray
import org.json.JSONObject
import org.koitharu.kotatsu.BuildConfig
import org.koitharu.kotatsu.core.model.MangaSource
import org.koitharu.kotatsu.core.model.isLocal
import org.koitharu.kotatsu.core.util.ext.printStackTraceDebug
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaSource
@@ -18,6 +24,7 @@ import org.koitharu.kotatsu.parsers.util.json.getIntOrDefault
import org.koitharu.kotatsu.parsers.util.json.getLongOrDefault
import org.koitharu.kotatsu.parsers.util.json.getStringOrNull
import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet
import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
import org.koitharu.kotatsu.parsers.util.toTitleCase
import java.io.File
@@ -186,15 +193,25 @@ class MangaIndex(source: String?) {
companion object {
@Blocking
@WorkerThread
fun read(file: File): MangaIndex? {
if (file.exists() && file.canRead()) {
val text = file.readText()
if (text.length > 2) {
return MangaIndex(text)
fun read(fileSystem: FileSystem, path: Path): MangaIndex? = runCatchingCancellable {
val text = fileSystem.source(path).use {
it.buffer().use { buffer ->
buffer.readUtf8()
}
}
return null
}
if (text.length > 2) {
MangaIndex(text)
} else {
null
}
}.onFailure { e ->
e.printStackTraceDebug()
}.getOrNull()
@Blocking
@WorkerThread
fun read(file: File): MangaIndex? = read(FileSystem.SYSTEM, file.toOkioPath())
}
}

View File

@@ -17,8 +17,8 @@ import org.koitharu.kotatsu.core.util.ext.resolveName
import org.koitharu.kotatsu.core.util.ext.writeAllCancellable
import org.koitharu.kotatsu.local.data.LocalStorageChanges
import org.koitharu.kotatsu.local.data.LocalStorageManager
import org.koitharu.kotatsu.local.data.hasCbzExtension
import org.koitharu.kotatsu.local.data.input.LocalMangaInput
import org.koitharu.kotatsu.local.data.hasZipExtension
import org.koitharu.kotatsu.local.data.input.LocalMangaParser
import org.koitharu.kotatsu.local.domain.model.LocalManga
import java.io.File
import java.io.IOException
@@ -46,7 +46,7 @@ class SingleMangaImporter @Inject constructor(
private suspend fun importFile(uri: Uri): LocalManga = withContext(Dispatchers.IO) {
val contentResolver = storageManager.contentResolver
val name = contentResolver.resolveName(uri) ?: throw IOException("Cannot fetch name from uri: $uri")
if (!hasCbzExtension(name)) {
if (!hasZipExtension(name)) {
throw UnsupportedFileException("Unsupported file $name on $uri")
}
val dest = File(getOutputDir(), name)
@@ -57,7 +57,7 @@ class SingleMangaImporter @Inject constructor(
output.writeAllCancellable(source.source())
}
} ?: throw IOException("Cannot open input stream: $uri")
LocalMangaInput.of(dest).getManga()
LocalMangaParser(dest).getManga(withDetails = false)
}
private suspend fun importDirectory(uri: Uri): LocalManga {
@@ -69,7 +69,7 @@ class SingleMangaImporter @Inject constructor(
for (docFile in root.listFiles()) {
docFile.copyTo(dest)
}
return LocalMangaInput.of(dest).getManga()
return LocalMangaParser(dest).getManga(withDetails = false)
}
private suspend fun DocumentFile.copyTo(destDir: File) {

View File

@@ -11,7 +11,7 @@ import org.koitharu.kotatsu.core.db.MangaDatabase
import org.koitharu.kotatsu.core.parser.MangaDataRepository
import org.koitharu.kotatsu.core.util.ext.printStackTraceDebug
import org.koitharu.kotatsu.local.data.LocalMangaRepository
import org.koitharu.kotatsu.local.data.input.LocalMangaInput
import org.koitharu.kotatsu.local.data.input.LocalMangaParser
import org.koitharu.kotatsu.local.domain.model.LocalManga
import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
import java.io.File
@@ -57,7 +57,7 @@ class LocalMangaIndex @Inject constructor(
}
}
suspend fun get(mangaId: Long): LocalManga? {
suspend fun get(mangaId: Long, withDetails: Boolean): LocalManga? {
updateIfRequired()
var path = db.getLocalMangaIndexDao().findPath(mangaId)
if (path == null && mutex.isLocked) { // wait for updating complete
@@ -67,7 +67,7 @@ class LocalMangaIndex @Inject constructor(
return null
}
return runCatchingCancellable {
LocalMangaInput.of(File(path)).getManga()
LocalMangaParser(File(path)).getManga(withDetails)
}.onFailure {
it.printStackTraceDebug()
}.getOrNull()

View File

@@ -1,159 +0,0 @@
package org.koitharu.kotatsu.local.data.input
import androidx.core.net.toFile
import androidx.core.net.toUri
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runInterruptible
import org.koitharu.kotatsu.core.model.LocalMangaSource
import org.koitharu.kotatsu.core.util.AlphanumComparator
import org.koitharu.kotatsu.core.util.ext.creationTime
import org.koitharu.kotatsu.core.util.ext.longHashCode
import org.koitharu.kotatsu.core.util.ext.toListSorted
import org.koitharu.kotatsu.core.util.ext.walkCompat
import org.koitharu.kotatsu.core.util.ext.withChildren
import org.koitharu.kotatsu.local.data.MangaIndex
import org.koitharu.kotatsu.local.data.hasCbzExtension
import org.koitharu.kotatsu.local.data.hasImageExtension
import org.koitharu.kotatsu.local.data.output.LocalMangaOutput
import org.koitharu.kotatsu.local.domain.model.LocalManga
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaPage
import org.koitharu.kotatsu.parsers.util.toCamelCase
import java.io.File
import java.util.TreeMap
import java.util.zip.ZipFile
/**
* Manga {Folder}
* |--- index.json (optional)
* |--- Chapter 1.cbz
* |--- Page 1.png
* :
* L--- Page x.png
* |--- Chapter 2.cbz
* :
* L--- Chapter x.cbz
*/
class LocalMangaDirInput(root: File) : LocalMangaInput(root) {
override suspend fun getManga(): LocalManga = runInterruptible(Dispatchers.IO) {
val index = MangaIndex.read(File(root, LocalMangaOutput.ENTRY_NAME_INDEX))
val mangaUri = root.toUri().toString()
val chapterFiles = getChaptersFiles()
val info = index?.getMangaInfo()
val cover = fileUri(
root,
index?.getCoverEntry() ?: findFirstImageEntry().orEmpty(),
)
val manga = info?.copy2(
source = LocalMangaSource,
url = mangaUri,
coverUrl = cover,
largeCoverUrl = cover,
chapters = info.chapters?.mapIndexedNotNull { i, c ->
val fileName = index.getChapterFileName(c.id)
val file = if (fileName != null) {
chapterFiles[fileName]
} else {
// old downloads
chapterFiles.values.elementAtOrNull(i)
} ?: return@mapIndexedNotNull null
c.copy(url = file.toUri().toString(), source = LocalMangaSource)
},
) ?: Manga(
id = root.absolutePath.longHashCode(),
title = root.name.toHumanReadable(),
url = mangaUri,
publicUrl = mangaUri,
source = LocalMangaSource,
coverUrl = findFirstImageEntry().orEmpty(),
chapters = chapterFiles.values.mapIndexed { i, f ->
MangaChapter(
id = "$i${f.name}".longHashCode(),
name = f.nameWithoutExtension.toHumanReadable(),
number = 0f,
volume = 0,
source = LocalMangaSource,
uploadDate = f.creationTime,
url = f.toUri().toString(),
scanlator = null,
branch = null,
)
},
altTitle = null,
rating = -1f,
isNsfw = false,
tags = setOf(),
state = null,
author = null,
largeCoverUrl = null,
description = null,
)
LocalManga(manga, root)
}
override suspend fun getMangaInfo(): Manga? = runInterruptible(Dispatchers.IO) {
val index = MangaIndex.read(File(root, LocalMangaOutput.ENTRY_NAME_INDEX))
index?.getMangaInfo()
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> = runInterruptible(Dispatchers.IO) {
val file = chapter.url.toUri().toFile()
if (file.isDirectory) {
file.withChildren { children ->
children
.filter { it.isFile && hasImageExtension(it) }
.toListSorted(compareBy(AlphanumComparator()) { x -> x.name })
}.map {
val pageUri = it.toUri().toString()
MangaPage(pageUri.longHashCode(), pageUri, null, LocalMangaSource)
}
} else {
ZipFile(file).use { zip ->
zip.entries()
.asSequence()
.filter { x -> !x.isDirectory }
.map { it.name }
.toListSorted(AlphanumComparator())
.map {
val pageUri = zipUri(file, it)
MangaPage(
id = pageUri.longHashCode(),
url = pageUri,
preview = null,
source = LocalMangaSource,
)
}
}
}
}
private fun String.toHumanReadable() = replace("_", " ").toCamelCase()
private fun getChaptersFiles() = root.walkCompat(includeDirectories = true)
.filter { it != root && it.isChapterDirectory() || it.hasCbzExtension() }
.associateByTo(TreeMap(AlphanumComparator())) { it.name }
private fun findFirstImageEntry(): String? {
return root.walkCompat(includeDirectories = false)
.firstOrNull { hasImageExtension(it) }?.toUri()?.toString()
?: run {
val cbz = root.walkCompat(includeDirectories = false)
.firstOrNull { it.hasCbzExtension() } ?: return null
ZipFile(cbz).use { zip ->
zip.entries().asSequence()
.firstOrNull { !it.isDirectory && hasImageExtension(it.name) }
?.let { zipUri(cbz, it.name) }
}
}
}
private fun fileUri(base: File, name: String): String {
return File(base, name).toUri().toString()
}
private fun File.isChapterDirectory(): Boolean {
return isDirectory && withChildren { children -> children.any { hasImageExtension(it) } }
}
}

View File

@@ -1,111 +0,0 @@
package org.koitharu.kotatsu.local.data.input
import android.net.Uri
import androidx.core.net.toFile
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.channelFlow
import kotlinx.coroutines.flow.firstOrNull
import kotlinx.coroutines.flow.flowOn
import kotlinx.coroutines.launch
import org.koitharu.kotatsu.core.util.ext.toZipUri
import org.koitharu.kotatsu.local.data.hasCbzExtension
import org.koitharu.kotatsu.local.domain.model.LocalManga
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaPage
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
import org.koitharu.kotatsu.parsers.util.toFileNameSafe
import java.io.File
sealed class LocalMangaInput(
protected val root: File,
) {
abstract suspend fun getManga(): LocalManga
abstract suspend fun getMangaInfo(): Manga?
abstract suspend fun getPages(chapter: MangaChapter): List<MangaPage>
companion object {
fun of(manga: Manga): LocalMangaInput = of(Uri.parse(manga.url).toFile())
fun of(chapter: MangaChapter): LocalMangaInput = of(Uri.parse(chapter.url).toFile())
fun of(file: File): LocalMangaInput = when {
file.isDirectory -> LocalMangaDirInput(file)
else -> LocalMangaZipInput(file)
}
fun ofOrNull(file: File): LocalMangaInput? = when {
file.isDirectory -> LocalMangaDirInput(file)
hasCbzExtension(file.name) -> LocalMangaZipInput(file)
else -> null
}
suspend fun find(roots: Iterable<File>, manga: Manga): LocalMangaInput? = channelFlow {
val fileName = manga.title.toFileNameSafe()
for (root in roots) {
launch {
val dir = File(root, fileName)
val zip = File(root, "$fileName.cbz")
val input = when {
dir.isDirectory -> LocalMangaDirInput(dir)
zip.isFile -> LocalMangaZipInput(zip)
else -> null
}
val info = runCatchingCancellable { input?.getMangaInfo() }.getOrNull()
if (info?.id == manga.id) {
send(input)
}
}
}
}.flowOn(Dispatchers.Default).firstOrNull()
@JvmStatic
protected fun zipUri(file: File, entryName: String): String = file.toZipUri(entryName).toString()
@JvmStatic
protected fun Manga.copy2(
url: String,
coverUrl: String,
largeCoverUrl: String,
chapters: List<MangaChapter>?,
source: MangaSource,
) = Manga(
id = id,
title = title,
altTitle = altTitle,
url = url,
publicUrl = publicUrl,
rating = rating,
isNsfw = isNsfw,
coverUrl = coverUrl,
tags = tags,
state = state,
author = author,
largeCoverUrl = largeCoverUrl,
description = description,
chapters = chapters,
source = source,
)
@JvmStatic
protected fun MangaChapter.copy(
url: String,
source: MangaSource,
) = MangaChapter(
id = id,
name = name,
number = number,
volume = volume,
url = url,
scanlator = scanlator,
uploadDate = uploadDate,
branch = branch,
source = source,
)
}
}

View File

@@ -0,0 +1,309 @@
package org.koitharu.kotatsu.local.data.input
import android.net.Uri
import android.webkit.MimeTypeMap
import androidx.core.net.toFile
import androidx.core.net.toUri
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.channelFlow
import kotlinx.coroutines.flow.firstOrNull
import kotlinx.coroutines.flow.flowOn
import kotlinx.coroutines.launch
import kotlinx.coroutines.runInterruptible
import okio.FileSystem
import okio.Path
import okio.Path.Companion.toOkioPath
import okio.Path.Companion.toPath
import okio.openZip
import org.jetbrains.annotations.Blocking
import org.koitharu.kotatsu.core.model.LocalMangaSource
import org.koitharu.kotatsu.core.util.AlphanumComparator
import org.koitharu.kotatsu.core.util.ext.URI_SCHEME_ZIP
import org.koitharu.kotatsu.core.util.ext.isFileUri
import org.koitharu.kotatsu.core.util.ext.isRegularFile
import org.koitharu.kotatsu.core.util.ext.isZipUri
import org.koitharu.kotatsu.core.util.ext.longHashCode
import org.koitharu.kotatsu.core.util.ext.printStackTraceDebug
import org.koitharu.kotatsu.core.util.ext.toListSorted
import org.koitharu.kotatsu.local.data.MangaIndex
import org.koitharu.kotatsu.local.data.isZipArchive
import org.koitharu.kotatsu.local.data.output.LocalMangaOutput.Companion.ENTRY_NAME_INDEX
import org.koitharu.kotatsu.local.domain.model.LocalManga
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaPage
import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
import org.koitharu.kotatsu.parsers.util.toCamelCase
import org.koitharu.kotatsu.parsers.util.toFileNameSafe
import java.io.File
/**
* Manga root {dir or zip file}
* |--- index.json (optional)
* |--- Page 1.png
* |--- Page 2.png
* |---Chapter 1/(dir or zip, optional)
* |------Page 1.1.png
* :
* L--- Page x.png
*/
class LocalMangaParser(private val uri: Uri) {
constructor(file: File) : this(file.toUri())
private val rootFile: File = File(uri.schemeSpecificPart)
suspend fun getManga(withDetails: Boolean): LocalManga = runInterruptible(Dispatchers.IO) {
val (fileSystem, rootPath) = uri.resolveFsAndPath()
val index = MangaIndex.read(fileSystem, rootPath / ENTRY_NAME_INDEX)
val mangaInfo = index?.getMangaInfo()
if (mangaInfo != null) {
val coverEntry: Path? = index.getCoverEntry()?.let { rootPath / it } ?: fileSystem.findFirstImage(rootPath)
mangaInfo.copyInternal(
source = LocalMangaSource,
url = rootFile.toUri().toString(),
coverUrl = coverEntry?.let { uri.child(it, resolve = true).toString() }.orEmpty(),
largeCoverUrl = null,
chapters = if (withDetails) {
mangaInfo.chapters?.map { c ->
c.copyInternal(
url = index.getChapterFileName(c.id)?.toPath()?.let {
uri.child(it, resolve = false).toString()
} ?: uri.toString(),
source = LocalMangaSource,
)
}
} else {
null
},
)
} else {
val title = rootFile.nameWithoutExtension.replace("_", " ").toCamelCase()
val coverEntry = fileSystem.findFirstImage(rootPath)
val mimeTypeMap = MimeTypeMap.getSingleton()
Manga(
id = rootFile.absolutePath.longHashCode(),
title = title,
url = rootFile.toUri().toString(),
publicUrl = rootFile.toUri().toString(),
source = LocalMangaSource,
coverUrl = coverEntry?.let {
uri.child(it, resolve = true).toString()
}.orEmpty(),
chapters = if (withDetails) {
val chapters = fileSystem.listRecursively(rootPath)
.mapNotNullTo(HashSet()) { path ->
if (path != coverEntry && fileSystem.isRegularFile(path) && mimeTypeMap.isImage(path)) {
path.parent
} else {
null
}
}.sortedWith(compareBy(AlphanumComparator()) { x -> x.toString() })
chapters.mapIndexed { i, p ->
val s = if (p.root == rootPath.root) {
p.relativeTo(rootPath).toString()
} else {
p
}.toString().removePrefix(Path.DIRECTORY_SEPARATOR)
MangaChapter(
id = "$i$s".longHashCode(),
name = s.ifEmpty { title },
number = 0f,
volume = 0,
source = LocalMangaSource,
uploadDate = 0L,
url = uri.child(p.relativeTo(rootPath), resolve = false).toString(),
scanlator = null,
branch = null,
)
}
} else {
null
},
altTitle = null,
rating = -1f,
isNsfw = false,
tags = setOf(),
state = null,
author = null,
largeCoverUrl = null,
description = null,
)
}.let { LocalManga(it, rootFile) }
}
suspend fun getMangaInfo(): Manga? = runInterruptible(Dispatchers.IO) {
val (fileSystem, rootPath) = uri.resolveFsAndPath()
val index = MangaIndex.read(fileSystem, rootPath / ENTRY_NAME_INDEX)
index?.getMangaInfo()
}
suspend fun getPages(chapter: MangaChapter): List<MangaPage> = runInterruptible(Dispatchers.IO) {
val chapterUri = chapter.url.toUri().resolve()
val (fileSystem, rootPath) = chapterUri.resolveFsAndPath()
val index = MangaIndex.read(fileSystem, rootPath / ENTRY_NAME_INDEX)
val entries = fileSystem.listRecursively(rootPath)
.filter { fileSystem.isRegularFile(it) }
if (index != null) {
val pattern = index.getChapterNamesPattern(chapter)
entries.filter { x -> x.name.substringBefore('.').matches(pattern) }
} else {
val mimeTypeMap = MimeTypeMap.getSingleton()
entries.filter { x ->
mimeTypeMap.isImage(x) && x.parent == rootPath
}
}.toListSorted(compareBy(AlphanumComparator()) { x -> x.toString() })
.map { x ->
val entryUri = chapterUri.child(x, resolve = true).toString()
MangaPage(
id = entryUri.longHashCode(),
url = entryUri,
preview = null,
source = LocalMangaSource,
)
}
}
private fun Uri.child(path: Path, resolve: Boolean): Uri {
val builder = buildUpon()
if (isZipUri() || !resolve) {
builder.fragment(path.toString().removePrefix(Path.DIRECTORY_SEPARATOR))
} else {
val file = toFile()
if (file.isZipArchive) {
builder.fragment(path.toString().removePrefix(Path.DIRECTORY_SEPARATOR))
builder.scheme(URI_SCHEME_ZIP)
} else {
builder.appendEncodedPath(path.relativeTo(file.toOkioPath()).toString())
}
}
return builder.build()
}
companion object {
@Blocking
fun getOrNull(file: File): LocalMangaParser? = if ((file.isDirectory || file.isZipArchive) && file.canRead()) {
LocalMangaParser(file)
} else {
null
}
suspend fun find(roots: Iterable<File>, manga: Manga): LocalMangaParser? = channelFlow {
val fileName = manga.title.toFileNameSafe()
for (root in roots) {
launch {
val parser = getOrNull(File(root, fileName)) ?: getOrNull(File(root, "$fileName.cbz"))
val info = runCatchingCancellable { parser?.getMangaInfo() }.getOrNull()
if (info?.id == manga.id) {
send(parser)
}
}
}
}.flowOn(Dispatchers.Default).firstOrNull()
private fun FileSystem.findFirstImage(rootPath: Path) = findFirstImageImpl(rootPath, false)
?: findFirstImageImpl(rootPath, true)
private fun FileSystem.findFirstImageImpl(
rootPath: Path,
recursive: Boolean
): Path? = runCatchingCancellable {
val mimeTypeMap = MimeTypeMap.getSingleton()
if (recursive) {
listRecursively(rootPath)
} else {
list(rootPath).asSequence()
}.filter { isRegularFile(it) && mimeTypeMap.isImage(it) }
.toListSorted(compareBy(AlphanumComparator()) { x -> x.toString() })
.firstOrNull()
}.onFailure { e ->
e.printStackTraceDebug()
}.getOrNull()
private fun MimeTypeMap.isImage(path: Path): Boolean =
getMimeTypeFromExtension(path.name.substringAfterLast('.'))
?.startsWith("image/") == true
private fun Uri.resolve(): Uri = if (isFileUri()) {
val file = toFile()
if (file.isZipArchive) {
this
} else if (file.isDirectory) {
file.resolve(fragment.orEmpty()).toUri()
} else {
this
}
} else {
this
}
@Blocking
private fun Uri.resolveFsAndPath(): Pair<FileSystem, Path> {
val resolved = resolve()
return when {
resolved.isZipUri() -> {
FileSystem.SYSTEM.openZip(resolved.schemeSpecificPart.toPath()) to resolved.fragment.orEmpty()
.toRootedPath()
}
isFileUri() -> {
val file = toFile()
if (file.isZipArchive) {
FileSystem.SYSTEM.openZip(schemeSpecificPart.toPath()) to fragment.orEmpty().toRootedPath()
} else {
FileSystem.SYSTEM to file.toOkioPath()
}
}
else -> throw IllegalArgumentException("Unsupported uri $resolved")
}
}
private fun String.toRootedPath(): Path = if (startsWith(Path.DIRECTORY_SEPARATOR)) {
this
} else {
Path.DIRECTORY_SEPARATOR + this
}.toPath()
private fun Manga.copyInternal(
url: String = this.url,
coverUrl: String = this.coverUrl,
largeCoverUrl: String? = this.largeCoverUrl,
chapters: List<MangaChapter>? = this.chapters,
source: MangaSource = this.source,
): Manga = Manga(
id = id,
title = title,
altTitle = altTitle,
url = url,
publicUrl = publicUrl,
rating = rating,
isNsfw = isNsfw,
coverUrl = coverUrl,
tags = tags,
state = state,
author = author,
largeCoverUrl = largeCoverUrl,
description = description,
chapters = chapters,
source = source,
)
private fun MangaChapter.copyInternal(
url: String = this.url,
source: MangaSource = this.source,
) = MangaChapter(
id = id,
name = name,
number = number,
volume = volume,
url = url,
scanlator = scanlator,
uploadDate = uploadDate,
branch = branch,
source = source,
)
}
}

View File

@@ -1,155 +0,0 @@
package org.koitharu.kotatsu.local.data.input
import android.net.Uri
import android.webkit.MimeTypeMap
import androidx.collection.ArraySet
import androidx.core.net.toFile
import androidx.core.net.toUri
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runInterruptible
import org.koitharu.kotatsu.core.model.LocalMangaSource
import org.koitharu.kotatsu.core.util.AlphanumComparator
import org.koitharu.kotatsu.core.util.ext.longHashCode
import org.koitharu.kotatsu.core.util.ext.readText
import org.koitharu.kotatsu.core.util.ext.toListSorted
import org.koitharu.kotatsu.local.data.MangaIndex
import org.koitharu.kotatsu.local.data.output.LocalMangaOutput
import org.koitharu.kotatsu.local.domain.model.LocalManga
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.model.MangaPage
import org.koitharu.kotatsu.parsers.util.toCamelCase
import java.io.File
import java.util.Enumeration
import java.util.zip.ZipEntry
import java.util.zip.ZipFile
/**
* Manga archive {.cbz or .zip file}
* |--- index.json (optional)
* |--- Page 1.png
* |--- Page 2.png
* :
* L--- Page x.png
*/
class LocalMangaZipInput(root: File) : LocalMangaInput(root) {
override suspend fun getManga(): LocalManga {
val manga = runInterruptible(Dispatchers.IO) {
ZipFile(root).use { zip ->
val fileUri = root.toUri().toString()
val entry = zip.getEntry(LocalMangaOutput.ENTRY_NAME_INDEX)
val index = entry?.let(zip::readText)?.let(::MangaIndex)
val info = index?.getMangaInfo()
if (info != null) {
val cover = zipUri(
root,
entryName = index.getCoverEntry() ?: findFirstImageEntry(zip.entries())?.name.orEmpty(),
)
return@use info.copy2(
source = LocalMangaSource,
url = fileUri,
coverUrl = cover,
largeCoverUrl = cover,
chapters = info.chapters?.map { c ->
c.copy(url = fileUri, source = LocalMangaSource)
},
)
}
// fallback
val title = root.nameWithoutExtension.replace("_", " ").toCamelCase()
val chapters = ArraySet<String>()
for (x in zip.entries()) {
if (!x.isDirectory) {
chapters += x.name.substringBeforeLast(File.separatorChar, "")
}
}
val uriBuilder = root.toUri().buildUpon()
Manga(
id = root.absolutePath.longHashCode(),
title = title,
url = fileUri,
publicUrl = fileUri,
source = LocalMangaSource,
coverUrl = zipUri(root, findFirstImageEntry(zip.entries())?.name.orEmpty()),
chapters = chapters.sortedWith(AlphanumComparator())
.mapIndexed { i, s ->
MangaChapter(
id = "$i$s".longHashCode(),
name = s.ifEmpty { title },
number = 0f,
volume = 0,
source = LocalMangaSource,
uploadDate = 0L,
url = uriBuilder.fragment(s).build().toString(),
scanlator = null,
branch = null,
)
},
altTitle = null,
rating = -1f,
isNsfw = false,
tags = setOf(),
state = null,
author = null,
largeCoverUrl = null,
description = null,
)
}
}
return LocalManga(manga, root)
}
override suspend fun getMangaInfo(): Manga? = runInterruptible(Dispatchers.IO) {
ZipFile(root).use { zip ->
val entry = zip.getEntry(LocalMangaOutput.ENTRY_NAME_INDEX)
val index = entry?.let(zip::readText)?.let(::MangaIndex)
index?.getMangaInfo()
}
}
override suspend fun getPages(chapter: MangaChapter): List<MangaPage> {
return runInterruptible(Dispatchers.IO) {
val uri = Uri.parse(chapter.url)
val file = uri.toFile()
ZipFile(file).use { zip ->
val index = zip.getEntry(LocalMangaOutput.ENTRY_NAME_INDEX)?.let(zip::readText)?.let(::MangaIndex)
var entries = zip.entries().asSequence()
entries = if (index != null) {
val pattern = index.getChapterNamesPattern(chapter)
entries.filter { x -> !x.isDirectory && x.name.substringBefore('.').matches(pattern) }
} else {
val parent = uri.fragment.orEmpty()
entries.filter { x ->
!x.isDirectory && x.name.substringBeforeLast(
File.separatorChar,
"",
) == parent
}
}
entries
.toListSorted(compareBy(AlphanumComparator()) { x -> x.name })
.map { x ->
val entryUri = zipUri(file, x.name)
MangaPage(
id = entryUri.longHashCode(),
url = entryUri,
preview = null,
source = LocalMangaSource,
)
}
}
}
}
private fun findFirstImageEntry(entries: Enumeration<out ZipEntry>): ZipEntry? {
val list = entries.toList()
.filterNot { it.isDirectory }
.sortedWith(compareBy(AlphanumComparator()) { x -> x.name })
val map = MimeTypeMap.getSingleton()
return list.firstOrNull {
map.getMimeTypeFromExtension(it.name.substringAfterLast('.'))
?.startsWith("image/") == true
}
}
}

View File

@@ -12,7 +12,7 @@ import org.koitharu.kotatsu.core.util.ext.deleteAwait
import org.koitharu.kotatsu.core.util.ext.takeIfReadable
import org.koitharu.kotatsu.core.zip.ZipOutput
import org.koitharu.kotatsu.local.data.MangaIndex
import org.koitharu.kotatsu.local.data.input.LocalMangaDirInput
import org.koitharu.kotatsu.local.data.input.LocalMangaParser
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.util.toFileNameSafe
@@ -96,7 +96,7 @@ class LocalMangaDirOutput(
}
suspend fun deleteChapters(ids: Set<Long>) = mutex.withLock {
val chapters = checkNotNull((index.getMangaInfo() ?: LocalMangaDirInput(rootFile).getManga().manga).chapters) {
val chapters = checkNotNull((index.getMangaInfo() ?: LocalMangaParser(rootFile).getManga(withDetails = true).manga).chapters) {
"No chapters found"
}.withIndex()
val victimsIds = ids.toMutableSet()

View File

@@ -7,7 +7,7 @@ import kotlinx.coroutines.withContext
import okio.Closeable
import org.koitharu.kotatsu.core.prefs.DownloadFormat
import org.koitharu.kotatsu.core.util.ext.printStackTraceDebug
import org.koitharu.kotatsu.local.data.input.LocalMangaInput
import org.koitharu.kotatsu.local.data.input.LocalMangaParser
import org.koitharu.kotatsu.parsers.model.Manga
import org.koitharu.kotatsu.parsers.model.MangaChapter
import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
@@ -100,7 +100,7 @@ sealed class LocalMangaOutput(
private suspend fun canWriteTo(file: File, manga: Manga): Boolean {
val info = runCatchingCancellable {
LocalMangaInput.of(file).getMangaInfo()
LocalMangaParser(file).getMangaInfo()
}.onFailure {
it.printStackTraceDebug()
}.getOrNull() ?: return false

View File

@@ -29,7 +29,7 @@ abstract class LocalObserveMapper<E : Any, R : Any>(
val mapped = if (m.isLocal) {
m
} else {
localMangaIndex.get(m.id)?.manga
localMangaIndex.get(m.id, withDetails = false)?.manga
}
mapped?.let { mm -> toResult(item, mm) }
}

View File

@@ -1,5 +1,6 @@
package org.koitharu.kotatsu.local.domain.model
import android.net.Uri
import androidx.core.net.toFile
import androidx.core.net.toUri
import org.koitharu.kotatsu.core.util.ext.creationTime
@@ -21,6 +22,8 @@ data class LocalManga(
return field
}
fun toUri(): Uri = manga.url.toUri()
fun isMatchesQuery(query: String): Boolean {
return manga.title.contains(query, ignoreCase = true) ||
manga.altTitle?.contains(query, ignoreCase = true) == true ||

View File

@@ -1,10 +1,8 @@
package org.koitharu.kotatsu.reader.domain
import android.content.ContentResolver.MimeTypeInfo
import android.content.Context
import android.graphics.Rect
import android.net.Uri
import android.webkit.MimeTypeMap
import androidx.annotation.AnyThread
import androidx.collection.LongSparseArray
import androidx.collection.set
@@ -61,8 +59,6 @@ import org.koitharu.kotatsu.parsers.model.MangaSource
import org.koitharu.kotatsu.parsers.util.mimeType
import org.koitharu.kotatsu.parsers.util.requireBody
import org.koitharu.kotatsu.parsers.util.runCatchingCancellable
import org.koitharu.kotatsu.core.image.BitmapDecoderCompat
import org.koitharu.kotatsu.core.util.ext.mimeType
import org.koitharu.kotatsu.reader.ui.pager.ReaderPage
import java.io.File
import java.util.LinkedList