diff --git a/src/main/kotlin/nl/astraeus/persistence/Datastore.kt b/src/main/kotlin/nl/astraeus/persistence/Datastore.kt index bbd46cc..d9c58f1 100644 --- a/src/main/kotlin/nl/astraeus/persistence/Datastore.kt +++ b/src/main/kotlin/nl/astraeus/persistence/Datastore.kt @@ -5,7 +5,6 @@ import java.io.ObjectInputStream import java.io.ObjectOutputStream import java.io.Serializable import java.text.DecimalFormat -import java.util.* import java.util.concurrent.ConcurrentHashMap import kotlin.reflect.KClass @@ -35,10 +34,6 @@ class Datastore( private val indexes: MutableMap, MutableMap> = ConcurrentHashMap() init { - if (!directory.exists()) { - directory.mkdirs() - } - for (index in indexes) { this.indexes.getOrPut(index.cls) { ConcurrentHashMap() @@ -48,49 +43,43 @@ class Datastore( loadTransactions() } - private fun loadTransactions() { - synchronized(this) { - val snapshots: Array? = directory.listFiles { _, name -> name.startsWith("transaction-") && name.endsWith(".snp") } - val files: Array? = directory.listFiles { _, name -> name.startsWith("transaction-") && name.endsWith(".trn") } + override fun toString(): String { + return "Datastore(directory=${fileManager.directory}, classes=${data.keys.size}, indexes=${indexes.keys.size})" + } - var lastSnapshot: Long? = null - var lastSnapshotFile: File? = null - - snapshots?.let { - it.forEach { - val trnx = getTrnx(it) - if (lastSnapshot == null || trnx > (lastSnapshot ?: 0L)) { - lastSnapshot = trnx - lastSnapshotFile = it - } - } + // print status, show number of entries for each class and index + fun printStatus() { + println(this) + for ((cls, typeData) in data) { + println(" ${cls.simpleName}: ${typeData.data.size}") + for ((name, index) in indexes.getOrDefault(cls, mutableMapOf())) { + println(" $name: ${index.index.size}") } + } + } - val lastSnapshotFile2 = fileManager.findLastSnapshot() + private fun loadTransactions() { + val start = System.nanoTime() + + synchronized(this) { + val (lastSnapshot, lastSnapshotFile) = fileManager.findLastSnapshot() if (lastSnapshotFile != null) { - ObjectInputStream(lastSnapshotFile?.inputStream()).use { ois -> + ObjectInputStream(lastSnapshotFile.inputStream()).use { ois -> readSnapshot(ois) } } - val trns = fileManager.findTransactionsAfter(lastSnapshot ?: 0L) + val transactions = fileManager.findTransactionsAfter(lastSnapshot ?: 0L) - files?.also { snaphotFiles -> - Arrays.sort(snaphotFiles) { o1, o2 -> if (getTrnx(o1) > getTrnx(o2)) 1 else -1} - - snaphotFiles.forEach { file -> - if (getTrnx(file) > (lastSnapshot ?: 0L)) { - ObjectInputStream(file.inputStream()).use { ois -> - val transactionNumber = ois.readLong() - nextTransactionNumber = transactionNumber + 1 - val actions = ois.readObject() as MutableList - execute(actions) - } - } + transactions?.forEach { file -> + ObjectInputStream(file.inputStream()).use { ois -> + readTransaction(ois) } } } + + Logger.debug("Loaded transactions in ${(System.nanoTime() - start) / 1_000_000}ms") } private fun getTrnx(file: File): Long { @@ -112,6 +101,7 @@ class Datastore( typeData.data[action.obj.id] = action.obj for (index in indexes[action.obj::class.java]?.values ?: listOf()) { + index.remove(action.obj.id) index.add(action.obj as Persistable) } } @@ -120,7 +110,7 @@ class Datastore( typeData.data.remove(action.obj.id) for (index in indexes[action.obj::class.java]?.values ?: listOf()) { - index.remove(action.obj) + index.remove(action.obj.id) } } } @@ -161,17 +151,29 @@ class Datastore( fun storeActions(actions: MutableList) { if (actions.isNotEmpty()) { synchronized(this) { - val number = transactionFormatter.format(nextTransactionNumber) - val file = File(directory, "transaction-$number.trn") - ObjectOutputStream(file.outputStream()).use { oos -> - oos.writeLong(nextTransactionNumber++) - oos.writeObject(actions) - } + writeTransaction(actions) } } } + private fun readTransaction(ois: ObjectInputStream) { + val transactionNumber = ois.readLong() + nextTransactionNumber = transactionNumber + 1 + val actions = ois.readObject() as MutableList + execute(actions) + } + + private fun writeTransaction(actions: MutableList) { + val number = transactionFormatter.format(nextTransactionNumber) + val file = File(directory, "transaction-$number.trn") + ObjectOutputStream(file.outputStream()).use { oos -> + oos.writeLong(nextTransactionNumber++) + oos.writeObject(actions) + } + } + fun snapshot() { + val start = System.nanoTime() synchronized(this) { val number = transactionFormatter.format(nextTransactionNumber) val file = File(directory, "transaction-$number.snp") @@ -189,6 +191,7 @@ class Datastore( } } } + Logger.debug("Snapshot in ${(System.nanoTime() - start) / 1_000_000}ms") } private fun readSnapshot(ois: ObjectInputStream) { @@ -226,4 +229,8 @@ class Datastore( } } + fun removeOldFiles() { + fileManager.removeOldFiles() + } + } diff --git a/src/main/kotlin/nl/astraeus/persistence/FileManager.kt b/src/main/kotlin/nl/astraeus/persistence/FileManager.kt new file mode 100644 index 0000000..70011a0 --- /dev/null +++ b/src/main/kotlin/nl/astraeus/persistence/FileManager.kt @@ -0,0 +1,62 @@ +package nl.astraeus.nl.astraeus.persistence + +import java.io.File + +class FileManager( + val directory: File +) { + + init { + if (!directory.exists()) { + directory.mkdirs() + } + } + + fun findLastSnapshot(): Pair { + directory.listFiles { _, name -> + name.startsWith("transaction-") && name.endsWith(".snp") + }?.maxByOrNull { + getTrnx(it) + }?.also { file -> + return getTrnx(file) to file + } + + return null to null + } + + fun findTransactionsAfter(trnx: Long): List? { + return directory.listFiles { _, name -> + name.startsWith("transaction-") && name.endsWith(".trn") + }?.filter { + getTrnx(it) > trnx + }?.sortedBy { + getTrnx(it) + } + } + + private fun getTrnx(file: File): Long { + // todo: add checks, improve performance + return file.name + .substringAfterLast('/') + .substringAfter("transaction-") + .substringBefore(".") + .toLong() + } + + fun removeOldFiles() { + val (lastSnapshot, _) = findLastSnapshot() + + if (lastSnapshot != null) { + val files = directory.listFiles { _, name -> + name.startsWith("transaction-") + }?.filter { + getTrnx(it) < lastSnapshot + } + + files?.forEach { + it.delete() + } + } + } + +} diff --git a/src/main/kotlin/nl/astraeus/persistence/Indexing.kt b/src/main/kotlin/nl/astraeus/persistence/Indexing.kt index 44528bb..b501495 100644 --- a/src/main/kotlin/nl/astraeus/persistence/Indexing.kt +++ b/src/main/kotlin/nl/astraeus/persistence/Indexing.kt @@ -27,11 +27,17 @@ class Index( index[key]?.remove(obj.id) } + fun remove(id: Long) { + for ((_, ids) in index) { + ids.remove(id) + } + } + fun find(key: Any): List { return index[key]?.mapNotNull { currentTransaction()?.find(cls.kotlin, it) } ?: emptyList() } - fun matches(obj: Persistable, value: Any): Boolean { + fun matches(obj: Persistable, value: Serializable): Boolean { return value(obj) == value } } diff --git a/src/main/kotlin/nl/astraeus/persistence/Logger.kt b/src/main/kotlin/nl/astraeus/persistence/Logger.kt new file mode 100644 index 0000000..aebf8ec --- /dev/null +++ b/src/main/kotlin/nl/astraeus/persistence/Logger.kt @@ -0,0 +1,8 @@ +package nl.astraeus.nl.astraeus.persistence + +object Logger { + var debug: (String) -> Unit = { println("DEBUG: $it") } + var info: (String) -> Unit = { println("INFO: $it") } + var warn: (String) -> Unit = { println("WARN: $it") } + var error: (String) -> Unit = { println("ERROR: $it") } +} \ No newline at end of file diff --git a/src/main/kotlin/nl/astraeus/persistence/Persistent.kt b/src/main/kotlin/nl/astraeus/persistence/Persistent.kt index 4c1affb..eddf633 100644 --- a/src/main/kotlin/nl/astraeus/persistence/Persistent.kt +++ b/src/main/kotlin/nl/astraeus/persistence/Persistent.kt @@ -35,4 +35,8 @@ class Persistent( fun snapshot() { datastore.snapshot() } + + fun removeOldFiles() { + datastore.removeOldFiles() + } } diff --git a/src/main/kotlin/nl/astraeus/persistence/Transaction.kt b/src/main/kotlin/nl/astraeus/persistence/Transaction.kt index 1305bd7..255db02 100644 --- a/src/main/kotlin/nl/astraeus/persistence/Transaction.kt +++ b/src/main/kotlin/nl/astraeus/persistence/Transaction.kt @@ -70,10 +70,11 @@ class Transaction( fun findByIndex( kClass: KClass, indexName: String, - search: Any + search: Serializable ): List { val result = mutableListOf() - val index = persistent.datastore.findIndex(kClass, indexName) ?: throw IllegalArgumentException("Index not found") + val index = persistent.datastore.findIndex(kClass, indexName) + ?: throw IllegalArgumentException("Index with name $indexName not found for class ${kClass.simpleName}") index.find(search).forEach { id -> result.add(id as T) @@ -97,4 +98,36 @@ class Transaction( return result } + fun searchIndex( + kClass: KClass, + indexName: String, + search: (Serializable) -> Boolean, + ): List { + val result = mutableListOf() + val index = persistent.datastore.findIndex(kClass, indexName) ?: throw IllegalArgumentException("Index not found") + + index.index.keys.forEach { key -> + if (search(key)) { + index.find(key).forEach { id -> + result.add(id as T) + } + } + } + + for (action in actions) { + if (action.obj::class == kClass) { + val indexedValue = index.value(action.obj) + if (indexedValue != null && index.matches(action.obj, indexedValue)) { + if (action.type == ActionType.DELETE) { + result.remove(action.obj as T) + } else if (action.type == ActionType.STORE) { + result.remove(action.obj) + result.add(action.obj as T) + } + } + } + } + + return result + } } diff --git a/src/test/kotlin/nl/astraeus/persistence/TestPersistence.kt b/src/test/kotlin/nl/astraeus/persistence/TestPersistence.kt index a867787..7a30f46 100644 --- a/src/test/kotlin/nl/astraeus/persistence/TestPersistence.kt +++ b/src/test/kotlin/nl/astraeus/persistence/TestPersistence.kt @@ -136,17 +136,43 @@ class TestPersistence { store( Person( id = 10L, - name = "Pipo", + name = "John Pipo", age = 23 ) ) store( Person( id = 11L, - name = "Clown", + name = "John Clown", age = 18 ) ) + + searchIndex(Person::class, "name") { + name -> (name as? String)?.startsWith("John") == true + }.forEach { p -> + println("Found person where name starts with 'John': ${p.name} - ${p.age}") + } } + + pst.transaction { + store( + Person( + id = 15L, + name = "Mama", + age = 26 + ) + ) + store( + Person( + id = 16L, + name = "Loe", + age = 16 + ) + ) + } + + pst.datastore.printStatus() + //pst.removeOldFiles() } }