configure

@@ -15,7 +15,7 @@ TOOLS="tools"
 
 # Default compiler flags. These must be supported by all POSIX C compilers.
 # "Fancy" compilers that have additional options must be detected and set below.
-CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE} -I${SRC}";
+CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE}"
 LIBS="-lm -lpthread"
 
 # Default args for all platforms.
@@ -24,10 +24,10 @@ SCRIPT_ARGS="--prefix=/usr/local --lib-name=Cytoplasm"
 # Set SSL flags depending on the platform.
 case "$(uname)" in
     OpenBSD)
-        SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl --disable-lmdb"
+        SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl"
         ;;
     *)
-        SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl --disable-lmdb"
+        SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl"
         ;;
 esac
 
@@ -80,14 +80,6 @@ for arg in $SCRIPT_ARGS; do
             TLS_IMPL=""
             TLS_LIBS=""
             ;;
-        --with-lmdb)
-            EDB_IMPL="EDB_LMDB"
-            EDB_LIBS="-llmdb"
-            ;;
-        --disable-lmdb)
-            EDB_IMPL=""
-            EDB_LIBS=""
-            ;;
         --prefix=*)
             PREFIX=$(echo "$arg" | cut -d '=' -f 2-)
             ;;
@@ -112,11 +104,6 @@ if [ -n "$TLS_IMPL" ]; then
     LIBS="${LIBS} ${TLS_LIBS}"
 fi
 
-if [ -n "$EDB_IMPL" ]; then
-    CFLAGS="${CFLAGS} -D${EDB_IMPL}"
-    LIBS="${LIBS} ${EDB_LIBS}"
-fi
-
 CFLAGS="${CFLAGS} '-DLIB_NAME=\"${LIB_NAME}\"' ${DEBUG}"
 LDFLAGS="${LIBS} ${LDFLAGS}"
 
@@ -153,7 +140,7 @@ print_obj() {
 get_deps() {
     src="$1"
 
-    ${CC} -I${SRC} -I${INCLUDE} -E "$src" \
+    ${CC} -I${INCLUDE} -E "$src" \
         | grep '^#'              \
         | awk '{print $3}'       \
         | cut -d '"' -f 2        \

include/Cytoplasm/Db.h

@@ -48,17 +48,6 @@
  */
 typedef struct Db Db;
 
-/**
- * Some "hints" for the database backend for operations. 
- * Hints are a way for the program to declare what to except of it
- * (and the program MUST adhere to these hints, but the backend 
- * MAY adhere).
- */
-typedef enum DbHint {
-    DB_HINT_READONLY,   /* The database reference is treated as read-only */
-    DB_HINT_WRITE       /* The database reference is treated as read/write */
-} DbHint;
-
 /**
  * When an object is locked, a reference is returned. This reference
  * is owned by the current thread, and the database is inaccessible to
@@ -79,15 +68,6 @@ typedef struct DbRef DbRef;
  */
 extern Db * DbOpen(char *, size_t);
 
-/**
- * Open a LMDB data directory. This function is similar to 
- * .Fn DbOpen ,
- * but works with a LMDB-based backend, with the second argument 
- * being the maximum filesize. This function fails with NULL if ever 
- * called without LMDB enabled at compiletime.
- */
-extern Db * DbOpenLMDB(char *, size_t);
-
 /**
  * Close the database. This function will flush anything in the cache
  * to the disk, and then close the data directory. It assumes that
@@ -129,15 +109,6 @@ extern DbRef * DbCreate(Db *, size_t,...);
  */
 extern DbRef * DbLock(Db *, size_t,...);
 
-/**
- * Behaves like
- * .Fn DbLock ,
- * but with hints on the reference itself, as 
- * .Fn DbLock 
- * itself is read/write.
- */
-extern DbRef * DbLockIntent(Db *, DbHint, size_t,...);
-
 /**
  * Immediately and permanently remove an object from the database.
  * This function assumes the object is not locked, otherwise undefined

src/Db.c

@@ -0,0 +1,977 @@
+/*
+ * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include <Db.h>
+
+#include <Memory.h>
+#include <Json.h>
+#include <Util.h>
+#include <Str.h>
+#include <Stream.h>
+#include <Log.h>
+
+#include <sys/types.h>
+#include <dirent.h>
+
+#include <pthread.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <errno.h>
+#include <string.h>
+
+struct Db
+{
+    char *dir;
+    pthread_mutex_t lock;
+
+    size_t cacheSize;
+    size_t maxCache;
+    HashMap *cache;
+
+    /*
+     * The cache uses a double linked list (see DbRef
+     * below) to know which objects are most and least
+     * recently used. The following diagram helps me
+     * know what way all the pointers go, because it
+     * can get very confusing sometimes. For example,
+     * there's nothing stopping "next" from pointing to
+     * least recent, and "prev" from pointing to most
+     * recent, so hopefully this clarifies the pointer
+     * terminology used when dealing with the linked
+     * list:
+     *
+     *         mostRecent            leastRecent
+     *             |   prev       prev   |   prev
+     *           +---+ ---> +---+ ---> +---+ ---> NULL
+     *           |ref|      |ref|      |ref|
+     * NULL <--- +---+ <--- +---+ <--- +---+
+     *      next       next       next
+     */
+    DbRef *mostRecent;
+    DbRef *leastRecent;
+};
+
+struct DbRef
+{
+    HashMap *json;
+
+    uint64_t ts;
+    size_t size;
+
+    Array *name;
+
+    DbRef *prev;
+    DbRef *next;
+
+    int fd;
+    Stream *stream;
+};
+
+static void
+StringArrayFree(Array * arr)
+{
+    size_t i;
+
+    for (i = 0; i < ArraySize(arr); i++)
+    {
+        Free(ArrayGet(arr, i));
+    }
+
+    ArrayFree(arr);
+}
+
+static ssize_t DbComputeSize(HashMap *);
+
+static ssize_t
+DbComputeSizeOfValue(JsonValue * val)
+{
+    MemoryInfo *a;
+    ssize_t total = 0;
+
+    size_t i;
+
+    union
+    {
+        char *str;
+        Array *arr;
+    } u;
+
+    if (!val)
+    {
+        return -1;
+    }
+
+    a = MemoryInfoGet(val);
+    if (a)
+    {
+        total += MemoryInfoGetSize(a);
+    }
+
+    switch (JsonValueType(val))
+    {
+        case JSON_OBJECT:
+            total += DbComputeSize(JsonValueAsObject(val));
+            break;
+        case JSON_ARRAY:
+            u.arr = JsonValueAsArray(val);
+            a = MemoryInfoGet(u.arr);
+
+            if (a)
+            {
+                total += MemoryInfoGetSize(a);
+            }
+
+            for (i = 0; i < ArraySize(u.arr); i++)
+            {
+                total += DbComputeSizeOfValue(ArrayGet(u.arr, i));
+            }
+            break;
+        case JSON_STRING:
+            u.str = JsonValueAsString(val);
+            a = MemoryInfoGet(u.str);
+            if (a)
+            {
+                total += MemoryInfoGetSize(a);
+            }
+            break;
+        case JSON_NULL:
+        case JSON_INTEGER:
+        case JSON_FLOAT:
+        case JSON_BOOLEAN:
+        default:
+            /* These don't use any extra heap space */
+            break;
+    }
+    return total;
+}
+
+static ssize_t
+DbComputeSize(HashMap * json)
+{
+    char *key;
+    JsonValue *val;
+    MemoryInfo *a;
+    size_t total;
+
+    if (!json)
+    {
+        return -1;
+    }
+
+    total = 0;
+
+    a = MemoryInfoGet(json);
+    if (a)
+    {
+        total += MemoryInfoGetSize(a);
+    }
+
+    while (HashMapIterate(json, &key, (void **) &val))
+    {
+        a = MemoryInfoGet(key);
+        if (a)
+        {
+            total += MemoryInfoGetSize(a);
+        }
+
+        total += DbComputeSizeOfValue(val);
+    }
+
+    return total;
+}
+
+static char *
+DbHashKey(Array * args)
+{
+    size_t i;
+    char *str = NULL;
+
+    for (i = 0; i < ArraySize(args); i++)
+    {
+        char *tmp = StrConcat(2, str, ArrayGet(args, i));
+
+        Free(str);
+        str = tmp;
+    }
+
+    return str;
+}
+
+static char
+DbSanitiseChar(char input)
+{
+    switch (input)
+    {
+        case '/':
+            return '_';
+        case '.':
+            return '-';
+    }
+    return input;
+}
+
+static char *
+DbDirName(Db * db, Array * args, size_t strip)
+{
+    size_t i, j;
+    char *str = StrConcat(2, db->dir, "/");
+
+    for (i = 0; i < ArraySize(args) - strip; i++)
+    {
+        char *tmp;
+        char *sanitise = StrDuplicate(ArrayGet(args, i));
+        for (j = 0; j < strlen(sanitise); j++)
+        {
+            sanitise[j] = DbSanitiseChar(sanitise[j]);
+        }
+
+        tmp = StrConcat(3, str, sanitise, "/");
+
+        Free(str);
+        Free(sanitise);
+
+        str = tmp;
+    }
+
+    return str;
+}
+
+static char *
+DbFileName(Db * db, Array * args)
+{
+    size_t i;
+    char *str = StrConcat(2, db->dir, "/");
+
+    for (i = 0; i < ArraySize(args); i++)
+    {
+        char *tmp;
+        char *arg = StrDuplicate(ArrayGet(args, i));
+        size_t j = 0;
+
+        /* Sanitize name to prevent directory traversal attacks */
+        while (arg[j])
+        {
+            arg[j] = DbSanitiseChar(arg[j]);
+            j++;
+        }
+
+        tmp = StrConcat(3, str, arg,
+                        (i < ArraySize(args) - 1) ? "/" : ".json");
+
+        Free(arg);
+        Free(str);
+
+        str = tmp;
+    }
+
+    return str;
+}
+
+static void
+DbCacheEvict(Db * db)
+{
+    DbRef *ref = db->leastRecent;
+    DbRef *tmp;
+
+    while (ref && db->cacheSize > db->maxCache)
+    {
+        char *hash;
+
+        JsonFree(ref->json);
+
+        hash = DbHashKey(ref->name);
+        HashMapDelete(db->cache, hash);
+        Free(hash);
+
+        StringArrayFree(ref->name);
+
+        db->cacheSize -= ref->size;
+
+        if (ref->next)
+        {
+            ref->next->prev = ref->prev;
+        }
+        else
+        {
+            db->mostRecent = ref->prev;
+        }
+
+        if (ref->prev)
+        {
+            ref->prev->next = ref->next;
+        }
+        else
+        {
+            db->leastRecent = ref->next;
+        }
+
+        tmp = ref->next;
+        Free(ref);
+
+        ref = tmp;
+    }
+}
+
+Db *
+DbOpen(char *dir, size_t cache)
+{
+    Db *db;
+    pthread_mutexattr_t attr;
+
+    if (!dir)
+    {
+        return NULL;
+    }
+
+    db = Malloc(sizeof(Db));
+    if (!db)
+    {
+        return NULL;
+    }
+
+    db->dir = dir;
+    db->maxCache = cache;
+
+    pthread_mutexattr_init(&attr);
+    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
+    pthread_mutex_init(&db->lock, &attr);
+    pthread_mutexattr_destroy(&attr);
+
+    db->mostRecent = NULL;
+    db->leastRecent = NULL;
+    db->cacheSize = 0;
+
+    if (db->maxCache)
+    {
+        db->cache = HashMapCreate();
+        if (!db->cache)
+        {
+            return NULL;
+        }
+    }
+    else
+    {
+        db->cache = NULL;
+    }
+
+    return db;
+}
+
+void
+DbMaxCacheSet(Db * db, size_t cache)
+{
+    if (!db)
+    {
+        return;
+    }
+
+    pthread_mutex_lock(&db->lock);
+
+    db->maxCache = cache;
+    if (db->maxCache && !db->cache)
+    {
+        db->cache = HashMapCreate();
+        db->cacheSize = 0;
+    }
+
+    DbCacheEvict(db);
+
+    pthread_mutex_unlock(&db->lock);
+}
+
+void
+DbClose(Db * db)
+{
+    if (!db)
+    {
+        return;
+    }
+
+    pthread_mutex_lock(&db->lock);
+
+    DbMaxCacheSet(db, 0);
+    DbCacheEvict(db);
+    HashMapFree(db->cache);
+
+    pthread_mutex_unlock(&db->lock);
+    pthread_mutex_destroy(&db->lock);
+
+    Free(db);
+}
+
+static DbRef *
+DbLockFromArr(Db * db, Array * args)
+{
+    char *file;
+    char *hash;
+    DbRef *ref;
+    struct flock lock;
+
+    int fd;
+    Stream *stream;
+
+    if (!db || !args)
+    {
+        return NULL;
+    }
+
+    ref = NULL;
+    hash = NULL;
+
+    pthread_mutex_lock(&db->lock);
+
+    /* Check if the item is in the cache */
+    hash = DbHashKey(args);
+    ref = HashMapGet(db->cache, hash);
+    file = DbFileName(db, args);
+
+    fd = open(file, O_RDWR);
+    if (fd == -1)
+    {
+        if (ref)
+        {
+            HashMapDelete(db->cache, hash);
+            JsonFree(ref->json);
+            StringArrayFree(ref->name);
+
+            db->cacheSize -= ref->size;
+
+            if (ref->next)
+            {
+                ref->next->prev = ref->prev;
+            }
+            else
+            {
+                db->mostRecent = ref->prev;
+            }
+
+            if (ref->prev)
+            {
+                ref->prev->next = ref->next;
+            }
+            else
+            {
+                db->leastRecent = ref->next;
+            }
+
+            if (!db->leastRecent)
+            {
+                db->leastRecent = db->mostRecent;
+            }
+
+            Free(ref);
+        }
+        ref = NULL;
+        goto finish;
+    }
+
+    stream = StreamFd(fd);
+
+    lock.l_start = 0;
+    lock.l_len = 0;
+    lock.l_type = F_WRLCK;
+    lock.l_whence = SEEK_SET;
+
+    /* Lock the file on the disk */
+    if (fcntl(fd, F_SETLK, &lock) < 0)
+    {
+        StreamClose(stream);
+        ref = NULL;
+        goto finish;
+    }
+
+    if (ref)                       /* In cache */
+    {
+        uint64_t diskTs = UtilLastModified(file);
+
+        ref->fd = fd;
+        ref->stream = stream;
+
+        if (diskTs > ref->ts)
+        {
+            /* File was modified on disk since it was cached */
+            HashMap *json = JsonDecode(ref->stream);
+
+            if (!json)
+            {
+                StreamClose(ref->stream);
+                ref = NULL;
+                goto finish;
+            }
+
+            JsonFree(ref->json);
+            ref->json = json;
+            ref->ts = diskTs;
+            ref->size = DbComputeSize(ref->json);
+        }
+
+        /* Float this ref to mostRecent */
+        if (ref->next)
+        {
+            ref->next->prev = ref->prev;
+
+            if (!ref->prev)
+            {
+                db->leastRecent = ref->next;
+            }
+            else
+            {
+                ref->prev->next = ref->next;
+            }
+
+            ref->prev = db->mostRecent;
+            ref->next = NULL;
+            if (db->mostRecent)
+            {
+                db->mostRecent->next = ref;
+            }
+            db->mostRecent = ref;
+        }
+
+        /* If there is no least recent, this is the only thing in the
+         * cache, so it is also least recent. */
+        if (!db->leastRecent)
+        {
+            db->leastRecent = ref;
+        }
+
+        /* The file on disk may be larger than what we have in memory,
+         * which may require items in cache to be evicted. */
+        DbCacheEvict(db);
+    }
+    else
+    {
+        Array *name;
+        size_t i;
+
+        /* Not in cache; load from disk */
+
+        ref = Malloc(sizeof(DbRef));
+        if (!ref)
+        {
+            StreamClose(stream);
+            goto finish;
+        }
+
+        ref->json = JsonDecode(stream);
+
+        if (!ref->json)
+        {
+            Free(ref);
+            StreamClose(stream);
+            ref = NULL;
+            goto finish;
+        }
+
+        ref->fd = fd;
+        ref->stream = stream;
+
+        name = ArrayCreate();
+        for (i = 0; i < ArraySize(args); i++)
+        {
+            ArrayAdd(name, StrDuplicate(ArrayGet(args, i)));
+        }
+        ref->name = name;
+
+        if (db->cache)
+        {
+            ref->ts = UtilTsMillis();
+            ref->size = DbComputeSize(ref->json);
+            HashMapSet(db->cache, hash, ref);
+            db->cacheSize += ref->size;
+
+            ref->next = NULL;
+            ref->prev = db->mostRecent;
+            if (db->mostRecent)
+            {
+                db->mostRecent->next = ref;
+            }
+            db->mostRecent = ref;
+
+            if (!db->leastRecent)
+            {
+                db->leastRecent = ref;
+            }
+
+            /* Adding this item to the cache may case it to grow too
+             * large, requiring some items to be evicted */
+            DbCacheEvict(db);
+        }
+    }
+
+finish:
+    if (!ref)
+    {
+        pthread_mutex_unlock(&db->lock);
+    }
+
+    Free(file);
+    Free(hash);
+
+    return ref;
+}
+
+DbRef *
+DbCreate(Db * db, size_t nArgs,...)
+{
+    Stream *fp;
+    char *file;
+    char *dir;
+    va_list ap;
+    Array *args;
+    DbRef *ret;
+
+    if (!db)
+    {
+        return NULL;
+    }
+
+    va_start(ap, nArgs);
+    args = ArrayFromVarArgs(nArgs, ap);
+    va_end(ap);
+
+    if (!args)
+    {
+        return NULL;
+    }
+
+    pthread_mutex_lock(&db->lock);
+
+    file = DbFileName(db, args);
+
+    if (UtilLastModified(file))
+    {
+        Free(file);
+        ArrayFree(args);
+        pthread_mutex_unlock(&db->lock);
+        return NULL;
+    }
+
+    dir = DbDirName(db, args, 1);
+    if (UtilMkdir(dir, 0750) < 0)
+    {
+        Free(file);
+        ArrayFree(args);
+        Free(dir);
+        pthread_mutex_unlock(&db->lock);
+        return NULL;
+    }
+
+    Free(dir);
+
+    fp = StreamOpen(file, "w");
+    Free(file);
+    if (!fp)
+    {
+        ArrayFree(args);
+        pthread_mutex_unlock(&db->lock);
+        return NULL;
+    }
+
+    StreamPuts(fp, "{}");
+    StreamClose(fp);
+
+    /* DbLockFromArr() will lock again for us */
+    pthread_mutex_unlock(&db->lock);
+
+    ret = DbLockFromArr(db, args);
+
+    ArrayFree(args);
+
+    return ret;
+}
+
+bool
+DbDelete(Db * db, size_t nArgs,...)
+{
+    va_list ap;
+    Array *args;
+    char *file;
+    char *hash;
+    bool ret = true;
+    DbRef *ref;
+
+    if (!db)
+    {
+        return false;
+    }
+
+    va_start(ap, nArgs);
+    args = ArrayFromVarArgs(nArgs, ap);
+    va_end(ap);
+
+    pthread_mutex_lock(&db->lock);
+
+    hash = DbHashKey(args);
+    file = DbFileName(db, args);
+
+    ref = HashMapGet(db->cache, hash);
+    if (ref)
+    {
+        HashMapDelete(db->cache, hash);
+        JsonFree(ref->json);
+        StringArrayFree(ref->name);
+
+        db->cacheSize -= ref->size;
+
+        if (ref->next)
+        {
+            ref->next->prev = ref->prev;
+        }
+        else
+        {
+            db->mostRecent = ref->prev;
+        }
+
+        if (ref->prev)
+        {
+            ref->prev->next = ref->next;
+        }
+        else
+        {
+            db->leastRecent = ref->next;
+        }
+
+        if (!db->leastRecent)
+        {
+            db->leastRecent = db->mostRecent;
+        }
+
+        Free(ref);
+    }
+
+    Free(hash);
+
+    if (UtilLastModified(file))
+    {
+        ret = (remove(file) == 0);
+    }
+
+    pthread_mutex_unlock(&db->lock);
+
+    ArrayFree(args);
+    Free(file);
+    return ret;
+}
+
+DbRef *
+DbLock(Db * db, size_t nArgs,...)
+{
+    va_list ap;
+    Array *args;
+    DbRef *ret;
+
+    va_start(ap, nArgs);
+    args = ArrayFromVarArgs(nArgs, ap);
+    va_end(ap);
+
+    if (!args)
+    {
+        return NULL;
+    }
+
+    ret = DbLockFromArr(db, args);
+
+    ArrayFree(args);
+
+    return ret;
+}
+
+bool
+DbUnlock(Db * db, DbRef * ref)
+{
+    bool destroy;
+
+    if (!db || !ref)
+    {
+        return false;
+    }
+
+    lseek(ref->fd, 0L, SEEK_SET);
+    if (ftruncate(ref->fd, 0) < 0)
+    {
+        pthread_mutex_unlock(&db->lock);
+        Log(LOG_ERR, "Failed to truncate file on disk.");
+        Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno));
+        return false;
+    }
+
+    JsonEncode(ref->json, ref->stream, JSON_DEFAULT);
+    StreamClose(ref->stream);
+
+    if (db->cache)
+    {
+        char *key = DbHashKey(ref->name);
+
+        if (HashMapGet(db->cache, key))
+        {
+            db->cacheSize -= ref->size;
+            ref->size = DbComputeSize(ref->json);
+            db->cacheSize += ref->size;
+
+            /* If this ref has grown significantly since we last
+             * computed its size, it may have filled the cache and
+             * require some items to be evicted. */
+            DbCacheEvict(db);
+
+            destroy = false;
+        }
+        else
+        {
+            destroy = true;
+        }
+
+        Free(key);
+    }
+    else
+    {
+        destroy = true;
+    }
+
+    if (destroy)
+    {
+        JsonFree(ref->json);
+        StringArrayFree(ref->name);
+
+        Free(ref);
+    }
+
+    pthread_mutex_unlock(&db->lock);
+    return true;
+}
+
+bool
+DbExists(Db * db, size_t nArgs,...)
+{
+    va_list ap;
+    Array *args;
+    char *file;
+    bool ret;
+
+    va_start(ap, nArgs);
+    args = ArrayFromVarArgs(nArgs, ap);
+    va_end(ap);
+
+    if (!args)
+    {
+        return false;
+    }
+
+    pthread_mutex_lock(&db->lock);
+
+    file = DbFileName(db, args);
+    ret = (UtilLastModified(file) != 0);
+
+    pthread_mutex_unlock(&db->lock);
+
+    Free(file);
+    ArrayFree(args);
+
+    return ret;
+}
+
+Array *
+DbList(Db * db, size_t nArgs,...)
+{
+    Array *result;
+    Array *path;
+    DIR *files;
+    struct dirent *file;
+    char *dir;
+    va_list ap;
+
+    if (!db || !nArgs)
+    {
+        return NULL;
+    }
+
+    result = ArrayCreate();
+    if (!result)
+    {
+        return NULL;
+    }
+
+    va_start(ap, nArgs);
+    path = ArrayFromVarArgs(nArgs, ap);
+    dir = DbDirName(db, path, 0);
+
+    pthread_mutex_lock(&db->lock);
+
+    files = opendir(dir);
+    if (!files)
+    {
+        ArrayFree(path);
+        ArrayFree(result);
+        Free(dir);
+        pthread_mutex_unlock(&db->lock);
+        return NULL;
+    }
+    while ((file = readdir(files)))
+    {
+        size_t namlen = strlen(file->d_name);
+
+        if (namlen > 5)
+        {
+            int nameOffset = namlen - 5;
+
+            if (StrEquals(file->d_name + nameOffset, ".json"))
+            {
+                file->d_name[nameOffset] = '\0';
+                ArrayAdd(result, StrDuplicate(file->d_name));
+            }
+        }
+    }
+    closedir(files);
+
+    ArrayFree(path);
+    Free(dir);
+    pthread_mutex_unlock(&db->lock);
+
+    return result;
+}
+
+void
+DbListFree(Array * arr)
+{
+    StringArrayFree(arr);
+}
+
+HashMap *
+DbJson(DbRef * ref)
+{
+    return ref ? ref->json : NULL;
+}
+
+bool
+DbJsonSet(DbRef * ref, HashMap * json)
+{
+    if (!ref || !json)
+    {
+        return false;
+    }
+
+    JsonFree(ref->json);
+    ref->json = JsonDuplicate(json);
+    return true;
+}

src/Db/Db.c

@@ -1,499 +0,0 @@
-/*
- * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation files
- * (the "Software"), to deal in the Software without restriction,
- * including without limitation the rights to use, copy, modify, merge,
- * publish, distribute, sublicense, and/or sell copies of the Software,
- * and to permit persons to whom the Software is furnished to do so,
- * subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#include <Db.h>
-
-#include <Memory.h>
-#include <Json.h>
-#include <Util.h>
-#include <Str.h>
-#include <Stream.h>
-#include <Log.h>
-
-#include <sys/types.h>
-#include <dirent.h>
-
-#include <pthread.h>
-#include <fcntl.h>
-#include <unistd.h>
-
-#include <errno.h>
-#include <string.h>
-
-#include "Db/Internal.h"
-
-void
-StringArrayFree(Array * arr)
-{
-    size_t i;
-
-    for (i = 0; i < ArraySize(arr); i++)
-    {
-        Free(ArrayGet(arr, i));
-    }
-
-    ArrayFree(arr);
-}
-
-static ssize_t DbComputeSize(HashMap *);
-
-static ssize_t
-DbComputeSizeOfValue(JsonValue * val)
-{
-    MemoryInfo *a;
-    ssize_t total = 0;
-
-    size_t i;
-
-    union
-    {
-        char *str;
-        Array *arr;
-    } u;
-
-    if (!val)
-    {
-        return -1;
-    }
-
-    a = MemoryInfoGet(val);
-    if (a)
-    {
-        total += MemoryInfoGetSize(a);
-    }
-
-    switch (JsonValueType(val))
-    {
-        case JSON_OBJECT:
-            total += DbComputeSize(JsonValueAsObject(val));
-            break;
-        case JSON_ARRAY:
-            u.arr = JsonValueAsArray(val);
-            a = MemoryInfoGet(u.arr);
-
-            if (a)
-            {
-                total += MemoryInfoGetSize(a);
-            }
-
-            for (i = 0; i < ArraySize(u.arr); i++)
-            {
-                total += DbComputeSizeOfValue(ArrayGet(u.arr, i));
-            }
-            break;
-        case JSON_STRING:
-            u.str = JsonValueAsString(val);
-            a = MemoryInfoGet(u.str);
-            if (a)
-            {
-                total += MemoryInfoGetSize(a);
-            }
-            break;
-        case JSON_NULL:
-        case JSON_INTEGER:
-        case JSON_FLOAT:
-        case JSON_BOOLEAN:
-        default:
-            /* These don't use any extra heap space */
-            break;
-    }
-    return total;
-}
-
-static ssize_t
-DbComputeSize(HashMap * json)
-{
-    char *key;
-    JsonValue *val;
-    MemoryInfo *a;
-    size_t total;
-
-    if (!json)
-    {
-        return -1;
-    }
-
-    total = 0;
-
-    a = MemoryInfoGet(json);
-    if (a)
-    {
-        total += MemoryInfoGetSize(a);
-    }
-
-    while (HashMapIterate(json, &key, (void **) &val))
-    {
-        a = MemoryInfoGet(key);
-        if (a)
-        {
-            total += MemoryInfoGetSize(a);
-        }
-
-        total += DbComputeSizeOfValue(val);
-    }
-
-    return total;
-}
-
-static char *
-DbHashKey(Array * args)
-{
-    size_t i;
-    char *str = NULL;
-
-    for (i = 0; i < ArraySize(args); i++)
-    {
-        char *tmp = StrConcat(2, str, ArrayGet(args, i));
-
-        Free(str);
-        str = tmp;
-    }
-
-    return str;
-}
-
-static void
-DbCacheEvict(Db * db)
-{
-    DbRef *ref = db->leastRecent;
-    DbRef *tmp;
-
-    while (ref && db->cacheSize > db->maxCache)
-    {
-        char *hash;
-
-        JsonFree(ref->json);
-
-        hash = DbHashKey(ref->name);
-        HashMapDelete(db->cache, hash);
-        Free(hash);
-
-        StringArrayFree(ref->name);
-
-        db->cacheSize -= ref->size;
-
-        if (ref->next)
-        {
-            ref->next->prev = ref->prev;
-        }
-        else
-        {
-            db->mostRecent = ref->prev;
-        }
-
-        if (ref->prev)
-        {
-            ref->prev->next = ref->next;
-        }
-        else
-        {
-            db->leastRecent = ref->next;
-        }
-
-        tmp = ref->next;
-        Free(ref);
-
-        ref = tmp;
-    }
-}
-
-void
-DbMaxCacheSet(Db * db, size_t cache)
-{
-    if (!db)
-    {
-        return;
-    }
-
-    pthread_mutex_lock(&db->lock);
-
-    db->maxCache = cache;
-    if (db->maxCache && !db->cache)
-    {
-        db->cache = HashMapCreate();
-        db->cacheSize = 0;
-    }
-
-    DbCacheEvict(db);
-
-    pthread_mutex_unlock(&db->lock);
-}
-
-void
-DbClose(Db * db)
-{
-    if (!db)
-    {
-        return;
-    }
-
-    pthread_mutex_lock(&db->lock);
-    if (db->close)
-    {
-        db->close(db);
-    }
-    DbMaxCacheSet(db, 0);
-    DbCacheEvict(db);
-    HashMapFree(db->cache);
-
-    pthread_mutex_unlock(&db->lock);
-    pthread_mutex_destroy(&db->lock);
-
-    Free(db);
-}
-
-DbRef *
-DbCreate(Db * db, size_t nArgs,...)
-{
-    va_list ap;
-    Array *args;
-    DbRef *ret;
-
-    if (!db || !db->create)
-    {
-        return NULL;
-    }
-
-    va_start(ap, nArgs);
-    args = ArrayFromVarArgs(nArgs, ap);
-    va_end(ap);
-
-    if (!args)
-    {
-        return NULL;
-    }
-
-    ret = db->create(db, args);
-    ArrayFree(args);
-
-    return ret;
-}
-
-bool
-DbDelete(Db * db, size_t nArgs,...)
-{
-    va_list ap;
-    Array *args;
-    bool ret = true;
-
-    if (!db)
-    {
-        return false;
-    }
-
-    va_start(ap, nArgs);
-    args = ArrayFromVarArgs(nArgs, ap);
-    va_end(ap);
-
-    ret = db->delete(db, args);
-
-    ArrayFree(args);
-    return ret;
-}
-
-DbRef *
-DbLock(Db * db, size_t nArgs,...)
-{
-    va_list ap;
-    Array *args;
-    DbRef *ret;
-
-    va_start(ap, nArgs);
-    args = ArrayFromVarArgs(nArgs, ap);
-    va_end(ap);
-
-    if (!args || !db->lockFunc)
-    {
-        return NULL;
-    }
-
-    ret = db->lockFunc(db, DB_HINT_WRITE, args);
-
-    ArrayFree(args);
-
-    return ret;
-}
-
-DbRef *
-DbLockIntent(Db * db, DbHint hint, size_t nArgs,...)
-{
-    va_list ap;
-    Array *args;
-    DbRef *ret;
-
-    va_start(ap, nArgs);
-    args = ArrayFromVarArgs(nArgs, ap);
-    va_end(ap);
-
-    if (!args || !db->lockFunc)
-    {
-        return NULL;
-    }
-
-    ret = db->lockFunc(db, hint, args);
-
-    ArrayFree(args);
-
-    return ret;
-}
-
-bool
-DbUnlock(Db * db, DbRef * ref)
-{
-    if (!db || !ref || !db->unlock)
-    {
-        return false;
-    }
-    
-    return db->unlock(db, ref);
-}
-
-bool
-DbExists(Db * db, size_t nArgs,...)
-{
-    va_list ap;
-    Array *args;
-    bool ret;
-    if (!db || !nArgs || !db->exists)
-    {
-        return false;
-    }
-
-    va_start(ap, nArgs);
-    args = ArrayFromVarArgs(nArgs, ap);
-    va_end(ap);
-
-    if (!args)
-    {
-        return false;
-    }
-
-    ret = db->exists(db, args);
-    ArrayFree(args);
-
-    return ret;
-}
-
-Array *
-DbList(Db * db, size_t nArgs,...)
-{
-    Array *result;
-    Array *path;
-    va_list ap;
-
-    if (!db || !nArgs || !db->list)
-    {
-        return NULL;
-    }
-
-    va_start(ap, nArgs);
-    path = ArrayFromVarArgs(nArgs, ap);
-    va_end(ap);
-
-    result = db->list(db, path);
-    
-    ArrayFree(path);
-    return result;
-}
-
-void
-DbListFree(Array * arr)
-{
-    StringArrayFree(arr);
-}
-
-HashMap *
-DbJson(DbRef * ref)
-{
-    return ref ? ref->json : NULL;
-}
-
-bool
-DbJsonSet(DbRef * ref, HashMap * json)
-{
-    if (!ref || !json)
-    {
-        return false;
-    }
-
-    JsonFree(ref->json);
-    ref->json = JsonDuplicate(json);
-    return true;
-}
-void
-DbInit(Db *db)
-{
-    pthread_mutexattr_t attr;
-    if (!db)
-    {
-        return;
-    }
-
-    pthread_mutexattr_init(&attr);
-    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
-    pthread_mutex_init(&db->lock, &attr);
-    pthread_mutexattr_destroy(&attr);
-
-    db->mostRecent = NULL;
-    db->leastRecent = NULL;
-    db->cacheSize = 0;
-    db->maxCache = 0;
-
-    if (db->maxCache)
-    {
-        db->cache = HashMapCreate();
-    }
-    else
-    {
-        db->cache = NULL;
-    }
-}
-void
-DbRefInit(Db *db, DbRef *ref)
-{
-    if (!db || !ref)
-    {
-        return;
-    }
-    ref->prev = db->mostRecent;
-    ref->next = NULL;
-    ref->json = NULL;
-    ref->name = NULL;
-
-    /* As default values to be overwritten by impls */
-    ref->ts = UtilTsMillis();
-    ref->size = 0;
-
-    /* TODO: Append the ref to the cache list.
-     * I removed it because it broke everything and crashed all the time.
-     * My bad! */
-}
-void
-StringArrayAppend(Array *arr, char *str)
-{
-    if (!arr || !str)
-    {
-        return;
-    }
-    ArrayAdd(arr, StrDuplicate(str));
-}

src/Db/Flat.c

@@ -1,374 +0,0 @@
-#include <Db.h>
-
-#include <Memory.h>
-#include <Json.h>
-#include <Util.h>
-#include <Str.h>
-#include <Stream.h>
-#include <Log.h>
-
-#include <sys/types.h>
-#include <dirent.h>
-
-#include <pthread.h>
-#include <fcntl.h>
-#include <unistd.h>
-
-#include <errno.h>
-#include <string.h>
-
-#include "Db/Internal.h"
-
-typedef struct FlatDb {
-    Db base;
-    char *dir;
-    /* Theres not much to do here. */
-} FlatDb;
-typedef struct FlatDbRef {
-    DbRef base;
-
-    Stream *stream;
-    int fd;
-} FlatDbRef;
-
-static char
-DbSanitiseChar(char input)
-{
-    switch (input)
-    {
-        case '/':
-            return '_';
-        case '.':
-            return '-';
-    }
-    return input;
-}
-
-static char *
-DbDirName(FlatDb * db, Array * args, size_t strip)
-{
-    size_t i, j;
-    char *str = StrConcat(2, db->dir, "/");
-
-    for (i = 0; i < ArraySize(args) - strip; i++)
-    {
-        char *tmp;
-        char *sanitise = StrDuplicate(ArrayGet(args, i));
-        for (j = 0; j < strlen(sanitise); j++)
-        {
-            sanitise[j] = DbSanitiseChar(sanitise[j]);
-        }
-
-        tmp = StrConcat(3, str, sanitise, "/");
-
-        Free(str);
-        Free(sanitise);
-
-        str = tmp;
-    }
-
-    return str;
-}
-static char *
-DbFileName(FlatDb * db, Array * args)
-{
-    size_t i;
-    char *str = StrConcat(2, db->dir, "/");
-
-    for (i = 0; i < ArraySize(args); i++)
-    {
-        char *tmp;
-        char *arg = StrDuplicate(ArrayGet(args, i));
-        size_t j = 0;
-
-        /* Sanitize name to prevent directory traversal attacks */
-        while (arg[j])
-        {
-            arg[j] = DbSanitiseChar(arg[j]);
-            j++;
-        }
-
-        tmp = StrConcat(3, str, arg,
-                        (i < ArraySize(args) - 1) ? "/" : ".json");
-
-        Free(arg);
-        Free(str);
-
-        str = tmp;
-    }
-
-    return str;
-}
-
-static DbRef *
-FlatLock(Db *d, DbHint hint, Array *dir)
-{
-    FlatDb *db = (FlatDb *) d;
-    FlatDbRef *ref = NULL;
-    size_t i;
-    char *path = NULL;
-    if (!d || !dir)
-    {
-        return NULL;
-    }
-
-    pthread_mutex_lock(&d->lock);
-    path = DbFileName(db, dir);
-    /* TODO: Caching */
-    {
-        int fd = open(path, O_RDWR);
-        Stream *stream;
-        struct flock lock;
-        if (fd == -1)
-        {
-            ref = NULL;
-            goto end;
-        }
-
-        stream = StreamFd(fd);
-        if (!stream)
-        {
-            ref = NULL;
-            goto end;
-        }
-        
-        lock.l_start = 0;
-        lock.l_len = 0;
-        lock.l_type = F_WRLCK;
-        lock.l_whence = SEEK_SET;
-
-        if (fcntl(fd, F_SETLK, &lock) < 0)
-        {
-            StreamClose(stream);
-            ref = NULL;
-            goto end;
-        }
-
-        ref = Malloc(sizeof(*ref));
-        DbRefInit(d, (DbRef *) ref);
-        /* TODO: Hints */
-        ref->base.hint = hint;
-        ref->base.ts = UtilLastModified(path);
-        ref->base.json = JsonDecode(stream);
-        ref->stream = stream;
-        ref->fd = fd;
-        if (!ref->base.json)
-        {
-            Free(ref);
-            StreamClose(stream);
-            ref = NULL;
-            goto end;
-        }
-        
-        
-        ref->base.name = ArrayCreate();
-        for (i = 0; i < ArraySize(dir); i++)
-        {
-            StringArrayAppend(ref->base.name, ArrayGet(dir, i));
-        }
-    }
-end:
-    Free(path);
-    if (!ref)
-    {
-        pthread_mutex_unlock(&d->lock);
-    }
-    return (DbRef *) ref;
-}
-
-static bool
-FlatUnlock(Db *d, DbRef *r)
-{
-    FlatDbRef *ref = (FlatDbRef *) r;
-
-    if (!d || !r)
-    {
-        return false;
-    }
-
-    lseek(ref->fd, 0L, SEEK_SET);
-    if (ftruncate(ref->fd, 0) < 0)
-    {
-        pthread_mutex_unlock(&d->lock);
-        Log(LOG_ERR, "Failed to truncate file on disk.");
-        Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno));
-        return false;
-    }
-
-    JsonEncode(ref->base.json, ref->stream, JSON_DEFAULT);
-    StreamClose(ref->stream);
-
-    JsonFree(ref->base.json);
-    StringArrayFree(ref->base.name);
-    Free(ref);
-
-    pthread_mutex_unlock(&d->lock);
-    return true;
-}
-static DbRef *
-FlatCreate(Db *d, Array *dir)
-{
-    FlatDb *db = (FlatDb *) d;
-    char *path, *dirPath;
-    Stream *fp;
-    DbRef *ret;
-
-    if (!d || !dir)
-    {
-        return NULL;
-    }
-    pthread_mutex_lock(&d->lock);
-
-    path = DbFileName(db, dir);
-    if (UtilLastModified(path))
-    {
-        Free(path);
-        pthread_mutex_unlock(&d->lock);
-        return NULL;
-    }
-
-    dirPath = DbDirName(db, dir, 1);
-    if (UtilMkdir(dirPath, 0750) < 0)
-    {
-        Free(path);
-        Free(dirPath);
-        pthread_mutex_unlock(&d->lock);
-        return NULL;
-    }
-    Free(dirPath);
-
-    fp = StreamOpen(path, "w");
-    Free(path);
-    if (!fp)
-    {
-        pthread_mutex_unlock(&d->lock);
-        return NULL;
-    }
-
-    StreamPuts(fp, "{}");
-    StreamClose(fp);
-
-    /* FlatLock() will lock again for us */
-    pthread_mutex_unlock(&d->lock);
-    ret = FlatLock(d, DB_HINT_WRITE, dir);
-
-    return ret;
-}
-
-static bool
-FlatDelete(Db *d, Array *dir)
-{
-    bool ret = false;
-    char *file;
-    FlatDb *db = (FlatDb *) d;
-    if (!d || !dir)
-    {
-        return false;
-    }
-
-    pthread_mutex_lock(&d->lock);
-    file = DbFileName(db, dir);
-
-    /* TODO: Unlink the entry from the linkedlist */
-    if (UtilLastModified(file))
-    {
-        ret = remove(file) == 0;
-    }
-
-    Free(file);
-    pthread_mutex_unlock(&d->lock);
-    return ret;
-}
-
-static Array *
-FlatList(Db *d, Array *dir)
-{
-    FlatDb *db = (FlatDb *) d;
-    struct dirent *file;
-    Array *ret;
-    DIR *files;
-    char *path;
-
-    if (!d || !dir)
-    {
-        return NULL;
-    }
-
-    pthread_mutex_lock(&d->lock);
-
-    path = DbDirName(db, dir, 0);
-    files = opendir(path);
-    if (!files)
-    {
-        Free(path);
-        pthread_mutex_unlock(&d->lock);
-        return NULL;
-    }
-
-    ret = ArrayCreate();
-    while ((file = readdir(files)))
-    {
-        size_t namlen = strlen(file->d_name);
-
-        if (namlen > 5)
-        {
-            int nameOffset = namlen - 5;
-
-            if (StrEquals(file->d_name + nameOffset, ".json"))
-            {
-                file->d_name[nameOffset] = '\0';
-                StringArrayAppend(ret, file->d_name);
-            }
-        }
-    }
-    closedir(files);
-    Free(path);
-
-    pthread_mutex_unlock(&d->lock);
-    return ret;
-}
-static bool
-FlatExists(Db *d, Array *dir)
-{
-    FlatDb *db = (FlatDb *) d;
-    char *path;
-    bool ret;
-    if (!d || !dir)
-    {
-        return NULL;
-    }
-
-    pthread_mutex_lock(&d->lock);
-
-    path = DbFileName(db, dir);
-    ret = UtilLastModified(path) != 0;
-    Free(path);
-
-    pthread_mutex_unlock(&d->lock);
-
-    return ret;
-}
-
-Db *
-DbOpen(char *dir, size_t cache)
-{
-    FlatDb *db;
-    if (!dir)
-    {
-        return NULL;
-    }
-    db = Malloc(sizeof(*db));
-    DbInit((Db *) db);
-    db->dir = dir;
-    db->base.cacheSize = cache;
-
-    db->base.lockFunc = FlatLock;
-    db->base.unlock = FlatUnlock;
-    db->base.create = FlatCreate;
-    db->base.delete = FlatDelete;
-    db->base.exists = FlatExists;
-    db->base.list   = FlatList;
-    db->base.close = NULL;
-
-    return (Db *) db;
-}

src/Db/Internal.h

@@ -1,101 +0,0 @@
-/*
- * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation files
- * (the "Software"), to deal in the Software without restriction,
- * including without limitation the rights to use, copy, modify, merge,
- * publish, distribute, sublicense, and/or sell copies of the Software,
- * and to permit persons to whom the Software is furnished to do so,
- * subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifndef CYTOPLASM_DB_INTERNAL_H
-#define CYTOPLASM_DB_INTERNAL_H
-
-#include <HashMap.h>
-#include <Db.h>
-
-#include <pthread.h>
-
-/* TODO: Define the base structures to define a database internally. 
- * All "implementations" shall have them as a first element, so that 
- * basic, general functions can work on them properly. */
-
-/* The base structure of a database */
-struct Db
-{
-    pthread_mutex_t lock;
-
-    size_t cacheSize;
-    size_t maxCache;
-    HashMap *cache;
-
-    /*
-     * The cache uses a double linked list (see DbRef
-     * below) to know which objects are most and least
-     * recently used. The following diagram helps me
-     * know what way all the pointers go, because it
-     * can get very confusing sometimes. For example,
-     * there's nothing stopping "next" from pointing to
-     * least recent, and "prev" from pointing to most
-     * recent, so hopefully this clarifies the pointer
-     * terminology used when dealing with the linked
-     * list:
-     *
-     *         mostRecent            leastRecent
-     *             |   prev       prev   |   prev
-     *           +---+ ---> +---+ ---> +---+ ---> NULL
-     *           |ref|      |ref|      |ref|
-     * NULL <--- +---+ <--- +---+ <--- +---+
-     *      next       next       next
-     */
-    DbRef *mostRecent;
-    DbRef *leastRecent;
-
-    /* Functions for implementation-specific operations
-     * (opening a ref, closing a db, removing an entry, ...) */
-    DbRef * (*lockFunc)(Db *, DbHint, Array *);
-    DbRef * (*create)(Db *, Array *);
-    Array * (*list)(Db *, Array *);
-    bool (*unlock)(Db *, DbRef *);
-    bool (*delete)(Db *, Array *);
-    bool (*exists)(Db *, Array *);
-    void (*close)(Db *);
-
-    /* Implementation-specific constructs */
-};
-
-struct DbRef
-{
-    HashMap *json;
-
-    uint64_t ts;
-    size_t size;
-
-    Array *name;
-
-    DbRef *prev;
-    DbRef *next;
-
-    DbHint hint;
-    /* Implementation-specific constructs */
-};
-
-extern void DbInit(Db *);
-extern void DbRefInit(Db *, DbRef *);
-extern void StringArrayFree(Array *);
-extern void StringArrayAppend(Array *, char *);
-
-#endif

src/Db/LMDB.c

@@ -1,590 +0,0 @@
-#include <Memory.h>
-#include <Json.h>
-#include <Log.h>
-#include <Db.h>
-
-#include "Db/Internal.h"
-
-#include <stdint.h>
-#include <string.h>
-#include <stdlib.h>
-
-#ifdef EDB_LMDB
-
-#include <lmdb.h>
-
-typedef struct LMDB {
-    Db base; /* The base implementation required to pass */
-
-    MDB_env *environ;
-    MDB_dbi dbi;
-} LMDB;
-typedef struct LMDBRef {
-    DbRef base;
-
-    /* TODO: LMDB-dependent stuff */
-    MDB_txn *transaction;
-    MDB_dbi dbi;
-} LMDBRef;
-
-/* Helper functions */
-static MDB_val
-LMDBTranslateKey(Array *key)
-{
-    MDB_val ret = { 0 };
-    char *data = NULL;
-    size_t length = 0;
-    size_t i;
-    if (!key || ArraySize(key) > 255)
-    {
-        return ret;
-    }
-
-    data = Realloc(data, ++length);
-    data[0] = ArraySize(key);
-
-    /* Now, let's push every item */
-    for (i = 0; i < ArraySize(key); i++)
-    {
-        char *entry = ArrayGet(key, i);
-        size_t offset = length;
-
-        data = Realloc(data, (length += strlen(entry) + 1));
-        memcpy(data + offset, entry, strlen(entry));
-        data[length - 1] = '\0';
-    }
-
-    /* We now have every key */
-    ret.mv_size = length;
-    ret.mv_data = data;
-    return ret;
-}
-static void
-LMDBKillKey(MDB_val key)
-{
-    if (!key.mv_data || !key.mv_size)
-    {
-        return;
-    }
-
-    Free(key.mv_data);
-}
-static HashMap *
-LMDBDecode(MDB_val val)
-{
-    FILE *fakefile;
-    Stream *fakestream;
-    HashMap *ret;
-    if (!val.mv_data || !val.mv_size)
-    {
-        return NULL;
-    }
-
-    fakefile = fmemopen(val.mv_data, val.mv_size, "r");
-    fakestream = StreamFile(fakefile);
-    ret = JsonDecode(fakestream);
-    StreamClose(fakestream);
-
-    return ret;
-}
-static bool
-LMDBKeyStartsWith(MDB_val key, MDB_val starts)
-{
-    size_t i;
-    if (!key.mv_size || !starts.mv_size)
-    {
-        return false;
-    }
-    if (key.mv_size < starts.mv_size)
-    {
-        return false;
-    }
-
-    for (i = 0; i < starts.mv_size; i++)
-    {
-        char keyC = ((char *) key.mv_data)[i];
-        char startC = ((char *) starts.mv_data)[i];
-
-        if (keyC != startC)
-        {
-            return false;
-        }
-    }
-    return true;
-}
-static char *
-LMDBKeyHead(MDB_val key)
-{
-    char *end;
-    if (!key.mv_size || !key.mv_data)
-    {
-        return NULL;
-    }
-    /* -2 because we have a NULL byte in there */
-    end = ((char *) key.mv_data) + key.mv_size - 1;
-    if ((void *) end < key.mv_data)
-    {
-        /* Uh oh. */
-        return NULL;
-    }
-
-    while ((void *) (end - 1) >= key.mv_data && *(end - 1))
-    {
-        end--;
-    }
-    return end;
-}
-
-static DbRef *
-LMDBLock(Db *d, DbHint hint, Array *k)
-{
-    LMDB *db = (LMDB *) d;
-    MDB_txn *transaction;
-    LMDBRef *ret = NULL;
-    MDB_val key, json_val;
-    int code, flags;
-    if (!d || !k)
-    {
-        return NULL;
-    }
-
-    pthread_mutex_lock(&d->lock);
-    key = LMDBTranslateKey(k);
-
-    /* Create a transaction, honoring hints. */
-    /* TODO: Do we want to create a "main" transaction that everyone inherits 
-     * from? */
-    flags = hint == DB_HINT_READONLY ? MDB_RDONLY : 0;
-    if ((code = mdb_txn_begin(db->environ, NULL, flags, &transaction)) != 0)
-    {
-        /* Very bad! */
-        Log(LOG_ERR,
-            "%s: could not begin transaction: %s",
-            __func__, mdb_strerror(code)
-        );
-        goto end;
-    }
-
-    json_val.mv_size = 0;
-    json_val.mv_data = NULL;
-    code = mdb_get(transaction, db->dbi, &key, &json_val);
-    if (code == MDB_NOTFOUND)
-    {
-        mdb_txn_abort(transaction);
-        goto end;
-    }
-    else if (code != 0)
-    {
-        Log(LOG_ERR, 
-            "%s: mdb_get failure: %s", 
-            __func__, mdb_strerror(code)
-        );
-        mdb_txn_abort(transaction);
-        goto end;
-    }
-
-    ret = Malloc(sizeof(*ret));
-    DbRefInit(d, (DbRef *) ret);
-    /* TODO: Timestamp */
-    {
-        size_t i;
-        ret->base.name = ArrayCreate();
-        for (i = 0; i < ArraySize(k); i++)
-        {
-            char *ent = ArrayGet(k, i);
-            StringArrayAppend(ret->base.name, ent);
-        }
-    }
-    ret->base.json = LMDBDecode(json_val);
-    ret->base.hint = hint;
-    ret->transaction = NULL;
-
-    if (hint == DB_HINT_WRITE)
-    {
-        ret->transaction = transaction;
-    }
-    else
-    {
-        mdb_txn_abort(transaction);
-    }
-end:
-    if (!ret || hint == DB_HINT_READONLY)
-    {
-        pthread_mutex_unlock(&d->lock);
-    }
-    LMDBKillKey(key);
-    return (DbRef *) ret;
-}
-static bool
-LMDBExists(Db *d, Array *k)
-{
-    MDB_val key, empty;
-    LMDB *db = (LMDB *) d;
-    MDB_txn *transaction;
-    int code;
-    bool ret = false;
-    if (!d || !k)
-    {
-        return false;
-    }
-
-    pthread_mutex_lock(&d->lock);
-    key = LMDBTranslateKey(k);
-
-    /* create a txn */
-    if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0)
-    {
-        /* Very bad! */
-        Log(LOG_ERR,
-            "%s: could not begin transaction: %s",
-            __func__, mdb_strerror(code)
-        );
-        goto end;
-    }
-
-    ret = mdb_get(transaction, db->dbi, &key, &empty) == 0;
-    mdb_txn_abort(transaction);
-end:
-    LMDBKillKey(key);
-    pthread_mutex_unlock(&d->lock);
-    return ret;
-}
-static bool
-LMDBDelete(Db *d, Array *k)
-{
-    MDB_val key, empty;
-    LMDB *db = (LMDB *) d;
-    MDB_txn *transaction;
-    int code;
-    bool ret = false;
-    if (!d || !k)
-    {
-        return false;
-    }
-
-    pthread_mutex_lock(&d->lock);
-    key = LMDBTranslateKey(k);
-
-    /* create a txn */
-    if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0)
-    {
-        /* Very bad! */
-        Log(LOG_ERR,
-            "%s: could not begin transaction: %s",
-            __func__, mdb_strerror(code)
-        );
-        goto end;
-    }
-
-    ret = mdb_del(transaction, db->dbi, &key, &empty) == 0;
-    mdb_txn_commit(transaction);
-end:
-    LMDBKillKey(key);
-    pthread_mutex_unlock(&d->lock);
-    return ret;
-}
-
-static bool
-LMDBUnlock(Db *d, DbRef *r)
-{
-    LMDBRef *ref = (LMDBRef *) r;
-    LMDB *db = (LMDB *) d;
-    FILE *fakestream;
-    Stream *stream;
-    MDB_val key, val;
-    bool ret = true;
-
-    if (!d || !r)
-    {
-        return false;
-    }
-
-    val.mv_data = NULL;
-    val.mv_size = 0;
-    if (ref->transaction && r->hint == DB_HINT_WRITE)
-    {
-        key = LMDBTranslateKey(r->name);
-
-        fakestream = open_memstream((char **) &val.mv_data, &val.mv_size);
-        stream = StreamFile(fakestream);
-        JsonEncode(r->json, stream, JSON_DEFAULT);
-        StreamFlush(stream);
-        StreamClose(stream);
-
-        ret = mdb_put(ref->transaction, db->dbi, &key, &val, 0) == 0;
-
-        mdb_txn_commit(ref->transaction);
-        LMDBKillKey(key);
-    }
-    StringArrayFree(ref->base.name);
-    JsonFree(ref->base.json);
-    Free(ref);
-
-    if (val.mv_data)
-    {
-        free(val.mv_data);
-    }
-    if (ret && r->hint == DB_HINT_WRITE)
-    {
-        pthread_mutex_unlock(&d->lock);
-    }
-    return ret;
-}
-static DbRef *
-LMDBCreate(Db *d, Array *k)
-{
-    LMDB *db = (LMDB *) d;
-    MDB_txn *transaction;
-    LMDBRef *ret = NULL;
-    MDB_val key, empty_json;
-    int code;
-    if (!d || !k)
-    {
-        return NULL;
-    }
-
-    pthread_mutex_lock(&d->lock);
-    key = LMDBTranslateKey(k);
-
-    /* create a txn */
-    if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0)
-    {
-        /* Very bad! */
-        Log(LOG_ERR,
-            "%s: could not begin transaction: %s",
-            __func__, mdb_strerror(code)
-        );
-        goto end;
-    }
-
-    empty_json.mv_size = 2;
-    empty_json.mv_data = "{}";
-    /* put data in it */
-    code = mdb_put(transaction, db->dbi, &key, &empty_json, MDB_NOOVERWRITE);
-    if (code == MDB_KEYEXIST)
-    {
-        mdb_txn_abort(transaction);
-        goto end;
-    }
-    else if (code == MDB_MAP_FULL)
-    {
-        Log(LOG_ERR, "%s: db is full", __func__);
-        mdb_txn_abort(transaction);
-        goto end;
-    }
-    else if (code != 0)
-    {
-        Log(LOG_ERR, "%s: mdb_put failure: %s", __func__, mdb_strerror(code));
-        mdb_txn_abort(transaction);
-        goto end;
-    }
-
-    ret = Malloc(sizeof(*ret));
-    DbRefInit(d, (DbRef *) ret);
-    /* TODO: Timestamp */
-    {
-        size_t i;
-        ret->base.name = ArrayCreate();
-        for (i = 0; i < ArraySize(k); i++)
-        {
-            char *ent = ArrayGet(k, i);
-            StringArrayAppend(ret->base.name, ent);
-        }
-    }
-    ret->base.hint = DB_HINT_WRITE;
-    ret->base.json = HashMapCreate();
-    ret->transaction = transaction;
-end:
-    if (!ret)
-    {
-        pthread_mutex_unlock(&d->lock);
-    }
-    LMDBKillKey(key);
-    return (DbRef *) ret;
-}
-
-static Array *
-LMDBList(Db *d, Array *k)
-{
-    LMDB *db = (LMDB *) d;
-    MDB_val key = { 0 };
-    MDB_val subKey;
-    MDB_val val;
-    Array *ret = NULL;
-
-    MDB_cursor *cursor;
-    MDB_cursor_op op = MDB_SET_RANGE;
-    MDB_txn *txn;
-    int code;
-
-    if (!d || !k)
-    {
-        return NULL;
-    }
-
-    pthread_mutex_lock(&d->lock);
-
-    /* Marked as read-only, as we just don't need to write anything 
-     * when listing */
-    if ((code = mdb_txn_begin(db->environ, NULL, MDB_RDONLY, &txn)) != 0)
-    {
-        /* Very bad! */
-        Log(LOG_ERR,
-            "%s: could not begin transaction: %s",
-            __func__, mdb_strerror(code)
-        );
-        goto end;
-    }
-    if ((code = mdb_cursor_open(txn, db->dbi, &cursor)) != 0)
-    {
-        Log(LOG_ERR,
-            "%s: could not get cursor: %s",
-            __func__, mdb_strerror(code)
-        );
-        mdb_txn_abort(txn);
-        goto end;
-    }
-
-    key = LMDBTranslateKey(k);
-
-    /* Small hack to get it to list subitems */
-    ((char *) key.mv_data)[0]++;
-
-    ret = ArrayCreate();
-    subKey = key;
-    while (mdb_cursor_get(cursor, &subKey, &val, op) == 0)
-    {
-        /* This searches by *increasing* order. The problem is that it may 
-         * extend to unwanted points. Since the values are sorted, we can 
-         * just exit if the subkey is incorrect. */
-        char *head = LMDBKeyHead(subKey);
-        if (!LMDBKeyStartsWith(subKey, key))
-        {
-            break;
-        }
-
-        StringArrayAppend(ret, head);
-        op = MDB_NEXT;
-    }
-
-    mdb_cursor_close(cursor);
-    mdb_txn_abort(txn);
-
-end:
-    LMDBKillKey(key);
-    pthread_mutex_unlock(&d->lock);
-    return ret;
-}
-
-/* Implementation functions */
-static void
-LMDBClose(Db *d)
-{
-    LMDB *db = (LMDB *) d;
-    if (!d)
-    {
-        return;
-    }
-
-    mdb_dbi_close(db->environ, db->dbi);
-    mdb_env_close(db->environ);
-}
-
-Db *
-DbOpenLMDB(char *dir, size_t size)
-{
-    /* TODO */
-    MDB_env *env = NULL;
-    MDB_txn *txn;
-    MDB_dbi dbi;
-    int code;
-    LMDB *db;
-    if (!dir || !size)
-    {
-        return NULL;
-    }
-
-    /* Try initialising LMDB */
-    if ((code = mdb_env_create(&env)) != 0)
-    {
-        Log(LOG_ERR, 
-            "%s: could not create LMDB env: %s",
-            __func__, mdb_strerror(code)
-        );
-        return NULL;
-    }
-    if ((code = mdb_env_set_mapsize(env, size) != 0))
-    {
-        Log(LOG_ERR, 
-            "%s: could not set mapsize to %lu: %s",
-            __func__, (unsigned long) size, 
-            mdb_strerror(code)
-        );
-        mdb_env_close(env);
-        return NULL;
-    }
-    mdb_env_set_maxdbs(env, 4);
-    if ((code = mdb_env_open(env, dir, MDB_NOTLS, 0644)) != 0)
-    {
-        Log(LOG_ERR, 
-            "%s: could not open LMDB env: %s",
-            __func__, mdb_strerror(code)
-        );
-        mdb_env_close(env);
-        return NULL;
-    }
-
-    /* Initialise a DBI */
-    {
-        if ((code = mdb_txn_begin(env, NULL, 0, &txn)) != 0)
-        {
-            Log(LOG_ERR,
-                "%s: could not begin transaction: %s",
-                __func__, mdb_strerror(code)
-            );
-            mdb_env_close(env);
-            return NULL;
-        }
-        if ((code = mdb_dbi_open(txn, "db", MDB_CREATE, &dbi)) != 0)
-        {
-            Log(LOG_ERR,
-                "%s: could not get transaction dbi: %s",
-                __func__, mdb_strerror(code)
-            );
-            mdb_txn_abort(txn);
-            mdb_env_close(env);
-            return NULL;
-        }
-        mdb_txn_commit(txn);
-    }
-
-
-    db = Malloc(sizeof(*db));
-    DbInit((Db *) db);
-    db->environ = env;
-    db->dbi = dbi;
-
-    db->base.lockFunc = LMDBLock;
-    db->base.create = LMDBCreate;
-    db->base.unlock = LMDBUnlock;
-    db->base.delete = LMDBDelete;
-    db->base.exists = LMDBExists;
-    db->base.close = LMDBClose;
-    db->base.list = LMDBList;
-    
-    return (Db *) db;
-}
-
-#else
-Db *
-DbOpenLMDB(char *dir, size_t size)
-{
-    /* Unimplemented function */
-    Log(LOG_ERR, 
-        "LMDB support is not enabled. Please compile with --use-lmdb"
-    );
-    (void) size;
-    (void) dir;
-    return NULL;
-}
-#endif