Merge pull request 'Add the DbOperationArgs set of functions, and fix bug with LMDB' (#62) from lda/Cytoplasm:db-args into master

Reviewed-on: Telodendria/Cytoplasm#62

.forgejo/workflows/compile.yaml

@@ -0,0 +1,18 @@
+name: Compile Cytoplasm
+run-name: Compile Cytoplasm on ${{ forgejo.actor }}
+on: [push, pull_request]
+
+jobs:
+    "Compile Cytoplasm":
+        strategy:
+            matrix:
+                os: [alpine]
+                arch: [aarch64]
+        runs-on: ["${{ matrix.os }}", "${{ matrix.arch }}"]
+        steps:
+            - name: Check out repository
+              uses: actions/checkout@v4
+            - name: Configure Cytoplasm
+              run: ./configure
+            - name: Build Cytoplasm
+              run: make

.gitea/workflows/compile.yaml

@@ -1,25 +0,0 @@
-name: Compile Cytoplasm
-run-name: Compile Cytoplasm on ${{ gitea.actor }}
-on: [push]
-
-jobs:
-    "Compile Cytoplasm":
-        strategy:
-            matrix:
-                os: [debian-v12.4, alpine-v3.19, openbsd-v7.4, freebsd-v14.0, netbsd-v9.3]
-                arch: [x86, x86_64]
-                exclude:
-                    # 32-bit OpenBSD does not behave well in QEMU. Even when using
-                    # QEMU to emulate i386, it utilizes 100% of its CPU core and is
-                    # still extremely sluggish. Thus, we don't have a working 32-bit
-                    # OpenBSD runner, so exclude it from the matrix configuration.
-                    - os: openbsd-v7.4
-                      arch: x86
-        runs-on: ["${{ matrix.os }}", "${{ matrix.arch }}"]
-        steps:
-            - name: Check out repository
-              uses: actions/checkout@v3
-            - name: Configure Cytoplasm
-              run: ./configure
-            - name: Build Cytoplasm
-              run: make

CHANGELOG.md

@@ -21,6 +21,8 @@ are not even initialized to a default value.
 - Make `HttpRouter` decode path parts before matching them on regular expressions.
 - Fixed a bug in `ArraySort()` that would crash programs if the passed array has no
   elements.
+- Adds `Db[OP]Args` functions that are equivalent to their `Db[OP]` counter parts, but 
+  uses an array of string instead of variadic arguments.
 
 ## v0.4.0
 

configure

@@ -15,7 +15,7 @@ TOOLS="tools"
 
 # Default compiler flags. These must be supported by all POSIX C compilers.
 # "Fancy" compilers that have additional options must be detected and set below.
-CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE}"
+CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE} -I${SRC}";
 LIBS="-lm -lpthread"
 
 # Default args for all platforms.
@@ -24,10 +24,10 @@ SCRIPT_ARGS="--prefix=/usr/local --lib-name=Cytoplasm"
 # Set SSL flags depending on the platform.
 case "$(uname)" in
     OpenBSD)
-        SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl"
+        SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl --disable-lmdb"
         ;;
     *)
-        SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl"
+        SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl --disable-lmdb"
         ;;
 esac
 
@@ -37,7 +37,7 @@ case "$(uname)" in
         # These systems typically use GCC.
         SCRIPT_ARGS="${SCRIPT_ARGS} --cc=gcc"
         ;;
-    OpenBSD|FreeBSD)
+    OpenBSD|FreeBSD|Darwin)
         # These systems typically use Clang.
         SCRIPT_ARGS="${SCRIPT_ARGS} --cc=clang"
         ;;
@@ -80,6 +80,14 @@ for arg in $SCRIPT_ARGS; do
             TLS_IMPL=""
             TLS_LIBS=""
             ;;
+        --with-lmdb)
+            EDB_IMPL="EDB_LMDB"
+            EDB_LIBS="-llmdb"
+            ;;
+        --disable-lmdb)
+            EDB_IMPL=""
+            EDB_LIBS=""
+            ;;
         --prefix=*)
             PREFIX=$(echo "$arg" | cut -d '=' -f 2-)
             ;;
@@ -104,6 +112,11 @@ if [ -n "$TLS_IMPL" ]; then
     LIBS="${LIBS} ${TLS_LIBS}"
 fi
 
+if [ -n "$EDB_IMPL" ]; then
+    CFLAGS="${CFLAGS} -D${EDB_IMPL}"
+    LIBS="${LIBS} ${EDB_LIBS}"
+fi
+
 CFLAGS="${CFLAGS} '-DLIB_NAME=\"${LIB_NAME}\"' ${DEBUG}"
 LDFLAGS="${LIBS} ${LDFLAGS}"
 
@@ -140,7 +153,7 @@ print_obj() {
 get_deps() {
     src="$1"
 
-    ${CC} -I${INCLUDE} -E "$src" \
+    ${CC} -I${SRC} -I${INCLUDE} -E "$src" \
         | grep '^#'              \
         | awk '{print $3}'       \
         | cut -d '"' -f 2        \
@@ -256,8 +269,10 @@ ${TAB}done
 ${LIB_NAME}: ${OUT}/lib/lib${LIB_NAME}.a ${OUT}/lib/lib${LIB_NAME}.so
 
 install: ${LIB_NAME}
-${TAB}install -D ${OUT}/lib/lib${LIB_NAME}.a \$(PREFIX)/lib/lib${LIB_NAME}.a
+${TAB}mkdir -p \$(PREFIX)/${OUT}/lib
-${TAB}install -D ${OUT}/lib/lib${LIB_NAME}.so \$(PREFIX)/lib/lib${LIB_NAME}.so
+${TAB}mkdir -p \$(PREFIX)/lib
+${TAB}cp ${OUT}/lib/lib${LIB_NAME}.a \$(PREFIX)/lib/lib${LIB_NAME}.a
+${TAB}cp ${OUT}/lib/lib${LIB_NAME}.so \$(PREFIX)/lib/lib${LIB_NAME}.so
 $(collect ${INCLUDE}/ '' '' \$\(PREFIX\)/include/${LIB_NAME}/ install_out)
 $(collect ${INCLUDE}/ .h .3 \$\(PREFIX\)/man/man3/${LIB_NAME}- install_man)
 $(collect ${TOOLS}/ '.c' '' \$\(PREFIX\)/bin/ install_tool)

include/Cytoplasm/Db.h

@@ -48,6 +48,17 @@
  */
 typedef struct Db Db;
 
+/**
+ * Some "hints" for the database backend for operations. 
+ * Hints are a way for the program to declare what to except of it
+ * (and the program MUST adhere to these hints, but the backend 
+ * MAY adhere).
+ */
+typedef enum DbHint {
+    DB_HINT_READONLY,   /* The database reference is treated as read-only */
+    DB_HINT_WRITE       /* The database reference is treated as read/write */
+} DbHint;
+
 /**
  * When an object is locked, a reference is returned. This reference
  * is owned by the current thread, and the database is inaccessible to
@@ -68,6 +79,15 @@ typedef struct DbRef DbRef;
  */
 extern Db * DbOpen(char *, size_t);
 
+/**
+ * Open a LMDB data directory. This function is similar to 
+ * .Fn DbOpen ,
+ * but works with a LMDB-based backend, with the second argument 
+ * being the maximum filesize. This function fails with NULL if ever 
+ * called without LMDB enabled at compiletime.
+ */
+extern Db * DbOpenLMDB(char *, size_t);
+
 /**
  * Close the database. This function will flush anything in the cache
  * to the disk, and then close the data directory. It assumes that
@@ -99,6 +119,13 @@ extern void DbMaxCacheSet(Db *, size_t);
  */
 extern DbRef * DbCreate(Db *, size_t,...);
 
+/** 
+ * Behaves like 
+ * .Fn DbCreate ,
+ * but with an array of strings instead of variadic arguments. 
+ */
+extern DbRef * DbCreateArgs(Db *, Array *);
+
 /**
  * Lock an existing object in the database. This function will fail
  * if the object does not exist. It takes a variable number of C
@@ -109,6 +136,29 @@ extern DbRef * DbCreate(Db *, size_t,...);
  */
 extern DbRef * DbLock(Db *, size_t,...);
 
+/** 
+ * Behaves like 
+ * .Fn DbLock ,
+ * but with an array of strings instead of variadic arguments. 
+ */
+extern DbRef * DbLockArgs(Db *, Array *);
+
+/**
+ * Behaves like
+ * .Fn DbLock ,
+ * but with hints on the reference itself, as 
+ * .Fn DbLock 
+ * itself is read/write.
+ */
+extern DbRef * DbLockIntent(Db *, DbHint, size_t,...);
+
+/** 
+ * Behaves like 
+ * .Fn DbLockIntent ,
+ * but with an array of strings instead of variadic arguments.
+ */
+extern DbRef * DbLockIntentArgs(Db *, DbHint, Array *);
+
 /**
  * Immediately and permanently remove an object from the database.
  * This function assumes the object is not locked, otherwise undefined
@@ -116,6 +166,13 @@ extern DbRef * DbLock(Db *, size_t,...);
  */
 extern bool DbDelete(Db *, size_t,...);
 
+/** 
+ * Behaves like 
+ * .Fn DbDelete ,
+ * but with an array of strings instead of variadic arguments. 
+ */
+extern bool DbDeleteArgs(Db *, Array *);
+
 /**
  * Unlock an object and return it back to the database. This function
  * immediately syncs the object to the filesystem. The cache is a
@@ -133,6 +190,13 @@ extern bool DbUnlock(Db *, DbRef *);
  */
 extern bool DbExists(Db *, size_t,...);
 
+/** 
+ * Behaves like 
+ * .Fn DbExists ,
+ * but with an array of strings instead of variadic arguments.
+ */
+extern bool DbExistsArgs(Db *, Array *);
+
 /**
  * List all of the objects at a given path. Unlike the other varargs
  * functions, this one does not take a path to a specific object; it
@@ -145,6 +209,13 @@ extern bool DbExists(Db *, size_t,...);
  */
 extern Array * DbList(Db *, size_t,...);
 
+/** 
+ * Behaves like 
+ * .Fn DbList ,
+ * but with an array of strings instead of variadic arguments.
+ */
+extern Array * DbListArgs(Db *, Array *);
+
 /**
  * Free the list returned by 
  * .Fn DbListFree .

include/Cytoplasm/Json.h

@@ -297,6 +297,12 @@ extern size_t JsonEncode(HashMap *, Stream *, int);
  */
 extern HashMap * JsonDecode(Stream *);
 
+/**
+ * Decodes a JSON value from thr current input strram and parse it 
+ * into a JsonValue.
+ */
+extern JsonValue * JsonValueDecode(Stream *);
+
 /**
  * A convenience function that allows the caller to retrieve and
  * arbitrarily deep keys within a JSON object. It takes a root JSON

include/Cytoplasm/Log.h

@@ -46,7 +46,6 @@
 
 #define LOG_FLAG_COLOR  (1 << 0)
 #define LOG_FLAG_SYSLOG (1 << 1)
-#define LOG_FLAG_STDOUT (1 << 2)
 
 /**
  * A log is defined as a configuration that describes the properties
@@ -111,9 +110,15 @@ extern void LogConfigUnindent(LogConfig *);
 extern void LogConfigIndentSet(LogConfig *, size_t);
 
 /**
- * Set the file stream that logging output should be written to.
+ * Set the file stream that logging output should be written to. This
+ * defaults to standard output, but it can be set to standard error,
+ * or any other arbitrary stream. Passing a NULL value for the stream
+ * pointer sets the log output to the standard output. Note that the
+ * output stream is only used if
+ * .Va LOG_FLAG_SYSLOG
+ * is not set.
  */
-extern void LogConfigFileSet(LogConfig *, Stream *);
+extern void LogConfigOutputSet(LogConfig *, Stream *);
 
 /**
  * Set a number of boolean options on a log configuration. This
@@ -130,7 +135,12 @@ extern void LogConfigFileSet(LogConfig *, Stream *);
  * is checked before writing any terminal sequences.
  * .It LOG_FLAG_SYSLOG
  * When set, log output to the syslog using
- * .Xr syslog 3
+ * .Xr syslog 3 ,
+ * instead of logging to the file set by
+ * .Fn LogConfigOutputSet .
+ * This flag always overrides the stream set by that function,
+ * regardless of when it was set, even if it was set after this flag
+ * was set.
  * .El
  */
 extern void LogConfigFlagSet(LogConfig *, int);

include/Cytoplasm/Platform.h

@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef CYTOPLASM_PLATFORM_H
+#define CYTOPLASM_PLATFORM_H
+
+/***
+ * @Nm Platform
+ * @Nd A simple macro header that determines what platform the application
+ * is being built for.
+ * @Dd September 21, 2024
+ */
+
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
+    #define PLATFORM_WINDOWS
+
+   #ifdef _WIN64
+      #define PLATFORM_WIN64
+   #else
+      #define PLATFORM_WIN32
+   #endif
+#elif __APPLE__
+    #define PLATFORM_DARWIN
+
+    #include <TargetConditionals.h>
+    #if TARGET_IPHONE_SIMULATOR
+        #define PLATFORM_IPHONE
+    #elif TARGET_OS_MACCATALYST
+        #define PLATFORM_CATALYST
+    #elif TARGET_OS_IPHONE
+        #define PLATFORM_IPHONE
+    #elif TARGET_OS_MAC
+        #define PLATFORM_MAC
+    #else
+    #   error "Unknown Apple platform"
+    #endif
+#elif __ANDROID__
+    #define PLATFORM_ANDROID
+#elif __linux__
+    #define PLATFORM_LINUX
+#elif __unix__
+    #define PLATFORM_UNIX
+#elif defined(_POSIX_VERSION)
+    #define PLATFORM_POSIX
+#else
+#   error "Unknown compiler"
+#endif
+
+#endif /* CYTOPLASM_PLATFORM_H */

include/Cytoplasm/Sha.h

@@ -41,6 +41,16 @@
  * due to the lack of a 64-bit integer type, so that hash
  * function has been omitted.
  */
+#include <stddef.h>
+
+/**
+ * This is an enum to be used to identify the type of SHA used.
+ */
+typedef enum HashType
+{
+    HASH_SHA1,
+    HASH_SHA256
+} HashType;
 
 /**
  * This function takes a pointer to a NULL-terminated C string, and
@@ -64,6 +74,20 @@ extern unsigned char * Sha256(char *);
  */
 extern unsigned char * Sha1(char *);
 
+/**
+ * This function behaves just like 
+ * .Fn Sha256 ,
+ * except that it allows for a generic byte array, instead of a string.
+ */
+extern unsigned char * Sha256Raw(unsigned char *, size_t);
+
+/**
+ * This function behaves just like 
+ * .Fn Sha1 ,
+ * except that it allows for a generic byte array, instead of a string.
+ */
+extern unsigned char * Sha1Raw(unsigned char *, size_t);
+
 /**
  * Convert a SHA byte buffer into a hex string. These hex strings
  * are typically what is transmitted, stored, and compared, however
@@ -71,6 +95,6 @@ extern unsigned char * Sha1(char *);
  * bytes directly, which is why the conversion to a hex string is
  * a separate step.
  */
-extern char * ShaToHex(unsigned char *);
+extern char * ShaToHex(unsigned char *, HashType);
 
 #endif                             /* CYTOPLASM_SHA_H */

src/Db.c

@@ -1,977 +0,0 @@
-/*
- * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation files
- * (the "Software"), to deal in the Software without restriction,
- * including without limitation the rights to use, copy, modify, merge,
- * publish, distribute, sublicense, and/or sell copies of the Software,
- * and to permit persons to whom the Software is furnished to do so,
- * subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#include <Db.h>
-
-#include <Memory.h>
-#include <Json.h>
-#include <Util.h>
-#include <Str.h>
-#include <Stream.h>
-#include <Log.h>
-
-#include <sys/types.h>
-#include <dirent.h>
-
-#include <pthread.h>
-#include <fcntl.h>
-#include <unistd.h>
-
-#include <errno.h>
-#include <string.h>
-
-struct Db
-{
-    char *dir;
-    pthread_mutex_t lock;
-
-    size_t cacheSize;
-    size_t maxCache;
-    HashMap *cache;
-
-    /*
-     * The cache uses a double linked list (see DbRef
-     * below) to know which objects are most and least
-     * recently used. The following diagram helps me
-     * know what way all the pointers go, because it
-     * can get very confusing sometimes. For example,
-     * there's nothing stopping "next" from pointing to
-     * least recent, and "prev" from pointing to most
-     * recent, so hopefully this clarifies the pointer
-     * terminology used when dealing with the linked
-     * list:
-     *
-     *         mostRecent            leastRecent
-     *             |   prev       prev   |   prev
-     *           +---+ ---> +---+ ---> +---+ ---> NULL
-     *           |ref|      |ref|      |ref|
-     * NULL <--- +---+ <--- +---+ <--- +---+
-     *      next       next       next
-     */
-    DbRef *mostRecent;
-    DbRef *leastRecent;
-};
-
-struct DbRef
-{
-    HashMap *json;
-
-    uint64_t ts;
-    size_t size;
-
-    Array *name;
-
-    DbRef *prev;
-    DbRef *next;
-
-    int fd;
-    Stream *stream;
-};
-
-static void
-StringArrayFree(Array * arr)
-{
-    size_t i;
-
-    for (i = 0; i < ArraySize(arr); i++)
-    {
-        Free(ArrayGet(arr, i));
-    }
-
-    ArrayFree(arr);
-}
-
-static ssize_t DbComputeSize(HashMap *);
-
-static ssize_t
-DbComputeSizeOfValue(JsonValue * val)
-{
-    MemoryInfo *a;
-    ssize_t total = 0;
-
-    size_t i;
-
-    union
-    {
-        char *str;
-        Array *arr;
-    } u;
-
-    if (!val)
-    {
-        return -1;
-    }
-
-    a = MemoryInfoGet(val);
-    if (a)
-    {
-        total += MemoryInfoGetSize(a);
-    }
-
-    switch (JsonValueType(val))
-    {
-        case JSON_OBJECT:
-            total += DbComputeSize(JsonValueAsObject(val));
-            break;
-        case JSON_ARRAY:
-            u.arr = JsonValueAsArray(val);
-            a = MemoryInfoGet(u.arr);
-
-            if (a)
-            {
-                total += MemoryInfoGetSize(a);
-            }
-
-            for (i = 0; i < ArraySize(u.arr); i++)
-            {
-                total += DbComputeSizeOfValue(ArrayGet(u.arr, i));
-            }
-            break;
-        case JSON_STRING:
-            u.str = JsonValueAsString(val);
-            a = MemoryInfoGet(u.str);
-            if (a)
-            {
-                total += MemoryInfoGetSize(a);
-            }
-            break;
-        case JSON_NULL:
-        case JSON_INTEGER:
-        case JSON_FLOAT:
-        case JSON_BOOLEAN:
-        default:
-            /* These don't use any extra heap space */
-            break;
-    }
-    return total;
-}
-
-static ssize_t
-DbComputeSize(HashMap * json)
-{
-    char *key;
-    JsonValue *val;
-    MemoryInfo *a;
-    size_t total;
-
-    if (!json)
-    {
-        return -1;
-    }
-
-    total = 0;
-
-    a = MemoryInfoGet(json);
-    if (a)
-    {
-        total += MemoryInfoGetSize(a);
-    }
-
-    while (HashMapIterate(json, &key, (void **) &val))
-    {
-        a = MemoryInfoGet(key);
-        if (a)
-        {
-            total += MemoryInfoGetSize(a);
-        }
-
-        total += DbComputeSizeOfValue(val);
-    }
-
-    return total;
-}
-
-static char *
-DbHashKey(Array * args)
-{
-    size_t i;
-    char *str = NULL;
-
-    for (i = 0; i < ArraySize(args); i++)
-    {
-        char *tmp = StrConcat(2, str, ArrayGet(args, i));
-
-        Free(str);
-        str = tmp;
-    }
-
-    return str;
-}
-
-static char
-DbSanitiseChar(char input)
-{
-    switch (input)
-    {
-        case '/':
-            return '_';
-        case '.':
-            return '-';
-    }
-    return input;
-}
-
-static char *
-DbDirName(Db * db, Array * args, size_t strip)
-{
-    size_t i, j;
-    char *str = StrConcat(2, db->dir, "/");
-
-    for (i = 0; i < ArraySize(args) - strip; i++)
-    {
-        char *tmp;
-        char *sanitise = StrDuplicate(ArrayGet(args, i));
-        for (j = 0; j < strlen(sanitise); j++)
-        {
-            sanitise[j] = DbSanitiseChar(sanitise[j]);
-        }
-
-        tmp = StrConcat(3, str, sanitise, "/");
-
-        Free(str);
-        Free(sanitise);
-
-        str = tmp;
-    }
-
-    return str;
-}
-
-static char *
-DbFileName(Db * db, Array * args)
-{
-    size_t i;
-    char *str = StrConcat(2, db->dir, "/");
-
-    for (i = 0; i < ArraySize(args); i++)
-    {
-        char *tmp;
-        char *arg = StrDuplicate(ArrayGet(args, i));
-        size_t j = 0;
-
-        /* Sanitize name to prevent directory traversal attacks */
-        while (arg[j])
-        {
-            arg[j] = DbSanitiseChar(arg[j]);
-            j++;
-        }
-
-        tmp = StrConcat(3, str, arg,
-                        (i < ArraySize(args) - 1) ? "/" : ".json");
-
-        Free(arg);
-        Free(str);
-
-        str = tmp;
-    }
-
-    return str;
-}
-
-static void
-DbCacheEvict(Db * db)
-{
-    DbRef *ref = db->leastRecent;
-    DbRef *tmp;
-
-    while (ref && db->cacheSize > db->maxCache)
-    {
-        char *hash;
-
-        JsonFree(ref->json);
-
-        hash = DbHashKey(ref->name);
-        HashMapDelete(db->cache, hash);
-        Free(hash);
-
-        StringArrayFree(ref->name);
-
-        db->cacheSize -= ref->size;
-
-        if (ref->next)
-        {
-            ref->next->prev = ref->prev;
-        }
-        else
-        {
-            db->mostRecent = ref->prev;
-        }
-
-        if (ref->prev)
-        {
-            ref->prev->next = ref->next;
-        }
-        else
-        {
-            db->leastRecent = ref->next;
-        }
-
-        tmp = ref->next;
-        Free(ref);
-
-        ref = tmp;
-    }
-}
-
-Db *
-DbOpen(char *dir, size_t cache)
-{
-    Db *db;
-    pthread_mutexattr_t attr;
-
-    if (!dir)
-    {
-        return NULL;
-    }
-
-    db = Malloc(sizeof(Db));
-    if (!db)
-    {
-        return NULL;
-    }
-
-    db->dir = dir;
-    db->maxCache = cache;
-
-    pthread_mutexattr_init(&attr);
-    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
-    pthread_mutex_init(&db->lock, &attr);
-    pthread_mutexattr_destroy(&attr);
-
-    db->mostRecent = NULL;
-    db->leastRecent = NULL;
-    db->cacheSize = 0;
-
-    if (db->maxCache)
-    {
-        db->cache = HashMapCreate();
-        if (!db->cache)
-        {
-            return NULL;
-        }
-    }
-    else
-    {
-        db->cache = NULL;
-    }
-
-    return db;
-}
-
-void
-DbMaxCacheSet(Db * db, size_t cache)
-{
-    if (!db)
-    {
-        return;
-    }
-
-    pthread_mutex_lock(&db->lock);
-
-    db->maxCache = cache;
-    if (db->maxCache && !db->cache)
-    {
-        db->cache = HashMapCreate();
-        db->cacheSize = 0;
-    }
-
-    DbCacheEvict(db);
-
-    pthread_mutex_unlock(&db->lock);
-}
-
-void
-DbClose(Db * db)
-{
-    if (!db)
-    {
-        return;
-    }
-
-    pthread_mutex_lock(&db->lock);
-
-    DbMaxCacheSet(db, 0);
-    DbCacheEvict(db);
-    HashMapFree(db->cache);
-
-    pthread_mutex_unlock(&db->lock);
-    pthread_mutex_destroy(&db->lock);
-
-    Free(db);
-}
-
-static DbRef *
-DbLockFromArr(Db * db, Array * args)
-{
-    char *file;
-    char *hash;
-    DbRef *ref;
-    struct flock lock;
-
-    int fd;
-    Stream *stream;
-
-    if (!db || !args)
-    {
-        return NULL;
-    }
-
-    ref = NULL;
-    hash = NULL;
-
-    pthread_mutex_lock(&db->lock);
-
-    /* Check if the item is in the cache */
-    hash = DbHashKey(args);
-    ref = HashMapGet(db->cache, hash);
-    file = DbFileName(db, args);
-
-    fd = open(file, O_RDWR);
-    if (fd == -1)
-    {
-        if (ref)
-        {
-            HashMapDelete(db->cache, hash);
-            JsonFree(ref->json);
-            StringArrayFree(ref->name);
-
-            db->cacheSize -= ref->size;
-
-            if (ref->next)
-            {
-                ref->next->prev = ref->prev;
-            }
-            else
-            {
-                db->mostRecent = ref->prev;
-            }
-
-            if (ref->prev)
-            {
-                ref->prev->next = ref->next;
-            }
-            else
-            {
-                db->leastRecent = ref->next;
-            }
-
-            if (!db->leastRecent)
-            {
-                db->leastRecent = db->mostRecent;
-            }
-
-            Free(ref);
-        }
-        ref = NULL;
-        goto finish;
-    }
-
-    stream = StreamFd(fd);
-
-    lock.l_start = 0;
-    lock.l_len = 0;
-    lock.l_type = F_WRLCK;
-    lock.l_whence = SEEK_SET;
-
-    /* Lock the file on the disk */
-    if (fcntl(fd, F_SETLK, &lock) < 0)
-    {
-        StreamClose(stream);
-        ref = NULL;
-        goto finish;
-    }
-
-    if (ref)                       /* In cache */
-    {
-        uint64_t diskTs = UtilLastModified(file);
-
-        ref->fd = fd;
-        ref->stream = stream;
-
-        if (diskTs > ref->ts)
-        {
-            /* File was modified on disk since it was cached */
-            HashMap *json = JsonDecode(ref->stream);
-
-            if (!json)
-            {
-                StreamClose(ref->stream);
-                ref = NULL;
-                goto finish;
-            }
-
-            JsonFree(ref->json);
-            ref->json = json;
-            ref->ts = diskTs;
-            ref->size = DbComputeSize(ref->json);
-        }
-
-        /* Float this ref to mostRecent */
-        if (ref->next)
-        {
-            ref->next->prev = ref->prev;
-
-            if (!ref->prev)
-            {
-                db->leastRecent = ref->next;
-            }
-            else
-            {
-                ref->prev->next = ref->next;
-            }
-
-            ref->prev = db->mostRecent;
-            ref->next = NULL;
-            if (db->mostRecent)
-            {
-                db->mostRecent->next = ref;
-            }
-            db->mostRecent = ref;
-        }
-
-        /* If there is no least recent, this is the only thing in the
-         * cache, so it is also least recent. */
-        if (!db->leastRecent)
-        {
-            db->leastRecent = ref;
-        }
-
-        /* The file on disk may be larger than what we have in memory,
-         * which may require items in cache to be evicted. */
-        DbCacheEvict(db);
-    }
-    else
-    {
-        Array *name;
-        size_t i;
-
-        /* Not in cache; load from disk */
-
-        ref = Malloc(sizeof(DbRef));
-        if (!ref)
-        {
-            StreamClose(stream);
-            goto finish;
-        }
-
-        ref->json = JsonDecode(stream);
-
-        if (!ref->json)
-        {
-            Free(ref);
-            StreamClose(stream);
-            ref = NULL;
-            goto finish;
-        }
-
-        ref->fd = fd;
-        ref->stream = stream;
-
-        name = ArrayCreate();
-        for (i = 0; i < ArraySize(args); i++)
-        {
-            ArrayAdd(name, StrDuplicate(ArrayGet(args, i)));
-        }
-        ref->name = name;
-
-        if (db->cache)
-        {
-            ref->ts = UtilTsMillis();
-            ref->size = DbComputeSize(ref->json);
-            HashMapSet(db->cache, hash, ref);
-            db->cacheSize += ref->size;
-
-            ref->next = NULL;
-            ref->prev = db->mostRecent;
-            if (db->mostRecent)
-            {
-                db->mostRecent->next = ref;
-            }
-            db->mostRecent = ref;
-
-            if (!db->leastRecent)
-            {
-                db->leastRecent = ref;
-            }
-
-            /* Adding this item to the cache may case it to grow too
-             * large, requiring some items to be evicted */
-            DbCacheEvict(db);
-        }
-    }
-
-finish:
-    if (!ref)
-    {
-        pthread_mutex_unlock(&db->lock);
-    }
-
-    Free(file);
-    Free(hash);
-
-    return ref;
-}
-
-DbRef *
-DbCreate(Db * db, size_t nArgs,...)
-{
-    Stream *fp;
-    char *file;
-    char *dir;
-    va_list ap;
-    Array *args;
-    DbRef *ret;
-
-    if (!db)
-    {
-        return NULL;
-    }
-
-    va_start(ap, nArgs);
-    args = ArrayFromVarArgs(nArgs, ap);
-    va_end(ap);
-
-    if (!args)
-    {
-        return NULL;
-    }
-
-    pthread_mutex_lock(&db->lock);
-
-    file = DbFileName(db, args);
-
-    if (UtilLastModified(file))
-    {
-        Free(file);
-        ArrayFree(args);
-        pthread_mutex_unlock(&db->lock);
-        return NULL;
-    }
-
-    dir = DbDirName(db, args, 1);
-    if (UtilMkdir(dir, 0750) < 0)
-    {
-        Free(file);
-        ArrayFree(args);
-        Free(dir);
-        pthread_mutex_unlock(&db->lock);
-        return NULL;
-    }
-
-    Free(dir);
-
-    fp = StreamOpen(file, "w");
-    Free(file);
-    if (!fp)
-    {
-        ArrayFree(args);
-        pthread_mutex_unlock(&db->lock);
-        return NULL;
-    }
-
-    StreamPuts(fp, "{}");
-    StreamClose(fp);
-
-    /* DbLockFromArr() will lock again for us */
-    pthread_mutex_unlock(&db->lock);
-
-    ret = DbLockFromArr(db, args);
-
-    ArrayFree(args);
-
-    return ret;
-}
-
-bool
-DbDelete(Db * db, size_t nArgs,...)
-{
-    va_list ap;
-    Array *args;
-    char *file;
-    char *hash;
-    bool ret = true;
-    DbRef *ref;
-
-    if (!db)
-    {
-        return false;
-    }
-
-    va_start(ap, nArgs);
-    args = ArrayFromVarArgs(nArgs, ap);
-    va_end(ap);
-
-    pthread_mutex_lock(&db->lock);
-
-    hash = DbHashKey(args);
-    file = DbFileName(db, args);
-
-    ref = HashMapGet(db->cache, hash);
-    if (ref)
-    {
-        HashMapDelete(db->cache, hash);
-        JsonFree(ref->json);
-        StringArrayFree(ref->name);
-
-        db->cacheSize -= ref->size;
-
-        if (ref->next)
-        {
-            ref->next->prev = ref->prev;
-        }
-        else
-        {
-            db->mostRecent = ref->prev;
-        }
-
-        if (ref->prev)
-        {
-            ref->prev->next = ref->next;
-        }
-        else
-        {
-            db->leastRecent = ref->next;
-        }
-
-        if (!db->leastRecent)
-        {
-            db->leastRecent = db->mostRecent;
-        }
-
-        Free(ref);
-    }
-
-    Free(hash);
-
-    if (UtilLastModified(file))
-    {
-        ret = (remove(file) == 0);
-    }
-
-    pthread_mutex_unlock(&db->lock);
-
-    ArrayFree(args);
-    Free(file);
-    return ret;
-}
-
-DbRef *
-DbLock(Db * db, size_t nArgs,...)
-{
-    va_list ap;
-    Array *args;
-    DbRef *ret;
-
-    va_start(ap, nArgs);
-    args = ArrayFromVarArgs(nArgs, ap);
-    va_end(ap);
-
-    if (!args)
-    {
-        return NULL;
-    }
-
-    ret = DbLockFromArr(db, args);
-
-    ArrayFree(args);
-
-    return ret;
-}
-
-bool
-DbUnlock(Db * db, DbRef * ref)
-{
-    bool destroy;
-
-    if (!db || !ref)
-    {
-        return false;
-    }
-
-    lseek(ref->fd, 0L, SEEK_SET);
-    if (ftruncate(ref->fd, 0) < 0)
-    {
-        pthread_mutex_unlock(&db->lock);
-        Log(LOG_ERR, "Failed to truncate file on disk.");
-        Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno));
-        return false;
-    }
-
-    JsonEncode(ref->json, ref->stream, JSON_DEFAULT);
-    StreamClose(ref->stream);
-
-    if (db->cache)
-    {
-        char *key = DbHashKey(ref->name);
-
-        if (HashMapGet(db->cache, key))
-        {
-            db->cacheSize -= ref->size;
-            ref->size = DbComputeSize(ref->json);
-            db->cacheSize += ref->size;
-
-            /* If this ref has grown significantly since we last
-             * computed its size, it may have filled the cache and
-             * require some items to be evicted. */
-            DbCacheEvict(db);
-
-            destroy = false;
-        }
-        else
-        {
-            destroy = true;
-        }
-
-        Free(key);
-    }
-    else
-    {
-        destroy = true;
-    }
-
-    if (destroy)
-    {
-        JsonFree(ref->json);
-        StringArrayFree(ref->name);
-
-        Free(ref);
-    }
-
-    pthread_mutex_unlock(&db->lock);
-    return true;
-}
-
-bool
-DbExists(Db * db, size_t nArgs,...)
-{
-    va_list ap;
-    Array *args;
-    char *file;
-    bool ret;
-
-    va_start(ap, nArgs);
-    args = ArrayFromVarArgs(nArgs, ap);
-    va_end(ap);
-
-    if (!args)
-    {
-        return false;
-    }
-
-    pthread_mutex_lock(&db->lock);
-
-    file = DbFileName(db, args);
-    ret = (UtilLastModified(file) != 0);
-
-    pthread_mutex_unlock(&db->lock);
-
-    Free(file);
-    ArrayFree(args);
-
-    return ret;
-}
-
-Array *
-DbList(Db * db, size_t nArgs,...)
-{
-    Array *result;
-    Array *path;
-    DIR *files;
-    struct dirent *file;
-    char *dir;
-    va_list ap;
-
-    if (!db || !nArgs)
-    {
-        return NULL;
-    }
-
-    result = ArrayCreate();
-    if (!result)
-    {
-        return NULL;
-    }
-
-    va_start(ap, nArgs);
-    path = ArrayFromVarArgs(nArgs, ap);
-    dir = DbDirName(db, path, 0);
-
-    pthread_mutex_lock(&db->lock);
-
-    files = opendir(dir);
-    if (!files)
-    {
-        ArrayFree(path);
-        ArrayFree(result);
-        Free(dir);
-        pthread_mutex_unlock(&db->lock);
-        return NULL;
-    }
-    while ((file = readdir(files)))
-    {
-        size_t namlen = strlen(file->d_name);
-
-        if (namlen > 5)
-        {
-            int nameOffset = namlen - 5;
-
-            if (StrEquals(file->d_name + nameOffset, ".json"))
-            {
-                file->d_name[nameOffset] = '\0';
-                ArrayAdd(result, StrDuplicate(file->d_name));
-            }
-        }
-    }
-    closedir(files);
-
-    ArrayFree(path);
-    Free(dir);
-    pthread_mutex_unlock(&db->lock);
-
-    return result;
-}
-
-void
-DbListFree(Array * arr)
-{
-    StringArrayFree(arr);
-}
-
-HashMap *
-DbJson(DbRef * ref)
-{
-    return ref ? ref->json : NULL;
-}
-
-bool
-DbJsonSet(DbRef * ref, HashMap * json)
-{
-    if (!ref || !json)
-    {
-        return false;
-    }
-
-    JsonFree(ref->json);
-    ref->json = JsonDuplicate(json);
-    return true;
-}

src/Db/Db.c

@@ -0,0 +1,562 @@
+/*
+ * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include <Db.h>
+
+#include <Memory.h>
+#include <Json.h>
+#include <Util.h>
+#include <Str.h>
+#include <Stream.h>
+#include <Log.h>
+
+#include <sys/types.h>
+#include <dirent.h>
+
+#include <pthread.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <errno.h>
+#include <string.h>
+
+#include "Db/Internal.h"
+
+void
+StringArrayFree(Array * arr)
+{
+    size_t i;
+
+    for (i = 0; i < ArraySize(arr); i++)
+    {
+        Free(ArrayGet(arr, i));
+    }
+
+    ArrayFree(arr);
+}
+
+static ssize_t DbComputeSize(HashMap *);
+
+static ssize_t
+DbComputeSizeOfValue(JsonValue * val)
+{
+    MemoryInfo *a;
+    ssize_t total = 0;
+
+    size_t i;
+
+    union
+    {
+        char *str;
+        Array *arr;
+    } u;
+
+    if (!val)
+    {
+        return -1;
+    }
+
+    a = MemoryInfoGet(val);
+    if (a)
+    {
+        total += MemoryInfoGetSize(a);
+    }
+
+    switch (JsonValueType(val))
+    {
+        case JSON_OBJECT:
+            total += DbComputeSize(JsonValueAsObject(val));
+            break;
+        case JSON_ARRAY:
+            u.arr = JsonValueAsArray(val);
+            a = MemoryInfoGet(u.arr);
+
+            if (a)
+            {
+                total += MemoryInfoGetSize(a);
+            }
+
+            for (i = 0; i < ArraySize(u.arr); i++)
+            {
+                total += DbComputeSizeOfValue(ArrayGet(u.arr, i));
+            }
+            break;
+        case JSON_STRING:
+            u.str = JsonValueAsString(val);
+            a = MemoryInfoGet(u.str);
+            if (a)
+            {
+                total += MemoryInfoGetSize(a);
+            }
+            break;
+        case JSON_NULL:
+        case JSON_INTEGER:
+        case JSON_FLOAT:
+        case JSON_BOOLEAN:
+        default:
+            /* These don't use any extra heap space */
+            break;
+    }
+    return total;
+}
+
+static ssize_t
+DbComputeSize(HashMap * json)
+{
+    char *key;
+    JsonValue *val;
+    MemoryInfo *a;
+    size_t total;
+
+    if (!json)
+    {
+        return -1;
+    }
+
+    total = 0;
+
+    a = MemoryInfoGet(json);
+    if (a)
+    {
+        total += MemoryInfoGetSize(a);
+    }
+
+    while (HashMapIterate(json, &key, (void **) &val))
+    {
+        a = MemoryInfoGet(key);
+        if (a)
+        {
+            total += MemoryInfoGetSize(a);
+        }
+
+        total += DbComputeSizeOfValue(val);
+    }
+
+    return total;
+}
+
+static char *
+DbHashKey(Array * args)
+{
+    size_t i;
+    char *str = NULL;
+
+    for (i = 0; i < ArraySize(args); i++)
+    {
+        char *tmp = StrConcat(2, str, ArrayGet(args, i));
+
+        Free(str);
+        str = tmp;
+    }
+
+    return str;
+}
+
+static void
+DbCacheEvict(Db * db)
+{
+    DbRef *ref = db->leastRecent;
+    DbRef *tmp;
+
+    while (ref && db->cacheSize > db->maxCache)
+    {
+        char *hash;
+
+        JsonFree(ref->json);
+
+        hash = DbHashKey(ref->name);
+        HashMapDelete(db->cache, hash);
+        Free(hash);
+
+        StringArrayFree(ref->name);
+
+        db->cacheSize -= ref->size;
+
+        if (ref->next)
+        {
+            ref->next->prev = ref->prev;
+        }
+        else
+        {
+            db->mostRecent = ref->prev;
+        }
+
+        if (ref->prev)
+        {
+            ref->prev->next = ref->next;
+        }
+        else
+        {
+            db->leastRecent = ref->next;
+        }
+
+        tmp = ref->next;
+        Free(ref);
+
+        ref = tmp;
+    }
+}
+
+void
+DbMaxCacheSet(Db * db, size_t cache)
+{
+    if (!db)
+    {
+        return;
+    }
+
+    pthread_mutex_lock(&db->lock);
+
+    db->maxCache = cache;
+    if (db->maxCache && !db->cache)
+    {
+        db->cache = HashMapCreate();
+        db->cacheSize = 0;
+    }
+
+    DbCacheEvict(db);
+
+    pthread_mutex_unlock(&db->lock);
+}
+
+void
+DbClose(Db * db)
+{
+    if (!db)
+    {
+        return;
+    }
+
+    pthread_mutex_lock(&db->lock);
+    if (db->close)
+    {
+        db->close(db);
+    }
+    DbMaxCacheSet(db, 0);
+    DbCacheEvict(db);
+    HashMapFree(db->cache);
+
+    pthread_mutex_unlock(&db->lock);
+    pthread_mutex_destroy(&db->lock);
+
+    Free(db);
+}
+
+DbRef *
+DbCreateArgs(Db * db, Array *args)
+{
+    if (!args || !db || !db->create)
+    {
+        return NULL;
+    }
+
+    return db->create(db, args);
+}
+DbRef *
+DbCreate(Db * db, size_t nArgs,...)
+{
+    va_list ap;
+    Array *args;
+    DbRef *ret;
+
+    if (!db || !db->create)
+    {
+        return NULL;
+    }
+
+    va_start(ap, nArgs);
+    args = ArrayFromVarArgs(nArgs, ap);
+    va_end(ap);
+
+    if (!args)
+    {
+        return NULL;
+    }
+
+    ret = DbCreateArgs(db, args);
+    ArrayFree(args);
+
+    return ret;
+}
+
+bool
+DbDeleteArgs(Db * db, Array *args)
+{
+    if (!args || !db || !db->delete)
+    {
+        return false;
+    }
+
+    return db->delete(db, args);
+}
+bool
+DbDelete(Db * db, size_t nArgs,...)
+{
+    va_list ap;
+    Array *args;
+    bool ret = true;
+
+    if (!db)
+    {
+        return false;
+    }
+
+    va_start(ap, nArgs);
+    args = ArrayFromVarArgs(nArgs, ap);
+    va_end(ap);
+
+    ret = DbDeleteArgs(db, args);
+
+    ArrayFree(args);
+    return ret;
+}
+
+DbRef *
+DbLockArgs(Db * db, Array *args)
+{
+    if (!args || !db || !db->lockFunc)
+    {
+        return NULL;
+    }
+
+    return db->lockFunc(db, DB_HINT_WRITE, args);
+}
+DbRef *
+DbLock(Db * db, size_t nArgs,...)
+{
+    va_list ap;
+    Array *args;
+    DbRef *ret;
+
+    if (!db)
+    {
+        return NULL;
+    }
+
+    va_start(ap, nArgs);
+    args = ArrayFromVarArgs(nArgs, ap);
+    va_end(ap);
+
+    if (!args || !db->lockFunc)
+    {
+        return NULL;
+    }
+
+    ret = DbLockArgs(db, args);
+
+    ArrayFree(args);
+
+    return ret;
+}
+
+DbRef *
+DbLockIntentArgs(Db * db, DbHint hint, Array *args)
+{
+   if (!db || !args || !db->lockFunc)
+   {
+        return NULL;
+   }
+   return db->lockFunc(db, hint, args);
+}
+DbRef *
+DbLockIntent(Db * db, DbHint hint, size_t nArgs,...)
+{
+    va_list ap;
+    Array *args;
+    DbRef *ret;
+
+    va_start(ap, nArgs);
+    args = ArrayFromVarArgs(nArgs, ap);
+    va_end(ap);
+
+    if (!args || !db->lockFunc)
+    {
+        return NULL;
+    }
+
+    ret = DbLockIntentArgs(db, hint, args);
+
+    ArrayFree(args);
+
+    return ret;
+}
+
+bool
+DbUnlock(Db * db, DbRef * ref)
+{
+    if (!db || !ref || !db->unlock)
+    {
+        return false;
+    }
+    
+    return db->unlock(db, ref);
+}
+
+bool
+DbExistsArgs(Db * db, Array *args)
+{
+    if (!args || !db || !db->exists)
+    {
+        return false;
+    }
+
+    return db->exists(db, args);
+}
+bool
+DbExists(Db * db, size_t nArgs,...)
+{
+    va_list ap;
+    Array *args;
+    bool ret;
+    if (!db || !nArgs || !db->exists)
+    {
+        return false;
+    }
+
+    va_start(ap, nArgs);
+    args = ArrayFromVarArgs(nArgs, ap);
+    va_end(ap);
+
+    if (!args)
+    {
+        return false;
+    }
+
+    ret = DbExistsArgs(db, args);
+    ArrayFree(args);
+
+    return ret;
+}
+
+Array * 
+DbListArgs(Db *db, Array *args)
+{
+    if (!db || !args || !db->list)
+    {
+        return NULL;
+    }
+    return db->list(db, args);
+}
+Array *
+DbList(Db * db, size_t nArgs,...)
+{
+    Array *result;
+    Array *path;
+    va_list ap;
+
+    if (!db || !nArgs || !db->list)
+    {
+        return NULL;
+    }
+
+    va_start(ap, nArgs);
+    path = ArrayFromVarArgs(nArgs, ap);
+    va_end(ap);
+
+    result = DbListArgs(db, path);
+    
+    ArrayFree(path);
+    return result;
+}
+
+void
+DbListFree(Array * arr)
+{
+    StringArrayFree(arr);
+}
+
+HashMap *
+DbJson(DbRef * ref)
+{
+    return ref ? ref->json : NULL;
+}
+
+bool
+DbJsonSet(DbRef * ref, HashMap * json)
+{
+    if (!ref || !json)
+    {
+        return false;
+    }
+
+    JsonFree(ref->json);
+    ref->json = JsonDuplicate(json);
+    return true;
+}
+void
+DbInit(Db *db)
+{
+    pthread_mutexattr_t attr;
+    if (!db)
+    {
+        return;
+    }
+
+    pthread_mutexattr_init(&attr);
+    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
+    pthread_mutex_init(&db->lock, &attr);
+    pthread_mutexattr_destroy(&attr);
+
+    db->mostRecent = NULL;
+    db->leastRecent = NULL;
+    db->cacheSize = 0;
+    db->maxCache = 0;
+
+    if (db->maxCache)
+    {
+        db->cache = HashMapCreate();
+    }
+    else
+    {
+        db->cache = NULL;
+    }
+}
+void
+DbRefInit(Db *db, DbRef *ref)
+{
+    if (!db || !ref)
+    {
+        return;
+    }
+    ref->prev = db->mostRecent;
+    ref->next = NULL;
+    ref->json = NULL;
+    ref->name = NULL;
+
+    /* As default values to be overwritten by impls */
+    ref->ts = UtilTsMillis();
+    ref->size = 0;
+
+    /* TODO: Append the ref to the cache list.
+     * I removed it because it broke everything and crashed all the time.
+     * My bad! */
+}
+void
+StringArrayAppend(Array *arr, char *str)
+{
+    if (!arr || !str)
+    {
+        return;
+    }
+    ArrayAdd(arr, StrDuplicate(str));
+}

src/Db/Flat.c

@@ -0,0 +1,375 @@
+#include <Db.h>
+
+#include <Memory.h>
+#include <Json.h>
+#include <Util.h>
+#include <Str.h>
+#include <Stream.h>
+#include <Log.h>
+
+#include <sys/types.h>
+#include <dirent.h>
+
+#include <pthread.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <errno.h>
+#include <string.h>
+
+#include "Db/Internal.h"
+
+typedef struct FlatDb {
+    Db base;
+    char *dir;
+    /* Theres not much to do here. */
+} FlatDb;
+typedef struct FlatDbRef {
+    DbRef base;
+
+    Stream *stream;
+    int fd;
+} FlatDbRef;
+
+static char
+DbSanitiseChar(char input)
+{
+    switch (input)
+    {
+        case '/':
+            return '_';
+        case '.':
+            return '-';
+    }
+    return input;
+}
+
+static char *
+DbDirName(FlatDb * db, Array * args, size_t strip)
+{
+    size_t i, j;
+    char *str = StrConcat(2, db->dir, "/");
+
+    for (i = 0; i < ArraySize(args) - strip; i++)
+    {
+        char *tmp;
+        char *sanitise = StrDuplicate(ArrayGet(args, i));
+        size_t len = strlen(sanitise);
+        for (j = 0; j < len; j++)
+        {
+            sanitise[j] = DbSanitiseChar(sanitise[j]);
+        }
+
+        tmp = StrConcat(3, str, sanitise, "/");
+
+        Free(str);
+        Free(sanitise);
+
+        str = tmp;
+    }
+
+    return str;
+}
+static char *
+DbFileName(FlatDb * db, Array * args)
+{
+    size_t i;
+    char *str = StrConcat(2, db->dir, "/");
+
+    for (i = 0; i < ArraySize(args); i++)
+    {
+        char *tmp;
+        char *arg = StrDuplicate(ArrayGet(args, i));
+        size_t j = 0;
+
+        /* Sanitize name to prevent directory traversal attacks */
+        while (arg[j])
+        {
+            arg[j] = DbSanitiseChar(arg[j]);
+            j++;
+        }
+
+        tmp = StrConcat(3, str, arg,
+                        (i < ArraySize(args) - 1) ? "/" : ".json");
+
+        Free(arg);
+        Free(str);
+
+        str = tmp;
+    }
+
+    return str;
+}
+
+static DbRef *
+FlatLock(Db *d, DbHint hint, Array *dir)
+{
+    FlatDb *db = (FlatDb *) d;
+    FlatDbRef *ref = NULL;
+    size_t i;
+    char *path = NULL;
+    if (!d || !dir)
+    {
+        return NULL;
+    }
+
+    pthread_mutex_lock(&d->lock);
+    path = DbFileName(db, dir);
+    /* TODO: Caching */
+    {
+        int fd = open(path, O_RDWR);
+        Stream *stream;
+        struct flock lock;
+        if (fd == -1)
+        {
+            ref = NULL;
+            goto end;
+        }
+
+        stream = StreamFd(fd);
+        if (!stream)
+        {
+            ref = NULL;
+            goto end;
+        }
+        
+        lock.l_start = 0;
+        lock.l_len = 0;
+        lock.l_type = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+        if (fcntl(fd, F_SETLK, &lock) < 0)
+        {
+            StreamClose(stream);
+            ref = NULL;
+            goto end;
+        }
+
+        ref = Malloc(sizeof(*ref));
+        DbRefInit(d, (DbRef *) ref);
+        /* TODO: Hints */
+        ref->base.hint = hint;
+        ref->base.ts = UtilLastModified(path);
+        ref->base.json = JsonDecode(stream);
+        ref->stream = stream;
+        ref->fd = fd;
+        if (!ref->base.json)
+        {
+            Free(ref);
+            StreamClose(stream);
+            ref = NULL;
+            goto end;
+        }
+        
+        
+        ref->base.name = ArrayCreate();
+        for (i = 0; i < ArraySize(dir); i++)
+        {
+            StringArrayAppend(ref->base.name, ArrayGet(dir, i));
+        }
+    }
+end:
+    Free(path);
+    if (!ref)
+    {
+        pthread_mutex_unlock(&d->lock);
+    }
+    return (DbRef *) ref;
+}
+
+static bool
+FlatUnlock(Db *d, DbRef *r)
+{
+    FlatDbRef *ref = (FlatDbRef *) r;
+
+    if (!d || !r)
+    {
+        return false;
+    }
+
+    lseek(ref->fd, 0L, SEEK_SET);
+    if (ftruncate(ref->fd, 0) < 0)
+    {
+        pthread_mutex_unlock(&d->lock);
+        Log(LOG_ERR, "Failed to truncate file on disk.");
+        Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno));
+        return false;
+    }
+
+    JsonEncode(ref->base.json, ref->stream, JSON_DEFAULT);
+    StreamClose(ref->stream);
+
+    JsonFree(ref->base.json);
+    StringArrayFree(ref->base.name);
+    Free(ref);
+
+    pthread_mutex_unlock(&d->lock);
+    return true;
+}
+static DbRef *
+FlatCreate(Db *d, Array *dir)
+{
+    FlatDb *db = (FlatDb *) d;
+    char *path, *dirPath;
+    Stream *fp;
+    DbRef *ret;
+
+    if (!d || !dir)
+    {
+        return NULL;
+    }
+    pthread_mutex_lock(&d->lock);
+
+    path = DbFileName(db, dir);
+    if (UtilLastModified(path))
+    {
+        Free(path);
+        pthread_mutex_unlock(&d->lock);
+        return NULL;
+    }
+
+    dirPath = DbDirName(db, dir, 1);
+    if (UtilMkdir(dirPath, 0750) < 0)
+    {
+        Free(path);
+        Free(dirPath);
+        pthread_mutex_unlock(&d->lock);
+        return NULL;
+    }
+    Free(dirPath);
+
+    fp = StreamOpen(path, "w");
+    Free(path);
+    if (!fp)
+    {
+        pthread_mutex_unlock(&d->lock);
+        return NULL;
+    }
+
+    StreamPuts(fp, "{}");
+    StreamClose(fp);
+
+    /* FlatLock() will lock again for us */
+    pthread_mutex_unlock(&d->lock);
+    ret = FlatLock(d, DB_HINT_WRITE, dir);
+
+    return ret;
+}
+
+static bool
+FlatDelete(Db *d, Array *dir)
+{
+    bool ret = false;
+    char *file;
+    FlatDb *db = (FlatDb *) d;
+    if (!d || !dir)
+    {
+        return false;
+    }
+
+    pthread_mutex_lock(&d->lock);
+    file = DbFileName(db, dir);
+
+    /* TODO: Unlink the entry from the linkedlist */
+    if (UtilLastModified(file))
+    {
+        ret = remove(file) == 0;
+    }
+
+    Free(file);
+    pthread_mutex_unlock(&d->lock);
+    return ret;
+}
+
+static Array *
+FlatList(Db *d, Array *dir)
+{
+    FlatDb *db = (FlatDb *) d;
+    struct dirent *file;
+    Array *ret;
+    DIR *files;
+    char *path;
+
+    if (!d || !dir)
+    {
+        return NULL;
+    }
+
+    pthread_mutex_lock(&d->lock);
+
+    path = DbDirName(db, dir, 0);
+    files = opendir(path);
+    if (!files)
+    {
+        Free(path);
+        pthread_mutex_unlock(&d->lock);
+        return NULL;
+    }
+
+    ret = ArrayCreate();
+    while ((file = readdir(files)))
+    {
+        size_t namlen = strlen(file->d_name);
+
+        if (namlen > 5)
+        {
+            int nameOffset = namlen - 5;
+
+            if (StrEquals(file->d_name + nameOffset, ".json"))
+            {
+                file->d_name[nameOffset] = '\0';
+                StringArrayAppend(ret, file->d_name);
+            }
+        }
+    }
+    closedir(files);
+    Free(path);
+
+    pthread_mutex_unlock(&d->lock);
+    return ret;
+}
+static bool
+FlatExists(Db *d, Array *dir)
+{
+    FlatDb *db = (FlatDb *) d;
+    char *path;
+    bool ret;
+    if (!d || !dir)
+    {
+        return NULL;
+    }
+
+    pthread_mutex_lock(&d->lock);
+
+    path = DbFileName(db, dir);
+    ret = UtilLastModified(path) != 0;
+    Free(path);
+
+    pthread_mutex_unlock(&d->lock);
+
+    return ret;
+}
+
+Db *
+DbOpen(char *dir, size_t cache)
+{
+    FlatDb *db;
+    if (!dir)
+    {
+        return NULL;
+    }
+    db = Malloc(sizeof(*db));
+    DbInit((Db *) db);
+    db->dir = dir;
+    db->base.cacheSize = cache;
+
+    db->base.lockFunc = FlatLock;
+    db->base.unlock = FlatUnlock;
+    db->base.create = FlatCreate;
+    db->base.delete = FlatDelete;
+    db->base.exists = FlatExists;
+    db->base.list   = FlatList;
+    db->base.close = NULL;
+
+    return (Db *) db;
+}

src/Db/Internal.h

@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef CYTOPLASM_DB_INTERNAL_H
+#define CYTOPLASM_DB_INTERNAL_H
+
+#include <HashMap.h>
+#include <Db.h>
+
+#include <pthread.h>
+
+/* TODO: Define the base structures to define a database internally. 
+ * All "implementations" shall have them as a first element, so that 
+ * basic, general functions can work on them properly. */
+
+/* The base structure of a database */
+struct Db
+{
+    pthread_mutex_t lock;
+
+    size_t cacheSize;
+    size_t maxCache;
+    HashMap *cache;
+
+    /*
+     * The cache uses a double linked list (see DbRef
+     * below) to know which objects are most and least
+     * recently used. The following diagram helps me
+     * know what way all the pointers go, because it
+     * can get very confusing sometimes. For example,
+     * there's nothing stopping "next" from pointing to
+     * least recent, and "prev" from pointing to most
+     * recent, so hopefully this clarifies the pointer
+     * terminology used when dealing with the linked
+     * list:
+     *
+     *         mostRecent            leastRecent
+     *             |   prev       prev   |   prev
+     *           +---+ ---> +---+ ---> +---+ ---> NULL
+     *           |ref|      |ref|      |ref|
+     * NULL <--- +---+ <--- +---+ <--- +---+
+     *      next       next       next
+     */
+    DbRef *mostRecent;
+    DbRef *leastRecent;
+
+    /* Functions for implementation-specific operations
+     * (opening a ref, closing a db, removing an entry, ...) */
+    DbRef * (*lockFunc)(Db *, DbHint, Array *);
+    DbRef * (*create)(Db *, Array *);
+    Array * (*list)(Db *, Array *);
+    bool (*unlock)(Db *, DbRef *);
+    bool (*delete)(Db *, Array *);
+    bool (*exists)(Db *, Array *);
+    void (*close)(Db *);
+
+    /* Implementation-specific constructs */
+};
+
+struct DbRef
+{
+    HashMap *json;
+
+    uint64_t ts;
+    size_t size;
+
+    Array *name;
+
+    DbRef *prev;
+    DbRef *next;
+
+    DbHint hint;
+    /* Implementation-specific constructs */
+};
+
+extern void DbInit(Db *);
+extern void DbRefInit(Db *, DbRef *);
+extern void StringArrayFree(Array *);
+extern void StringArrayAppend(Array *, char *);
+
+#endif

src/Db/LMDB.c

@@ -0,0 +1,593 @@
+#include <Memory.h>
+#include <Json.h>
+#include <Log.h>
+#include <Db.h>
+
+#include "Db/Internal.h"
+
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+
+#ifdef EDB_LMDB
+
+#include <lmdb.h>
+
+typedef struct LMDB {
+    Db base; /* The base implementation required to pass */
+
+    MDB_env *environ;
+    MDB_dbi dbi;
+} LMDB;
+typedef struct LMDBRef {
+    DbRef base;
+
+    /* TODO: LMDB-dependent stuff */
+    MDB_txn *transaction;
+    MDB_dbi dbi;
+} LMDBRef;
+
+/* Helper functions */
+static MDB_val
+LMDBTranslateKey(Array *key)
+{
+    MDB_val ret = { 0 };
+    char *data = NULL;
+    size_t length = 0;
+    size_t i;
+    if (!key || ArraySize(key) > 255)
+    {
+        return ret;
+    }
+
+    data = Realloc(data, ++length);
+    data[0] = ArraySize(key);
+
+    /* Now, let's push every item */
+    for (i = 0; i < ArraySize(key); i++)
+    {
+        char *entry = ArrayGet(key, i);
+        size_t offset = length;
+
+        data = Realloc(data, (length += strlen(entry) + 1));
+        memcpy(data + offset, entry, strlen(entry));
+        data[length - 1] = '\0';
+    }
+
+    /* We now have every key */
+    ret.mv_size = length;
+    ret.mv_data = data;
+    return ret;
+}
+static void
+LMDBKillKey(MDB_val key)
+{
+    if (!key.mv_data || !key.mv_size)
+    {
+        return;
+    }
+
+    Free(key.mv_data);
+}
+static HashMap *
+LMDBDecode(MDB_val val)
+{
+    FILE *fakefile;
+    Stream *fakestream;
+    HashMap *ret;
+    if (!val.mv_data || !val.mv_size)
+    {
+        return NULL;
+    }
+
+    fakefile = fmemopen(val.mv_data, val.mv_size, "r");
+    fakestream = StreamFile(fakefile);
+    ret = JsonDecode(fakestream);
+    StreamClose(fakestream);
+
+    return ret;
+}
+static bool
+LMDBKeyStartsWith(MDB_val key, MDB_val starts)
+{
+    size_t i;
+    if (!key.mv_size || !starts.mv_size)
+    {
+        return false;
+    }
+    if (key.mv_size < starts.mv_size)
+    {
+        return false;
+    }
+
+    for (i = 0; i < starts.mv_size; i++)
+    {
+        char keyC = ((char *) key.mv_data)[i];
+        char startC = ((char *) starts.mv_data)[i];
+
+        if (keyC != startC)
+        {
+            return false;
+        }
+    }
+    return true;
+}
+static char *
+LMDBKeyHead(MDB_val key)
+{
+    char *end;
+    if (!key.mv_size || !key.mv_data)
+    {
+        return NULL;
+    }
+    /* -2 because we have a NULL byte in there */
+    end = ((char *) key.mv_data) + key.mv_size - 1;
+    if ((void *) end < key.mv_data)
+    {
+        /* Uh oh. */
+        return NULL;
+    }
+
+    /* Doing >= will lead to cases where it is sent straight to the 
+     * start. Don't do that. */
+    while ((void *) (end - 1) > key.mv_data && *(end - 1))
+    {
+        end--;
+    }
+    return end;
+}
+
+static DbRef *
+LMDBLock(Db *d, DbHint hint, Array *k)
+{
+    LMDB *db = (LMDB *) d;
+    MDB_txn *transaction;
+    LMDBRef *ret = NULL;
+    MDB_val key, json_val;
+    int code, flags;
+    if (!d || !k)
+    {
+        return NULL;
+    }
+
+    pthread_mutex_lock(&d->lock);
+    key = LMDBTranslateKey(k);
+
+    /* Create a transaction, honoring hints. */
+    /* TODO: Do we want to create a "main" transaction that everyone inherits 
+     * from? */
+    flags = hint == DB_HINT_READONLY ? MDB_RDONLY : 0;
+    if ((code = mdb_txn_begin(db->environ, NULL, flags, &transaction)) != 0)
+    {
+        /* Very bad! */
+        Log(LOG_ERR,
+            "%s: could not begin transaction: %s",
+            __func__, mdb_strerror(code)
+        );
+        goto end;
+    }
+
+    json_val.mv_size = 0;
+    json_val.mv_data = NULL;
+    code = mdb_get(transaction, db->dbi, &key, &json_val);
+    if (code == MDB_NOTFOUND)
+    {
+        mdb_txn_abort(transaction);
+        goto end;
+    }
+    else if (code != 0)
+    {
+        Log(LOG_ERR, 
+            "%s: mdb_get failure: %s", 
+            __func__, mdb_strerror(code)
+        );
+        mdb_txn_abort(transaction);
+        goto end;
+    }
+
+    ret = Malloc(sizeof(*ret));
+    DbRefInit(d, (DbRef *) ret);
+    /* TODO: Timestamp */
+    {
+        size_t i;
+        ret->base.name = ArrayCreate();
+        for (i = 0; i < ArraySize(k); i++)
+        {
+            char *ent = ArrayGet(k, i);
+            StringArrayAppend(ret->base.name, ent);
+        }
+    }
+    ret->base.json = LMDBDecode(json_val);
+    ret->base.hint = hint;
+    ret->transaction = NULL;
+
+    if (hint == DB_HINT_WRITE)
+    {
+        ret->transaction = transaction;
+    }
+    else
+    {
+        mdb_txn_abort(transaction);
+    }
+end:
+    if (!ret || hint == DB_HINT_READONLY)
+    {
+        pthread_mutex_unlock(&d->lock);
+    }
+    LMDBKillKey(key);
+    return (DbRef *) ret;
+}
+static bool
+LMDBExists(Db *d, Array *k)
+{
+    MDB_val key, empty;
+    LMDB *db = (LMDB *) d;
+    MDB_txn *transaction;
+    int code;
+    bool ret = false;
+    if (!d || !k)
+    {
+        return false;
+    }
+
+    pthread_mutex_lock(&d->lock);
+    key = LMDBTranslateKey(k);
+
+    /* create a txn */
+    if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0)
+    {
+        /* Very bad! */
+        Log(LOG_ERR,
+            "%s: could not begin transaction: %s",
+            __func__, mdb_strerror(code)
+        );
+        goto end;
+    }
+
+    ret = mdb_get(transaction, db->dbi, &key, &empty) == 0;
+    mdb_txn_abort(transaction);
+end:
+    LMDBKillKey(key);
+    pthread_mutex_unlock(&d->lock);
+    return ret;
+}
+static bool
+LMDBDelete(Db *d, Array *k)
+{
+    MDB_val key, empty;
+    LMDB *db = (LMDB *) d;
+    MDB_txn *transaction;
+    int code;
+    bool ret = false;
+    if (!d || !k)
+    {
+        return false;
+    }
+
+    pthread_mutex_lock(&d->lock);
+    key = LMDBTranslateKey(k);
+
+    /* create a txn */
+    if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0)
+    {
+        /* Very bad! */
+        Log(LOG_ERR,
+            "%s: could not begin transaction: %s",
+            __func__, mdb_strerror(code)
+        );
+        goto end;
+    }
+
+    ret = mdb_del(transaction, db->dbi, &key, &empty) == 0;
+    mdb_txn_commit(transaction);
+end:
+    LMDBKillKey(key);
+    pthread_mutex_unlock(&d->lock);
+    return ret;
+}
+
+static bool
+LMDBUnlock(Db *d, DbRef *r)
+{
+    LMDBRef *ref = (LMDBRef *) r;
+    LMDB *db = (LMDB *) d;
+    FILE *fakestream;
+    Stream *stream;
+    MDB_val key, val;
+    bool ret = true;
+    DbHint hint = r ? r->hint : 0;
+
+    if (!d || !r)
+    {
+        return false;
+    }
+
+    val.mv_data = NULL;
+    val.mv_size = 0;
+    if (ref->transaction && r->hint == DB_HINT_WRITE)
+    {
+        key = LMDBTranslateKey(r->name);
+
+        fakestream = open_memstream((char **) &val.mv_data, &val.mv_size);
+        stream = StreamFile(fakestream);
+        JsonEncode(r->json, stream, JSON_DEFAULT);
+        StreamFlush(stream);
+        StreamClose(stream);
+
+        ret = mdb_put(ref->transaction, db->dbi, &key, &val, 0) == 0;
+
+        mdb_txn_commit(ref->transaction);
+        LMDBKillKey(key);
+    }
+    StringArrayFree(ref->base.name);
+    JsonFree(ref->base.json);
+    Free(ref);
+
+    if (val.mv_data)
+    {
+        free(val.mv_data);
+    }
+    if (ret && hint == DB_HINT_WRITE)
+    {
+        pthread_mutex_unlock(&d->lock);
+    }
+    return ret;
+}
+static DbRef *
+LMDBCreate(Db *d, Array *k)
+{
+    LMDB *db = (LMDB *) d;
+    MDB_txn *transaction;
+    LMDBRef *ret = NULL;
+    MDB_val key, empty_json;
+    int code;
+    if (!d || !k)
+    {
+        return NULL;
+    }
+
+    pthread_mutex_lock(&d->lock);
+    key = LMDBTranslateKey(k);
+
+    /* create a txn */
+    if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0)
+    {
+        /* Very bad! */
+        Log(LOG_ERR,
+            "%s: could not begin transaction: %s",
+            __func__, mdb_strerror(code)
+        );
+        goto end;
+    }
+
+    empty_json.mv_size = 2;
+    empty_json.mv_data = "{}";
+    /* put data in it */
+    code = mdb_put(transaction, db->dbi, &key, &empty_json, MDB_NOOVERWRITE);
+    if (code == MDB_KEYEXIST)
+    {
+        mdb_txn_abort(transaction);
+        goto end;
+    }
+    else if (code == MDB_MAP_FULL)
+    {
+        Log(LOG_ERR, "%s: db is full", __func__);
+        mdb_txn_abort(transaction);
+        goto end;
+    }
+    else if (code != 0)
+    {
+        Log(LOG_ERR, "%s: mdb_put failure: %s", __func__, mdb_strerror(code));
+        mdb_txn_abort(transaction);
+        goto end;
+    }
+
+    ret = Malloc(sizeof(*ret));
+    DbRefInit(d, (DbRef *) ret);
+    /* TODO: Timestamp */
+    {
+        size_t i;
+        ret->base.name = ArrayCreate();
+        for (i = 0; i < ArraySize(k); i++)
+        {
+            char *ent = ArrayGet(k, i);
+            StringArrayAppend(ret->base.name, ent);
+        }
+    }
+    ret->base.hint = DB_HINT_WRITE;
+    ret->base.json = HashMapCreate();
+    ret->transaction = transaction;
+end:
+    if (!ret)
+    {
+        pthread_mutex_unlock(&d->lock);
+    }
+    LMDBKillKey(key);
+    return (DbRef *) ret;
+}
+
+static Array *
+LMDBList(Db *d, Array *k)
+{
+    LMDB *db = (LMDB *) d;
+    MDB_val key = { 0 };
+    MDB_val subKey;
+    MDB_val val;
+    Array *ret = NULL;
+
+    MDB_cursor *cursor;
+    MDB_cursor_op op = MDB_SET_RANGE;
+    MDB_txn *txn;
+    int code;
+
+    if (!d || !k)
+    {
+        return NULL;
+    }
+
+    pthread_mutex_lock(&d->lock);
+
+    /* Marked as read-only, as we just don't need to write anything 
+     * when listing */
+    if ((code = mdb_txn_begin(db->environ, NULL, MDB_RDONLY, &txn)) != 0)
+    {
+        /* Very bad! */
+        Log(LOG_ERR,
+            "%s: could not begin transaction: %s",
+            __func__, mdb_strerror(code)
+        );
+        goto end;
+    }
+    if ((code = mdb_cursor_open(txn, db->dbi, &cursor)) != 0)
+    {
+        Log(LOG_ERR,
+            "%s: could not get cursor: %s",
+            __func__, mdb_strerror(code)
+        );
+        mdb_txn_abort(txn);
+        goto end;
+    }
+
+    key = LMDBTranslateKey(k);
+
+    /* Small hack to get it to list subitems */
+    ((char *) key.mv_data)[0]++;
+
+    ret = ArrayCreate();
+    subKey = key;
+    while (mdb_cursor_get(cursor, &subKey, &val, op) == 0)
+    {
+        /* This searches by *increasing* order. The problem is that it may 
+         * extend to unwanted points. Since the values are sorted, we can 
+         * just exit if the subkey is incorrect. */
+        char *head = LMDBKeyHead(subKey);
+        if (!LMDBKeyStartsWith(subKey, key))
+        {
+            break;
+        }
+
+        StringArrayAppend(ret, head);
+        op = MDB_NEXT;
+    }
+
+    mdb_cursor_close(cursor);
+    mdb_txn_abort(txn);
+
+end:
+    LMDBKillKey(key);
+    pthread_mutex_unlock(&d->lock);
+    return ret;
+}
+
+/* Implementation functions */
+static void
+LMDBClose(Db *d)
+{
+    LMDB *db = (LMDB *) d;
+    if (!d)
+    {
+        return;
+    }
+
+    mdb_dbi_close(db->environ, db->dbi);
+    mdb_env_close(db->environ);
+}
+
+Db *
+DbOpenLMDB(char *dir, size_t size)
+{
+    /* TODO */
+    MDB_env *env = NULL;
+    MDB_txn *txn;
+    MDB_dbi dbi;
+    int code;
+    LMDB *db;
+    if (!dir || !size)
+    {
+        return NULL;
+    }
+
+    /* Try initialising LMDB */
+    if ((code = mdb_env_create(&env)) != 0)
+    {
+        Log(LOG_ERR, 
+            "%s: could not create LMDB env: %s",
+            __func__, mdb_strerror(code)
+        );
+        return NULL;
+    }
+    if ((code = mdb_env_set_mapsize(env, size) != 0))
+    {
+        Log(LOG_ERR, 
+            "%s: could not set mapsize to %lu: %s",
+            __func__, (unsigned long) size, 
+            mdb_strerror(code)
+        );
+        mdb_env_close(env);
+        return NULL;
+    }
+    mdb_env_set_maxdbs(env, 4);
+    if ((code = mdb_env_open(env, dir, MDB_NOTLS, 0644)) != 0)
+    {
+        Log(LOG_ERR, 
+            "%s: could not open LMDB env: %s",
+            __func__, mdb_strerror(code)
+        );
+        mdb_env_close(env);
+        return NULL;
+    }
+
+    /* Initialise a DBI */
+    {
+        if ((code = mdb_txn_begin(env, NULL, 0, &txn)) != 0)
+        {
+            Log(LOG_ERR,
+                "%s: could not begin transaction: %s",
+                __func__, mdb_strerror(code)
+            );
+            mdb_env_close(env);
+            return NULL;
+        }
+        if ((code = mdb_dbi_open(txn, "db", MDB_CREATE, &dbi)) != 0)
+        {
+            Log(LOG_ERR,
+                "%s: could not get transaction dbi: %s",
+                __func__, mdb_strerror(code)
+            );
+            mdb_txn_abort(txn);
+            mdb_env_close(env);
+            return NULL;
+        }
+        mdb_txn_commit(txn);
+    }
+
+
+    db = Malloc(sizeof(*db));
+    DbInit((Db *) db);
+    db->environ = env;
+    db->dbi = dbi;
+
+    db->base.lockFunc = LMDBLock;
+    db->base.create = LMDBCreate;
+    db->base.unlock = LMDBUnlock;
+    db->base.delete = LMDBDelete;
+    db->base.exists = LMDBExists;
+    db->base.close = LMDBClose;
+    db->base.list = LMDBList;
+    
+    return (Db *) db;
+}
+
+#else
+Db *
+DbOpenLMDB(char *dir, size_t size)
+{
+    /* Unimplemented function */
+    Log(LOG_ERR, 
+        "LMDB support is not enabled. Please compile with --use-lmdb"
+    );
+    (void) size;
+    (void) dir;
+    return NULL;
+}
+#endif

src/Json.c

@@ -1348,6 +1348,25 @@ JsonDecode(Stream * stream)
 
     return result;
 }
+JsonValue * 
+JsonValueDecode(Stream *stream)
+{
+    JsonValue *result;
+    JsonParserState state;
+
+    state.stream = stream;
+    state.token = NULL;
+
+    JsonTokenSeek(&state);
+    result = JsonDecodeValue(&state);
+
+    if (state.token)
+    {
+        Free(state.token);
+    }
+
+    return result;
+}
 
 JsonValue *
 JsonGet(HashMap * json, size_t nArgs,...)

src/Log.c

@@ -39,7 +39,7 @@ struct LogConfig
 {
     int level;
     size_t indent;
-    Stream *file;
+    Stream *out;
     int flags;
     char *tsFmt;
 
@@ -65,8 +65,8 @@ LogConfigCreate(void)
 
     LogConfigLevelSet(config, LOG_INFO);
     LogConfigIndentSet(config, 0);
-    LogConfigFileSet(config, NULL);
+    LogConfigOutputSet(config, NULL);   /* Will set to stdout */
-    LogConfigFlagSet(config, LOG_FLAG_COLOR | LOG_FLAG_STDOUT);
+    LogConfigFlagSet(config, LOG_FLAG_COLOR);
     LogConfigTimeStampFormatSet(config, "%y-%m-%d %H:%M:%S");
 
     return config;
@@ -185,20 +185,20 @@ LogConfigLevelSet(LogConfig * config, int level)
 }
 
 void
-LogConfigFileSet(LogConfig * config, Stream * file)
+LogConfigOutputSet(LogConfig * config, Stream * out)
 {
     if (!config)
     {
         return;
     }
 
-    if (file)
+    if (out)
     {
-        config->file = file;
+        config->out = out;
     }
     else
     {
-        config->file = StreamStdout();
+        config->out = StreamStdout();
     }
 
 }
@@ -221,15 +221,43 @@ LogConfigUnindent(LogConfig * config)
     }
 }
 
-static void
+void
-LogPrint(Stream * stream, LogConfig * config, int level, const char *msg, va_list argp)
+Logv(LogConfig * config, int level, const char *msg, va_list argp)
 {
     size_t i;
     int doColor;
     char indicator;
 
+    /*
+     * Only proceed if we have a config and its log level is set to a
+     * value that permits us to log. This is as close as we can get
+     * to a no-op function if we aren't logging anything, without doing
+     * some crazy macro magic.
+     */
+    if (!config || level > config->level)
+    {
+        return;
+    }
+
+    /* Misconfiguration */
+    if (!config->out)
+    {
+        return;
+    }
+
+    pthread_mutex_lock(&config->lock);
+
+    if (LogConfigFlagGet(config, LOG_FLAG_SYSLOG))
+    {
+        /* No further print logic is needed; syslog will handle it all
+         * for us. */
+        vsyslog(level, msg, argp);
+        pthread_mutex_unlock(&config->lock);
+        return;
+    }
+
     doColor = LogConfigFlagGet(config, LOG_FLAG_COLOR)
-            && isatty(StreamFileno(stream));
+            && isatty(StreamFileno(config->out));
 
     if (doColor)
     {
@@ -265,10 +293,10 @@ LogPrint(Stream * stream, LogConfig * config, int level, const char *msg, va_lis
                 break;
         }
 
-        StreamPuts(stream, ansi);
+        StreamPuts(config->out, ansi);
     }
 
-    StreamPutc(stream, '[');
+    StreamPutc(config->out, '[');
 
     if (config->tsFmt)
     {
@@ -281,15 +309,15 @@ LogPrint(Stream * stream, LogConfig * config, int level, const char *msg, va_lis
 
         if (tsLength)
         {
-            StreamPuts(stream, tsBuffer);
+            StreamPuts(config->out, tsBuffer);
             if (!isspace((unsigned char) tsBuffer[tsLength - 1]))
             {
-                StreamPutc(stream, ' ');
+                StreamPutc(config->out, ' ');
             }
         }
     }
 
-    StreamPrintf(stream, "(%lu) ", UtilThreadNo());
+    StreamPrintf(config->out, "(%lu) ", UtilThreadNo());
 
     switch (level)
     {
@@ -322,60 +350,24 @@ LogPrint(Stream * stream, LogConfig * config, int level, const char *msg, va_lis
             break;
     }
 
-    StreamPrintf(stream, "%c]", indicator);
+    StreamPrintf(config->out, "%c]", indicator);
 
     if (doColor)
     {
         /* ANSI Reset */
-        StreamPuts(stream, "\033[0m");
+        StreamPuts(config->out, "\033[0m");
     }
 
-    StreamPutc(stream, ' ');
+    StreamPutc(config->out, ' ');
     for (i = 0; i < config->indent; i++)
     {
-        StreamPutc(config->file, ' ');
+        StreamPutc(config->out, ' ');
     }
 
-    StreamVprintf(stream, msg, argp);
+    StreamVprintf(config->out, msg, argp);
-    StreamPutc(stream, '\n');
+    StreamPutc(config->out, '\n');
 
-    StreamFlush(stream);
+    StreamFlush(config->out);
-}
-
-void
-Logv(LogConfig * config, int level, const char *msg, va_list argp)
-{
-    /*
-     * Only proceed if we have a config and its log level is set to a
-     * value that permits us to log. This is as close as we can get
-     * to a no-op function if we aren't logging anything, without doing
-     * some crazy macro magic.
-     */
-    if (!config || level > config->level)
-    {
-        return;
-    }
-
-    /* Misconfiguration */
-    if (!config->file)
-    {
-        return;
-    }
-
-    pthread_mutex_lock(&config->lock);
-
-    if (LogConfigFlagGet(config, LOG_FLAG_SYSLOG))
-    {
-        vsyslog(level, msg, argp);
-    }
-
-    if (LogConfigFlagGet(config, LOG_FLAG_STDOUT)) {
-        LogPrint(StreamStdout(), config, level, msg, argp);
-    }
-
-    if (config->file) {
-        LogPrint(config->file, config, level, msg, argp);
-    }
 
     pthread_mutex_unlock(&config->lock);
 }

src/Memory.c

@@ -42,29 +42,52 @@
 
 #define MEMORY_FILE_SIZE 256
 
+#define MEM_BOUND_TYPE uint64_t
+#define MEM_BOUND 0xDEADBEEFBEEFDEAD
+#define MEM_MAGIC 0xDEADBEEFDEADBEEF
+
 struct MemoryInfo
 {
+    uint64_t magic;
+
     size_t size;
+    size_t unalignedSize;
     char file[MEMORY_FILE_SIZE];
     int line;
     void *pointer;
+
+    MemoryInfo *prev;
+    MemoryInfo *next;
+
+    MEM_BOUND_TYPE leftBoundary;
 };
 
-#define MEM_BOUND_TYPE uint32_t
+#define MEM_SIZE_ACTUAL(x) (MemoryAlignBoundary((x) * sizeof(uint8_t)) + sizeof(MEM_BOUND_TYPE))
-#define MEM_BOUND 0xDEADBEEF
+#define MEM_START_BOUNDARY(info) (info->leftBoundary)
+#define MEM_END_BOUNDARY(info) (*(((MEM_BOUND_TYPE *) (((uint8_t *) info->pointer) + info->size)) - 1))
 
-#define MEM_BOUND_LOWER(p) *((MEM_BOUND_TYPE *) p)
-#define MEM_BOUND_UPPER(p, x) *(((MEM_BOUND_TYPE *) (((uint8_t *) p) + x)) + 1)
-#define MEM_SIZE_ACTUAL(x) (((x) * sizeof(uint8_t)) + (2 * sizeof(MEM_BOUND_TYPE)))
 
 static pthread_mutex_t lock;
 static void (*hook) (MemoryAction, MemoryInfo *, void *) = MemoryDefaultHook;
 static void *hookArgs = NULL;
 
-static MemoryInfo **allocations = NULL;
-static size_t allocationsSize = 0;
 static size_t allocationsLen = 0;
 
+static MemoryInfo *allocationTail = NULL;
+
+/* Simple range of "plausible" boundaries for heap, serving as an extra 
+ * check */
+static void *heapStart, *heapEnd;
+
+static size_t MemoryAlignBoundary(size_t size)
+{
+    size_t boundSize = sizeof(MEM_BOUND_TYPE);
+    size_t remainder = size % boundSize;
+    size_t closest   = size / boundSize + !!remainder;
+
+    return closest * boundSize;
+}
+
 int
 MemoryRuntimeInit(void)
 {
@@ -79,6 +102,8 @@ MemoryRuntimeInit(void)
     pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
     ret = pthread_mutex_init(&lock, &attr);
     pthread_mutexattr_destroy(&attr);
+    heapStart = NULL;
+    heapEnd = NULL;
 
     ret = (ret == 0);
 
@@ -93,70 +118,27 @@ MemoryRuntimeDestroy(void)
     return pthread_mutex_destroy(&lock) == 0;
 }
 
-static size_t
-MemoryHash(void *p)
-{
-    return (((size_t) p) >> 2 * 7) % allocationsSize;
-}
-
 static int
 MemoryInsert(MemoryInfo * a)
 {
-    size_t hash;
+    if (allocationTail)
-
-    if (!allocations)
     {
-        allocationsSize = MEMORY_TABLE_CHUNK;
+        allocationTail->next = a;
-        allocations = calloc(allocationsSize, sizeof(void *));
-        if (!allocations)
-        {
-            return 0;
     }
+    a->next = NULL;
+    a->prev = allocationTail;
+    a->magic = MEM_MAGIC;
+
+    if (!heapStart || heapStart > (void *) a)
+    {
+        heapStart = a;
+    }
+    if (!heapEnd || heapEnd < (void *) a)
+    {
+        heapEnd = a;
     }
 
-    /* If the next insertion would cause the table to be at least 3/4
+    allocationTail = a;
-     * full, re-allocate and re-hash. */
-    if ((allocationsLen + 1) >= ((allocationsSize * 3) >> 2))
-    {
-        size_t i;
-        size_t tmpAllocationsSize = allocationsSize;
-        MemoryInfo **tmpAllocations;
-
-        allocationsSize += MEMORY_TABLE_CHUNK;
-        tmpAllocations = calloc(allocationsSize, sizeof(void *));
-
-        if (!tmpAllocations)
-        {
-            return 0;
-        }
-
-        for (i = 0; i < tmpAllocationsSize; i++)
-        {
-            if (allocations[i])
-            {
-                hash = MemoryHash(allocations[i]->pointer);
-
-                while (tmpAllocations[hash])
-                {
-                    hash = (hash + 1) % allocationsSize;
-                }
-
-                tmpAllocations[hash] = allocations[i];
-            }
-        }
-
-        free(allocations);
-        allocations = tmpAllocations;
-    }
-
-    hash = MemoryHash(a->pointer);
-
-    while (allocations[hash])
-    {
-        hash = (hash + 1) % allocationsSize;
-    }
-
-    allocations[hash] = a;
     allocationsLen++;
 
     return 1;
@@ -165,30 +147,32 @@ MemoryInsert(MemoryInfo * a)
 static void
 MemoryDelete(MemoryInfo * a)
 {
-    size_t hash = MemoryHash(a->pointer);
+    MemoryInfo *aPrev = a->prev;
-    size_t count = 0;
+    MemoryInfo *aNext = a->next;
 
-    while (count <= allocationsSize)
+    if (aPrev)
     {
-        if (allocations[hash] && allocations[hash] == a)
+        aPrev->next = aNext;
+    }
+    if (aNext)
     {
-            allocations[hash] = NULL;
+        aNext->prev = aPrev;
-            allocationsLen--;
-            return;
     }
-        else
+
+    if (a == allocationTail)
     {
-            hash = (hash + 1) % allocationsSize;
+        allocationTail = aPrev;
-            count++;
-        }
     }
+
+    a->magic = ~MEM_MAGIC;
 }
 
 static int
 MemoryCheck(MemoryInfo * a)
 {
-    if (MEM_BOUND_LOWER(a->pointer) != MEM_BOUND ||
+    if (MEM_START_BOUNDARY(a) != MEM_BOUND || 
-        MEM_BOUND_UPPER(a->pointer, a->size - (2 * sizeof(MEM_BOUND_TYPE))) != MEM_BOUND)
+        a->magic != MEM_MAGIC ||
+        MEM_END_BOUNDARY(a) != MEM_BOUND)
     {
         if (hook)
         {
@@ -205,38 +189,32 @@ MemoryAllocate(size_t size, const char *file, int line)
     void *p;
     MemoryInfo *a;
 
-    MemoryIterate(NULL, NULL);
+    //MemoryIterate(NULL, NULL);
 
     pthread_mutex_lock(&lock);
 
-    a = malloc(sizeof(MemoryInfo));
+    a = malloc(sizeof(MemoryInfo) + MEM_SIZE_ACTUAL(size));
     if (!a)
     {
         pthread_mutex_unlock(&lock);
         return NULL;
     }
 
-    p = malloc(MEM_SIZE_ACTUAL(size));
+    p = a + 1;
-    if (!p)
-    {
-        free(a);
-        pthread_mutex_unlock(&lock);
-        return NULL;
-    }
 
     memset(p, 0, MEM_SIZE_ACTUAL(size));
-    MEM_BOUND_LOWER(p) = MEM_BOUND;
-    MEM_BOUND_UPPER(p, size) = MEM_BOUND;
 
     a->size = MEM_SIZE_ACTUAL(size);
+    a->unalignedSize = size;
     strncpy(a->file, file, MEMORY_FILE_SIZE - 1);
     a->line = line;
     a->pointer = p;
+    MEM_START_BOUNDARY(a) = MEM_BOUND;
+    MEM_END_BOUNDARY(a) = MEM_BOUND;
 
     if (!MemoryInsert(a))
     {
         free(a);
-        free(p);
         pthread_mutex_unlock(&lock);
         return NULL;
     }
@@ -247,7 +225,7 @@ MemoryAllocate(size_t size, const char *file, int line)
     }
 
     pthread_mutex_unlock(&lock);
-    return ((MEM_BOUND_TYPE *) p) + 1;
+    return p;
 }
 
 void *
@@ -256,7 +234,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line)
     MemoryInfo *a;
     void *new = NULL;
 
-    MemoryIterate(NULL, NULL);
+    //MemoryIterate(NULL, NULL);
 
     if (!p)
     {
@@ -267,19 +245,22 @@ MemoryReallocate(void *p, size_t size, const char *file, int line)
     if (a)
     {
         pthread_mutex_lock(&lock);
-        new = realloc(a->pointer, MEM_SIZE_ACTUAL(size));
+        MemoryDelete(a);
+        new = realloc(a, sizeof(MemoryInfo) + MEM_SIZE_ACTUAL(size));
         if (new)
         {
-            MemoryDelete(a);
+            a = new;
+            a->unalignedSize = size;
             a->size = MEM_SIZE_ACTUAL(size);
             strncpy(a->file, file, MEMORY_FILE_SIZE - 1);
             a->line = line;
+            a->magic = MEM_MAGIC;
 
-            a->pointer = new;
+            a->pointer = a + 1;
             MemoryInsert(a);
 
-            MEM_BOUND_LOWER(a->pointer) = MEM_BOUND;
+            MEM_START_BOUNDARY(a) = MEM_BOUND;
-            MEM_BOUND_UPPER(a->pointer, size) = MEM_BOUND;
+            MEM_END_BOUNDARY(a) = MEM_BOUND;
 
             if (hook)
             {
@@ -303,7 +284,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line)
         }
     }
 
-    return ((MEM_BOUND_TYPE *) new) + 1;
+    return a->pointer;
 }
 
 void
@@ -311,7 +292,7 @@ MemoryFree(void *p, const char *file, int line)
 {
     MemoryInfo *a;
 
-    MemoryIterate(NULL, NULL);
+    //MemoryIterate(NULL, NULL);
 
     if (!p)
     {
@@ -329,7 +310,6 @@ MemoryFree(void *p, const char *file, int line)
             hook(MEMORY_FREE, a, hookArgs);
         }
         MemoryDelete(a);
-        free(a->pointer);
         free(a);
 
         pthread_mutex_unlock(&lock);
@@ -352,17 +332,15 @@ MemoryFree(void *p, const char *file, int line)
 size_t
 MemoryAllocated(void)
 {
-    size_t i;
     size_t total = 0;
+    MemoryInfo *cur;
 
     pthread_mutex_lock(&lock);
 
-    for (i = 0; i < allocationsSize; i++)
+    /* TODO */
+    for (cur = allocationTail; cur; cur = cur->prev)
     {
-        if (allocations[i])
+        total += MemoryInfoGetSize(cur);
-        {
-            total += MemoryInfoGetSize(allocations[i]);
-        }
     }
 
     pthread_mutex_unlock(&lock);
@@ -373,55 +351,44 @@ MemoryAllocated(void)
 void
 MemoryFreeAll(void)
 {
-    size_t i;
+    MemoryInfo *cur;
+    MemoryInfo *prev;
 
     pthread_mutex_lock(&lock);
 
-    for (i = 0; i < allocationsSize; i++)
+    /* TODO */
+    for (cur = allocationTail; cur; cur = prev)
     {
-        if (allocations[i])
+        prev = cur->prev;
-        {
+        free(cur);
-            free(allocations[i]->pointer);
-            free(allocations[i]);
-        }
     }
 
-    free(allocations);
-    allocations = NULL;
-    allocationsSize = 0;
     allocationsLen = 0;
 
     pthread_mutex_unlock(&lock);
 }
 
 MemoryInfo *
-MemoryInfoGet(void *p)
+MemoryInfoGet(void *po)
 {
-    size_t hash, count;
+    void *p = po;
 
     pthread_mutex_lock(&lock);
 
-    p = ((MEM_BOUND_TYPE *) p) - 1;
+    p = ((MemoryInfo *) p) - 1;
-    hash = MemoryHash(p);
+    if (p < heapStart || p > heapEnd)
-
-    count = 0;
-    while (count <= allocationsSize)
     {
-        if (!allocations[hash] || allocations[hash]->pointer != p)
-        {
-            hash = (hash + 1) % allocationsSize;
-            count++;
-        }
-        else
-        {
-            MemoryCheck(allocations[hash]);
-            pthread_mutex_unlock(&lock);
-            return allocations[hash];
-        }
-    }
-
         pthread_mutex_unlock(&lock);
         return NULL;
+    }
+
+    if (((MemoryInfo *)p)->magic != MEM_MAGIC)
+    {
+        p = NULL;
+    }
+
+    pthread_mutex_unlock(&lock);
+    return p;
 }
 
 size_t
@@ -432,7 +399,7 @@ MemoryInfoGetSize(MemoryInfo * a)
         return 0;
     }
 
-    return a->size ? a->size - (2 * sizeof(MEM_BOUND_TYPE)) : 0;
+    return a->size ? a->unalignedSize : 0;
 }
 
 const char *
@@ -465,25 +432,22 @@ MemoryInfoGetPointer(MemoryInfo * a)
         return NULL;
     }
 
-    return ((MEM_BOUND_TYPE *) a->pointer) + 1;
+    return a->pointer;
 }
 
 void
- MemoryIterate(void (*iterFunc) (MemoryInfo *, void *), void *args)
+MemoryIterate(void (*iterFunc) (MemoryInfo *, void *), void *args)
 {
-    size_t i;
+    MemoryInfo *cur;
 
     pthread_mutex_lock(&lock);
 
-    for (i = 0; i < allocationsSize; i++)
+    for (cur = allocationTail; cur; cur = cur->prev)
     {
-        if (allocations[i])
+        MemoryCheck(cur);
-        {
-            MemoryCheck(allocations[i]);
         if (iterFunc)
         {
-                iterFunc(allocations[i], args);
+            iterFunc(cur, args);
-            }
         }
     }
 

src/Sha.c

@@ -28,21 +28,44 @@
 #include <string.h>
 
 char *
-ShaToHex(unsigned char *bytes)
+ShaToHex(unsigned char *bytes, HashType type)
 {
-    size_t i = 0;
+    size_t i = 0, size;
-    char *str = Malloc(((strlen((char *) bytes) * 2) + 1) * sizeof(char));
+    char *str;
 
+    switch (type)
+    {
+        case HASH_SHA1:
+            size = 20;
+            break;
+        case HASH_SHA256:
+            size = 32;
+            break;
+        default:
+            return NULL;
+    }
+
+    str = Malloc(((size * 2) + 1) * sizeof(char));
     if (!str)
     {
         return NULL;
     }
 
-    while (bytes[i] != '\0')
+    for (i = 0; i < size; i++)
     {
         snprintf(str + (2 * i), 3, "%02x", bytes[i]);
-        i++;
     }
 
     return str;
 }
+unsigned char *
+Sha256(char *str)
+{
+    return Sha256Raw((unsigned char *) str, str ? strlen(str) : 0);
+}
+
+unsigned char *
+Sha1(char *str)
+{
+    return Sha1Raw((unsigned char *) str, str ? strlen(str) : 0);
+}

src/Sha/Sha1.c

@@ -28,6 +28,27 @@
 
 #include <limits.h>
 
+/* TODO: Verify LibreSSL support later */
+#if defined(TLS_IMPL) && (TLS_IMPL == TLS_OPENSSL)
+
+#include <openssl/sha.h>
+    
+unsigned char *
+Sha1Raw(unsigned char *str, size_t len)
+{
+    unsigned char *digest;
+    if (!str)
+    {
+        return NULL;
+    }
+
+    digest = Malloc(20 + 1);
+    SHA1(str, len, digest);
+    digest[20] = '\0';
+    return digest;
+}
+#else
+
 #define LOAD32H(x, y) \
 	{ \
 		x = ((uint32_t)((y)[0] & 255) << 24) | \
@@ -240,7 +261,7 @@ Sha1Calculate(Sha1Context * ctx, unsigned char *out)
 }
 
 unsigned char *
-Sha1(char *str)
+Sha1Raw(unsigned char *str, size_t len)
 {
     Sha1Context ctx;
     unsigned char *out;
@@ -257,10 +278,11 @@ Sha1(char *str)
     }
 
     Sha1Init(&ctx);
-    Sha1Update(&ctx, str, strlen(str));
+    Sha1Update(&ctx, str, len);
     Sha1Calculate(&ctx, out);
 
     out[160 / 8] = '\0';
 
     return out;
 }
+#endif

src/Sha/Sha256.c

@@ -21,14 +21,36 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
-#include <Sha.h>
 #include <Memory.h>
+#include <Sha.h>
 
 #include <stdio.h>
 #include <string.h>
 
 #include <limits.h>
 
+
+/* TODO: Verify LibreSSL support later */
+#if defined(TLS_IMPL) && (TLS_IMPL == TLS_OPENSSL)
+
+#include <openssl/sha.h>
+
+unsigned char *
+Sha256Raw(unsigned char *str, size_t len)
+{
+    unsigned char *digest;
+    if (!str)
+    {
+        return NULL;
+    }
+
+    digest = Malloc(32 + 1);
+    SHA256(str, len, digest);
+    digest[32] = '\0';
+    return digest;
+}
+#else
+
 #define GET_UINT32(x) \
     (((uint32_t)(x)[0] << 24) | \
      ((uint32_t)(x)[1] << 16) | \
@@ -170,7 +192,7 @@ Sha256Process(Sha256Context * context, unsigned char *data, size_t length)
 }
 
 unsigned char *
-Sha256(char *str)
+Sha256Raw(unsigned char *str, size_t len)
 {
     Sha256Context context;
     size_t i;
@@ -206,7 +228,7 @@ Sha256(char *str)
     context.length = 0;
     memset(context.buffer, 0, 64);
 
-    Sha256Process(&context, (unsigned char *) str, strlen(str));
+    Sha256Process(&context, str, len);
 
     memset(fill, 0, 64);
     fill[0] = 0x80;
@@ -230,3 +252,4 @@ Sha256(char *str)
 
     return out;
 }
+#endif

src/Tls/TlsLibreSSL.c

@@ -23,7 +23,7 @@
  */
 #include <Tls.h>
 
-#if TLS_IMPL == TLS_LIBRESSL
+#if defined(TLS_IMPL) && (TLS_IMPL == TLS_LIBRESSL)
 
 #include <Memory.h>
 #include <Log.h>

src/Tls/TlsOpenSSL.c

@@ -23,7 +23,7 @@
  */
 #include <Tls.h>
 
-#if TLS_IMPL == TLS_OPENSSL
+#if defined(TLS_IMPL) && (TLS_IMPL == TLS_OPENSSL)
 
 #include <Memory.h>
 #include <Log.h>

src/Util.c

@@ -24,6 +24,7 @@
 #include <Util.h>
 
 #include <Memory.h>
+#include <Platform.h>
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -89,6 +90,10 @@ UtilTsMillis(void)
     return ts;
 }
 
+#ifdef PLATFORM_DARWIN
+#define st_mtim st_mtimespec
+#endif
+
 uint64_t
 UtilLastModified(char *path)
 {
@@ -105,6 +110,10 @@ UtilLastModified(char *path)
     return ts;
 }
 
+#ifdef PLATFORM_DARWIN
+#undef st_mtim
+#endif
+
 int
 UtilMkdir(const char *dir, const mode_t mode)
 {

tools/j2s.c

@@ -38,6 +38,7 @@
 
 #define MAX_DEPENDENCIES 32
 
+#define IsDelimited(field, c1, c2) (*field == c1 && field[strlen(field) - 1] == c2)
 static char *
 Trim(char c, char *str)
 {
@@ -75,10 +76,14 @@ TypeToJsonType(char *type)
     }
     else
     {
-        if (*type == '[' && type[strlen(type) - 1] == ']')
+        if (IsDelimited(type, '[', ']'))
         {
             return JSON_ARRAY;
         }
+        else if (IsDelimited(type, '{', '}'))
+        {
+            return JSON_OBJECT;
+        }
         else
         {
             return JSON_OBJECT;
@@ -325,6 +330,7 @@ Main(Array * args)
             {
                 char *fieldType;
                 bool isArrType = false;
+                bool isObjType = false;
                 JsonValue *requiredVal;
                 JsonValue *ignoreVal;
 
@@ -343,12 +349,18 @@ Main(Array * args)
                     goto finish;
                 }
 
-                if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']')
+                if (IsDelimited(fieldType, '[', ']'))
                 {
                     fieldType++;
                     fieldType[strlen(fieldType) - 1] = '\0';
                     isArrType = true;
                 }
+                else if (IsDelimited(fieldType, '{', '}'))
+                {
+                    fieldType++;
+                    fieldType[strlen(fieldType) - 1] = '\0';
+                    isObjType = true;
+                }
 
                 if (!StrEquals(fieldType, "object") &&
                     !StrEquals(fieldType, "array") &&
@@ -374,6 +386,10 @@ Main(Array * args)
                 {
                     fieldType[strlen(fieldType)] = ']';
                 }
+                else if (isObjType)
+                {
+                    fieldType[strlen(fieldType)] = '}';
+                }
 
                 requiredVal = HashMapGet(fieldObj, "required");
                 if (requiredVal && JsonValueType(requiredVal) != JSON_BOOLEAN)
@@ -529,11 +545,13 @@ Main(Array * args)
                 {
                     cType = "double";
                 }
-                else if (StrEquals(fieldType, "object"))
+                else if (StrEquals(fieldType, "object") || 
+                        IsDelimited(fieldType, '{', '}'))
                 {
                     cType = "HashMap *";
                 }
-                else if (StrEquals(fieldType, "array") || (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']'))
+                else if (StrEquals(fieldType, "array") || 
+                        IsDelimited(fieldType, '[', ']'))
                 {
                     cType = "Array *";
                 }
@@ -542,8 +560,21 @@ Main(Array * args)
                     cType = fieldType;
                 }
 
+                if (IsDelimited(fieldType, '{', '}') ||
+                    IsDelimited(fieldType, '[', ']'))
+                {
+                    char end = fieldType[strlen(fieldType) - 1];
+                    fieldType[strlen(fieldType) - 1] = '\0';
+
+                    StreamPrintf(headerFile, "    %s /* of %s */ %s;\n", cType, fieldType + 1, field);
+
+                    fieldType[strlen(fieldType)] = end;
+                }
+                else
+                {
                     StreamPrintf(headerFile, "    %s %s;\n", cType, field);
                 }
+            }
 
             StreamPrintf(headerFile, "} %s;\n\n", type);
         }
@@ -673,7 +704,134 @@ Main(Array * args)
                     StreamPrintf(implFile, "        out->%s = JsonValueAsObject(val);\n", key);
                     StreamPrintf(implFile, "        Free(val); /* Not JsonValueFree() because we want the inner value. */\n");
                 }
-                else if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']')
+                else if (IsDelimited(fieldType, '{', '}'))
+                {
+                    fieldType++;
+                    fieldType[strlen(fieldType) - 1] = '\0';
+                    isEnum = StrEquals(JsonValueAsString(JsonGet(types, 2, fieldType, "type")), "enum");
+                    jsonType = isEnum ? JSON_STRING : TypeToJsonType(fieldType);
+
+                    StreamPrintf(implFile, "        out->%s = HashMapCreate();\n", key);
+                    StreamPrintf(implFile, "        if (!out->%s)\n", key);
+                    StreamPrintf(implFile, "        {\n");
+                    StreamPrintf(implFile, "            *errp = \"Failed to allocate memory for %s.%s.\";\n", type, key);
+                    StreamPrintf(implFile, "            return false;\n");
+                    StreamPrintf(implFile, "        }\n");
+                    StreamPrintf(implFile, "        else\n");
+                    StreamPrintf(implFile, "        {\n");
+                    StreamPrintf(implFile, "            HashMap *obj = JsonValueAsObject(val);\n");
+                    StreamPrintf(implFile, "            char *objKey;\n");
+                    StreamPrintf(implFile, "            JsonValue *v;\n");
+                    StreamPrintf(implFile, "\n");
+                    StreamPrintf(implFile, "            while (HashMapIterate(obj, &objKey, (void **) &v))\n");
+                    StreamPrintf(implFile, "            {\n");
+
+                    if (StrEquals(fieldType, "integer") ||
+                        StrEquals(fieldType, "float") ||
+                        StrEquals(fieldType, "boolean"))
+                    {
+                        char *cType;
+
+                        if (StrEquals(fieldType, "integer"))
+                        {
+                            cType = "int64_t";
+                        }
+                        else if (StrEquals(fieldType, "float"))
+                        {
+                            cType = "double";
+                        }
+                        else if (StrEquals(fieldType, "boolean"))
+                        {
+                            cType = "bool";
+                        }
+                        else
+                        {
+                            /* Should never happen */
+                            cType = NULL;
+                        }
+
+                        *fieldType = toupper(*fieldType);
+
+                        StreamPrintf(implFile, "                %s *ref;\n", cType);
+                        StreamPrintf(implFile, "                if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType));
+                        StreamPrintf(implFile, "                {\n");
+                        StreamPrintf(implFile, "                    *errp = \"%s.%s{} contains an invalid value.\";\n", type, key);
+                        StreamPrintf(implFile, "                    return false;\n");
+                        StreamPrintf(implFile, "                }\n");
+                        StreamPrintf(implFile, "                ref = Malloc(sizeof(%s));\n", cType);
+                        StreamPrintf(implFile, "                if (!ref)\n");
+                        StreamPrintf(implFile, "                {\n");
+                        StreamPrintf(implFile, "                    *errp = \"Unable to allocate memory for object value.\";\n");
+                        StreamPrintf(implFile, "                    return false;\n");
+                        StreamPrintf(implFile, "                }\n");
+                        StreamPrintf(implFile, "                *ref = JsonValueAs%s(v);\n", fieldType);
+                        StreamPrintf(implFile, "                HashMapSet(out->%s, objKey, ref);\n", key);
+
+                        *fieldType = tolower(*fieldType);
+                    }
+                    else if (StrEquals(fieldType, "string"))
+                    {
+                        StreamPrintf(implFile, "                if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType));
+                        StreamPrintf(implFile, "                {\n");
+                        StreamPrintf(implFile, "                    *errp = \"%s.%s[] contains an invalid value.\";\n", type, key);
+                        StreamPrintf(implFile, "                    return false;\n");
+                        StreamPrintf(implFile, "                }\n");
+                        StreamPrintf(implFile, "                HashMapSet(out->%s, objKey, StrDuplicate(JsonValueAsString(v)));\n", key);
+                    }
+                    else if (StrEquals(fieldType, "object"))
+                    {
+                        StreamPrintf(implFile, "                if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType));
+                        StreamPrintf(implFile, "                {\n");
+                        StreamPrintf(implFile, "                    *errp = \"%s.%s[] contains an invalid value.\";\n", type, key);
+                        StreamPrintf(implFile, "                    return false;\n");
+                        StreamPrintf(implFile, "                }\n");
+                        StreamPrintf(implFile, "                HashMapSet(out->%s, objKey, JsonDuplicate(JsonValueAsObject(v)));\n", key);
+                    }
+                    else
+                    {
+                        if (isEnum)
+                        {
+                            StreamPrintf(implFile, "                int parseResult;\n");
+                        }
+                        StreamPrintf(implFile, "                %s *parsed;\n", fieldType);
+                        StreamPrintf(implFile, "                if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType));
+                        StreamPrintf(implFile, "                {\n");
+                        StreamPrintf(implFile, "                    *errp = \"%s.%s[] contains an invalid value.\";\n", type, key);
+                        StreamPrintf(implFile, "                    return false;\n");
+                        StreamPrintf(implFile, "                }\n");
+                        StreamPrintf(implFile, "                parsed = Malloc(sizeof(%s));\n", fieldType);
+                        StreamPrintf(implFile, "                if (!parsed)\n");
+                        StreamPrintf(implFile, "                {\n");
+                        StreamPrintf(implFile, "                    *errp = \"Unable to allocate memory for array value.\";\n");
+                        StreamPrintf(implFile, "                    return false;\n");
+                        StreamPrintf(implFile, "                }\n");
+                        if (isEnum)
+                        {
+                            StreamPrintf(implFile, "                parseResult = %sFromStr(JsonValueAsString(v));\n", fieldType);
+                            StreamPrintf(implFile, "                *parsed = parseResult;\n");
+                            StreamPrintf(implFile, "                if (parseResult == -1)\n");
+                        }
+                        else
+                        {
+                            StreamPrintf(implFile, "                if (!%sFromJson(JsonValueAsObject(v), parsed, errp))\n", fieldType);
+                        }
+                        StreamPrintf(implFile, "                {\n");
+                        if (!isEnum)
+                        {
+                            StreamPrintf(implFile, "                    %sFree(parsed);\n", fieldType);
+                        }
+                        StreamPrintf(implFile, "                    Free(parsed);\n");
+                        StreamPrintf(implFile, "                    return false;\n");
+                        StreamPrintf(implFile, "                }\n");
+                        StreamPrintf(implFile, "                HashMapSet(out->%s, objKey, parsed);\n", key);
+                    }
+
+                    StreamPrintf(implFile, "            }\n");
+                    StreamPrintf(implFile, "        }\n");
+
+                    fieldType[strlen(fieldType)] = '}';
+                }
+                else if (IsDelimited(fieldType, '[', ']'))
                 {
                     fieldType++;
                     fieldType[strlen(fieldType) - 1] = '\0';
@@ -915,7 +1073,73 @@ Main(Array * args)
                 {
                     StreamPrintf(implFile, "    HashMapSet(json, \"%s\", JsonValueObject(JsonDuplicate(val->%s)));\n", Trim('_', key), key);
                 }
-                else if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']')
+                else if (IsDelimited(fieldType, '{', '}'))
+                {
+                    int isPrimitive;
+
+                    fieldType++;
+                    fieldType[strlen(fieldType) - 1] = '\0';
+                    isEnum = StrEquals(JsonValueAsString(JsonGet(types, 2, fieldType, "type")), "enum");
+                    isPrimitive = StrEquals(fieldType, "integer") ||
+                            StrEquals(fieldType, "boolean") ||
+                            StrEquals(fieldType, "float");
+
+
+                    StreamPrintf(implFile, "    if (val->%s)\n", key);
+                    StreamPrintf(implFile, "    {\n");
+                    StreamPrintf(implFile, "        char *objKey;\n");
+                    StreamPrintf(implFile, "        void *objVal;\n");
+                    StreamPrintf(implFile, "        HashMap *jsonObj = HashMapCreate();\n");
+                    StreamPrintf(implFile, "        if (!jsonObj)\n");
+                    StreamPrintf(implFile, "        {\n");
+                    StreamPrintf(implFile, "            JsonFree(json);\n");
+                    StreamPrintf(implFile, "            return NULL;\n");
+                    StreamPrintf(implFile, "        }\n");
+                    StreamPrintf(implFile, "        while (HashMapIterate(val->%s, &objKey, &objVal))\n", key);
+                    StreamPrintf(implFile, "        {\n");
+
+                    if (StrEquals(fieldType, "string"))
+                    {
+                        StreamPrintf(implFile, "            HashMapSet(jsonObj, objKey, JsonValueString(objVal));\n", key);
+                    }
+                    else if (!isPrimitive)
+                    {
+                        StreamPrintf(implFile, "            HashMapSet(jsonObj, objKey, JsonValueObject(%sToJson(objVal)));\n", fieldType, key);
+                    }
+                    else
+                    {
+                        char *cType;
+
+                        if (StrEquals(fieldType, "integer"))
+                        {
+                            cType = "int64_t";
+                        }
+                        else if (StrEquals(fieldType, "float"))
+                        {
+                            cType = "double";
+                        }
+                        else if (StrEquals(fieldType, "boolean"))
+                        {
+                            cType = "bool";
+                        }
+                        else
+                        {
+                            /* Should never happen */
+                            cType = NULL;
+                        }
+
+                        *fieldType = toupper(*fieldType);
+                        StreamPrintf(implFile, "            HashMapSet(jsonObj, objKey, (JsonValue%s(*((%s *) objVal))));\n", fieldType, cType, key);
+                        *fieldType = tolower(*fieldType);
+                    }
+
+                    StreamPrintf(implFile, "        }\n");
+                    StreamPrintf(implFile, "        HashMapSet(json, \"%s\", JsonValueObject(jsonObj));\n", Trim('_', key));
+                    StreamPrintf(implFile, "    }\n");
+
+                    fieldType[strlen(fieldType)] = '}';
+                }
+                else if (IsDelimited(fieldType, '[', ']'))
                 {
                     int isPrimitive;
 
@@ -1036,7 +1260,36 @@ Main(Array * args)
                 {
                     StreamPrintf(implFile, "    Free(val->%s);\n", key);
                 }
-                else if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']')
+                else if (IsDelimited(fieldType, '{', '}'))
+                {
+                    int isPrimitive;
+
+                    fieldType++;
+                    fieldType[strlen(fieldType) - 1] = '\0';
+                    isEnum = StrEquals(JsonValueAsString(JsonGet(types, 2, fieldType, "type")), "enum");
+                    isPrimitive = StrEquals(fieldType, "boolean") ||
+                            StrEquals(fieldType, "float") ||
+                            StrEquals(fieldType, "integer") ||
+                            StrEquals(fieldType, "string");
+
+                    StreamPrintf(implFile, "    if (val->%s)\n", key);
+                    StreamPrintf(implFile, "    {\n");
+                    StreamPrintf(implFile, "        char *objKey;\n");
+                    StreamPrintf(implFile, "        void *objVal;\n");
+                    StreamPrintf(implFile, "        while (HashMapIterate(val->%s, &objKey, &objVal))\n", key);
+                    StreamPrintf(implFile, "        {\n");
+                    StreamPrintf(implFile, "            %sFree(objVal);\n", (!isEnum && !isPrimitive) ? fieldType : "", key);
+                    if (!isEnum && !isPrimitive)
+                    {
+                        StreamPrintf(implFile, "            Free(objVal);\n", key);
+                    }
+                    StreamPrintf(implFile, "        }\n");
+                    StreamPrintf(implFile, "        HashMapFree(val->%s);\n", key);
+                    StreamPrintf(implFile, "    }\n");
+
+                    fieldType[strlen(fieldType)] = '}';
+                }
+                else if (IsDelimited(fieldType, '[', ']'))
                 {
                     int isPrimitive;