diff --git a/configure b/configure index 1afdd91..d50d752 100755 --- a/configure +++ b/configure @@ -15,7 +15,7 @@ TOOLS="tools" # Default compiler flags. These must be supported by all POSIX C compilers. # "Fancy" compilers that have additional options must be detected and set below. -CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE}" +CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE} -I${SRC}"; LIBS="-lm -lpthread" # Default args for all platforms. @@ -24,10 +24,10 @@ SCRIPT_ARGS="--prefix=/usr/local --lib-name=Cytoplasm" # Set SSL flags depending on the platform. case "$(uname)" in OpenBSD) - SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl" + SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl --disable-lmdb" ;; *) - SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl" + SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl --disable-lmdb" ;; esac @@ -80,6 +80,14 @@ for arg in $SCRIPT_ARGS; do TLS_IMPL="" TLS_LIBS="" ;; + --with-lmdb) + EDB_IMPL="EDB_LMDB" + EDB_LIBS="-llmdb" + ;; + --disable-lmdb) + EDB_IMPL="" + EDB_LIBS="" + ;; --prefix=*) PREFIX=$(echo "$arg" | cut -d '=' -f 2-) ;; @@ -104,6 +112,11 @@ if [ -n "$TLS_IMPL" ]; then LIBS="${LIBS} ${TLS_LIBS}" fi +if [ -n "$EDB_IMPL" ]; then + CFLAGS="${CFLAGS} -D${EDB_IMPL}" + LIBS="${LIBS} ${EDB_LIBS}" +fi + CFLAGS="${CFLAGS} '-DLIB_NAME=\"${LIB_NAME}\"' ${DEBUG}" LDFLAGS="${LIBS} ${LDFLAGS}" @@ -140,7 +153,7 @@ print_obj() { get_deps() { src="$1" - ${CC} -I${INCLUDE} -E "$src" \ + ${CC} -I${SRC} -I${INCLUDE} -E "$src" \ | grep '^#' \ | awk '{print $3}' \ | cut -d '"' -f 2 \ diff --git a/include/Cytoplasm/Db.h b/include/Cytoplasm/Db.h index f4c41d2..14a1064 100644 --- a/include/Cytoplasm/Db.h +++ b/include/Cytoplasm/Db.h @@ -48,6 +48,17 @@ */ typedef struct Db Db; +/** + * Some "hints" for the database backend for operations. + * Hints are a way for the program to declare what to except of it + * (and the program MUST adhere to these hints, but the backend + * MAY adhere). + */ +typedef enum DbHint { + DB_HINT_READONLY, /* The database reference is treated as read-only */ + DB_HINT_WRITE /* The database reference is treated as read/write */ +} DbHint; + /** * When an object is locked, a reference is returned. This reference * is owned by the current thread, and the database is inaccessible to @@ -68,6 +79,15 @@ typedef struct DbRef DbRef; */ extern Db * DbOpen(char *, size_t); +/** + * Open a LMDB data directory. This function is similar to + * .Fn DbOpen , + * but works with a LMDB-based backend, with the second argument + * being the maximum filesize. This function fails with NULL if ever + * called without LMDB enabled at compiletime. + */ +extern Db * DbOpenLMDB(char *, size_t); + /** * Close the database. This function will flush anything in the cache * to the disk, and then close the data directory. It assumes that @@ -109,6 +129,15 @@ extern DbRef * DbCreate(Db *, size_t,...); */ extern DbRef * DbLock(Db *, size_t,...); +/** + * Behaves like + * .Fn DbLock , + * but with hints on the reference itself, as + * .Fn DbLock + * itself is read/write. + */ +extern DbRef * DbLockIntent(Db *, DbHint, size_t,...); + /** * Immediately and permanently remove an object from the database. * This function assumes the object is not locked, otherwise undefined diff --git a/src/Db.c b/src/Db.c deleted file mode 100644 index 4128a89..0000000 --- a/src/Db.c +++ /dev/null @@ -1,977 +0,0 @@ -/* - * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net> - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation files - * (the "Software"), to deal in the Software without restriction, - * including without limitation the rights to use, copy, modify, merge, - * publish, distribute, sublicense, and/or sell copies of the Software, - * and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include - -#include -#include - -struct Db -{ - char *dir; - pthread_mutex_t lock; - - size_t cacheSize; - size_t maxCache; - HashMap *cache; - - /* - * The cache uses a double linked list (see DbRef - * below) to know which objects are most and least - * recently used. The following diagram helps me - * know what way all the pointers go, because it - * can get very confusing sometimes. For example, - * there's nothing stopping "next" from pointing to - * least recent, and "prev" from pointing to most - * recent, so hopefully this clarifies the pointer - * terminology used when dealing with the linked - * list: - * - * mostRecent leastRecent - * | prev prev | prev - * +---+ ---> +---+ ---> +---+ ---> NULL - * |ref| |ref| |ref| - * NULL <--- +---+ <--- +---+ <--- +---+ - * next next next - */ - DbRef *mostRecent; - DbRef *leastRecent; -}; - -struct DbRef -{ - HashMap *json; - - uint64_t ts; - size_t size; - - Array *name; - - DbRef *prev; - DbRef *next; - - int fd; - Stream *stream; -}; - -static void -StringArrayFree(Array * arr) -{ - size_t i; - - for (i = 0; i < ArraySize(arr); i++) - { - Free(ArrayGet(arr, i)); - } - - ArrayFree(arr); -} - -static ssize_t DbComputeSize(HashMap *); - -static ssize_t -DbComputeSizeOfValue(JsonValue * val) -{ - MemoryInfo *a; - ssize_t total = 0; - - size_t i; - - union - { - char *str; - Array *arr; - } u; - - if (!val) - { - return -1; - } - - a = MemoryInfoGet(val); - if (a) - { - total += MemoryInfoGetSize(a); - } - - switch (JsonValueType(val)) - { - case JSON_OBJECT: - total += DbComputeSize(JsonValueAsObject(val)); - break; - case JSON_ARRAY: - u.arr = JsonValueAsArray(val); - a = MemoryInfoGet(u.arr); - - if (a) - { - total += MemoryInfoGetSize(a); - } - - for (i = 0; i < ArraySize(u.arr); i++) - { - total += DbComputeSizeOfValue(ArrayGet(u.arr, i)); - } - break; - case JSON_STRING: - u.str = JsonValueAsString(val); - a = MemoryInfoGet(u.str); - if (a) - { - total += MemoryInfoGetSize(a); - } - break; - case JSON_NULL: - case JSON_INTEGER: - case JSON_FLOAT: - case JSON_BOOLEAN: - default: - /* These don't use any extra heap space */ - break; - } - return total; -} - -static ssize_t -DbComputeSize(HashMap * json) -{ - char *key; - JsonValue *val; - MemoryInfo *a; - size_t total; - - if (!json) - { - return -1; - } - - total = 0; - - a = MemoryInfoGet(json); - if (a) - { - total += MemoryInfoGetSize(a); - } - - while (HashMapIterate(json, &key, (void **) &val)) - { - a = MemoryInfoGet(key); - if (a) - { - total += MemoryInfoGetSize(a); - } - - total += DbComputeSizeOfValue(val); - } - - return total; -} - -static char * -DbHashKey(Array * args) -{ - size_t i; - char *str = NULL; - - for (i = 0; i < ArraySize(args); i++) - { - char *tmp = StrConcat(2, str, ArrayGet(args, i)); - - Free(str); - str = tmp; - } - - return str; -} - -static char -DbSanitiseChar(char input) -{ - switch (input) - { - case '/': - return '_'; - case '.': - return '-'; - } - return input; -} - -static char * -DbDirName(Db * db, Array * args, size_t strip) -{ - size_t i, j; - char *str = StrConcat(2, db->dir, "/"); - - for (i = 0; i < ArraySize(args) - strip; i++) - { - char *tmp; - char *sanitise = StrDuplicate(ArrayGet(args, i)); - for (j = 0; j < strlen(sanitise); j++) - { - sanitise[j] = DbSanitiseChar(sanitise[j]); - } - - tmp = StrConcat(3, str, sanitise, "/"); - - Free(str); - Free(sanitise); - - str = tmp; - } - - return str; -} - -static char * -DbFileName(Db * db, Array * args) -{ - size_t i; - char *str = StrConcat(2, db->dir, "/"); - - for (i = 0; i < ArraySize(args); i++) - { - char *tmp; - char *arg = StrDuplicate(ArrayGet(args, i)); - size_t j = 0; - - /* Sanitize name to prevent directory traversal attacks */ - while (arg[j]) - { - arg[j] = DbSanitiseChar(arg[j]); - j++; - } - - tmp = StrConcat(3, str, arg, - (i < ArraySize(args) - 1) ? "/" : ".json"); - - Free(arg); - Free(str); - - str = tmp; - } - - return str; -} - -static void -DbCacheEvict(Db * db) -{ - DbRef *ref = db->leastRecent; - DbRef *tmp; - - while (ref && db->cacheSize > db->maxCache) - { - char *hash; - - JsonFree(ref->json); - - hash = DbHashKey(ref->name); - HashMapDelete(db->cache, hash); - Free(hash); - - StringArrayFree(ref->name); - - db->cacheSize -= ref->size; - - if (ref->next) - { - ref->next->prev = ref->prev; - } - else - { - db->mostRecent = ref->prev; - } - - if (ref->prev) - { - ref->prev->next = ref->next; - } - else - { - db->leastRecent = ref->next; - } - - tmp = ref->next; - Free(ref); - - ref = tmp; - } -} - -Db * -DbOpen(char *dir, size_t cache) -{ - Db *db; - pthread_mutexattr_t attr; - - if (!dir) - { - return NULL; - } - - db = Malloc(sizeof(Db)); - if (!db) - { - return NULL; - } - - db->dir = dir; - db->maxCache = cache; - - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutex_init(&db->lock, &attr); - pthread_mutexattr_destroy(&attr); - - db->mostRecent = NULL; - db->leastRecent = NULL; - db->cacheSize = 0; - - if (db->maxCache) - { - db->cache = HashMapCreate(); - if (!db->cache) - { - return NULL; - } - } - else - { - db->cache = NULL; - } - - return db; -} - -void -DbMaxCacheSet(Db * db, size_t cache) -{ - if (!db) - { - return; - } - - pthread_mutex_lock(&db->lock); - - db->maxCache = cache; - if (db->maxCache && !db->cache) - { - db->cache = HashMapCreate(); - db->cacheSize = 0; - } - - DbCacheEvict(db); - - pthread_mutex_unlock(&db->lock); -} - -void -DbClose(Db * db) -{ - if (!db) - { - return; - } - - pthread_mutex_lock(&db->lock); - - DbMaxCacheSet(db, 0); - DbCacheEvict(db); - HashMapFree(db->cache); - - pthread_mutex_unlock(&db->lock); - pthread_mutex_destroy(&db->lock); - - Free(db); -} - -static DbRef * -DbLockFromArr(Db * db, Array * args) -{ - char *file; - char *hash; - DbRef *ref; - struct flock lock; - - int fd; - Stream *stream; - - if (!db || !args) - { - return NULL; - } - - ref = NULL; - hash = NULL; - - pthread_mutex_lock(&db->lock); - - /* Check if the item is in the cache */ - hash = DbHashKey(args); - ref = HashMapGet(db->cache, hash); - file = DbFileName(db, args); - - fd = open(file, O_RDWR); - if (fd == -1) - { - if (ref) - { - HashMapDelete(db->cache, hash); - JsonFree(ref->json); - StringArrayFree(ref->name); - - db->cacheSize -= ref->size; - - if (ref->next) - { - ref->next->prev = ref->prev; - } - else - { - db->mostRecent = ref->prev; - } - - if (ref->prev) - { - ref->prev->next = ref->next; - } - else - { - db->leastRecent = ref->next; - } - - if (!db->leastRecent) - { - db->leastRecent = db->mostRecent; - } - - Free(ref); - } - ref = NULL; - goto finish; - } - - stream = StreamFd(fd); - - lock.l_start = 0; - lock.l_len = 0; - lock.l_type = F_WRLCK; - lock.l_whence = SEEK_SET; - - /* Lock the file on the disk */ - if (fcntl(fd, F_SETLK, &lock) < 0) - { - StreamClose(stream); - ref = NULL; - goto finish; - } - - if (ref) /* In cache */ - { - uint64_t diskTs = UtilLastModified(file); - - ref->fd = fd; - ref->stream = stream; - - if (diskTs > ref->ts) - { - /* File was modified on disk since it was cached */ - HashMap *json = JsonDecode(ref->stream); - - if (!json) - { - StreamClose(ref->stream); - ref = NULL; - goto finish; - } - - JsonFree(ref->json); - ref->json = json; - ref->ts = diskTs; - ref->size = DbComputeSize(ref->json); - } - - /* Float this ref to mostRecent */ - if (ref->next) - { - ref->next->prev = ref->prev; - - if (!ref->prev) - { - db->leastRecent = ref->next; - } - else - { - ref->prev->next = ref->next; - } - - ref->prev = db->mostRecent; - ref->next = NULL; - if (db->mostRecent) - { - db->mostRecent->next = ref; - } - db->mostRecent = ref; - } - - /* If there is no least recent, this is the only thing in the - * cache, so it is also least recent. */ - if (!db->leastRecent) - { - db->leastRecent = ref; - } - - /* The file on disk may be larger than what we have in memory, - * which may require items in cache to be evicted. */ - DbCacheEvict(db); - } - else - { - Array *name; - size_t i; - - /* Not in cache; load from disk */ - - ref = Malloc(sizeof(DbRef)); - if (!ref) - { - StreamClose(stream); - goto finish; - } - - ref->json = JsonDecode(stream); - - if (!ref->json) - { - Free(ref); - StreamClose(stream); - ref = NULL; - goto finish; - } - - ref->fd = fd; - ref->stream = stream; - - name = ArrayCreate(); - for (i = 0; i < ArraySize(args); i++) - { - ArrayAdd(name, StrDuplicate(ArrayGet(args, i))); - } - ref->name = name; - - if (db->cache) - { - ref->ts = UtilTsMillis(); - ref->size = DbComputeSize(ref->json); - HashMapSet(db->cache, hash, ref); - db->cacheSize += ref->size; - - ref->next = NULL; - ref->prev = db->mostRecent; - if (db->mostRecent) - { - db->mostRecent->next = ref; - } - db->mostRecent = ref; - - if (!db->leastRecent) - { - db->leastRecent = ref; - } - - /* Adding this item to the cache may case it to grow too - * large, requiring some items to be evicted */ - DbCacheEvict(db); - } - } - -finish: - if (!ref) - { - pthread_mutex_unlock(&db->lock); - } - - Free(file); - Free(hash); - - return ref; -} - -DbRef * -DbCreate(Db * db, size_t nArgs,...) -{ - Stream *fp; - char *file; - char *dir; - va_list ap; - Array *args; - DbRef *ret; - - if (!db) - { - return NULL; - } - - va_start(ap, nArgs); - args = ArrayFromVarArgs(nArgs, ap); - va_end(ap); - - if (!args) - { - return NULL; - } - - pthread_mutex_lock(&db->lock); - - file = DbFileName(db, args); - - if (UtilLastModified(file)) - { - Free(file); - ArrayFree(args); - pthread_mutex_unlock(&db->lock); - return NULL; - } - - dir = DbDirName(db, args, 1); - if (UtilMkdir(dir, 0750) < 0) - { - Free(file); - ArrayFree(args); - Free(dir); - pthread_mutex_unlock(&db->lock); - return NULL; - } - - Free(dir); - - fp = StreamOpen(file, "w"); - Free(file); - if (!fp) - { - ArrayFree(args); - pthread_mutex_unlock(&db->lock); - return NULL; - } - - StreamPuts(fp, "{}"); - StreamClose(fp); - - /* DbLockFromArr() will lock again for us */ - pthread_mutex_unlock(&db->lock); - - ret = DbLockFromArr(db, args); - - ArrayFree(args); - - return ret; -} - -bool -DbDelete(Db * db, size_t nArgs,...) -{ - va_list ap; - Array *args; - char *file; - char *hash; - bool ret = true; - DbRef *ref; - - if (!db) - { - return false; - } - - va_start(ap, nArgs); - args = ArrayFromVarArgs(nArgs, ap); - va_end(ap); - - pthread_mutex_lock(&db->lock); - - hash = DbHashKey(args); - file = DbFileName(db, args); - - ref = HashMapGet(db->cache, hash); - if (ref) - { - HashMapDelete(db->cache, hash); - JsonFree(ref->json); - StringArrayFree(ref->name); - - db->cacheSize -= ref->size; - - if (ref->next) - { - ref->next->prev = ref->prev; - } - else - { - db->mostRecent = ref->prev; - } - - if (ref->prev) - { - ref->prev->next = ref->next; - } - else - { - db->leastRecent = ref->next; - } - - if (!db->leastRecent) - { - db->leastRecent = db->mostRecent; - } - - Free(ref); - } - - Free(hash); - - if (UtilLastModified(file)) - { - ret = (remove(file) == 0); - } - - pthread_mutex_unlock(&db->lock); - - ArrayFree(args); - Free(file); - return ret; -} - -DbRef * -DbLock(Db * db, size_t nArgs,...) -{ - va_list ap; - Array *args; - DbRef *ret; - - va_start(ap, nArgs); - args = ArrayFromVarArgs(nArgs, ap); - va_end(ap); - - if (!args) - { - return NULL; - } - - ret = DbLockFromArr(db, args); - - ArrayFree(args); - - return ret; -} - -bool -DbUnlock(Db * db, DbRef * ref) -{ - bool destroy; - - if (!db || !ref) - { - return false; - } - - lseek(ref->fd, 0L, SEEK_SET); - if (ftruncate(ref->fd, 0) < 0) - { - pthread_mutex_unlock(&db->lock); - Log(LOG_ERR, "Failed to truncate file on disk."); - Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno)); - return false; - } - - JsonEncode(ref->json, ref->stream, JSON_DEFAULT); - StreamClose(ref->stream); - - if (db->cache) - { - char *key = DbHashKey(ref->name); - - if (HashMapGet(db->cache, key)) - { - db->cacheSize -= ref->size; - ref->size = DbComputeSize(ref->json); - db->cacheSize += ref->size; - - /* If this ref has grown significantly since we last - * computed its size, it may have filled the cache and - * require some items to be evicted. */ - DbCacheEvict(db); - - destroy = false; - } - else - { - destroy = true; - } - - Free(key); - } - else - { - destroy = true; - } - - if (destroy) - { - JsonFree(ref->json); - StringArrayFree(ref->name); - - Free(ref); - } - - pthread_mutex_unlock(&db->lock); - return true; -} - -bool -DbExists(Db * db, size_t nArgs,...) -{ - va_list ap; - Array *args; - char *file; - bool ret; - - va_start(ap, nArgs); - args = ArrayFromVarArgs(nArgs, ap); - va_end(ap); - - if (!args) - { - return false; - } - - pthread_mutex_lock(&db->lock); - - file = DbFileName(db, args); - ret = (UtilLastModified(file) != 0); - - pthread_mutex_unlock(&db->lock); - - Free(file); - ArrayFree(args); - - return ret; -} - -Array * -DbList(Db * db, size_t nArgs,...) -{ - Array *result; - Array *path; - DIR *files; - struct dirent *file; - char *dir; - va_list ap; - - if (!db || !nArgs) - { - return NULL; - } - - result = ArrayCreate(); - if (!result) - { - return NULL; - } - - va_start(ap, nArgs); - path = ArrayFromVarArgs(nArgs, ap); - dir = DbDirName(db, path, 0); - - pthread_mutex_lock(&db->lock); - - files = opendir(dir); - if (!files) - { - ArrayFree(path); - ArrayFree(result); - Free(dir); - pthread_mutex_unlock(&db->lock); - return NULL; - } - while ((file = readdir(files))) - { - size_t namlen = strlen(file->d_name); - - if (namlen > 5) - { - int nameOffset = namlen - 5; - - if (StrEquals(file->d_name + nameOffset, ".json")) - { - file->d_name[nameOffset] = '\0'; - ArrayAdd(result, StrDuplicate(file->d_name)); - } - } - } - closedir(files); - - ArrayFree(path); - Free(dir); - pthread_mutex_unlock(&db->lock); - - return result; -} - -void -DbListFree(Array * arr) -{ - StringArrayFree(arr); -} - -HashMap * -DbJson(DbRef * ref) -{ - return ref ? ref->json : NULL; -} - -bool -DbJsonSet(DbRef * ref, HashMap * json) -{ - if (!ref || !json) - { - return false; - } - - JsonFree(ref->json); - ref->json = JsonDuplicate(json); - return true; -} diff --git a/src/Db/Db.c b/src/Db/Db.c new file mode 100644 index 0000000..1015bbf --- /dev/null +++ b/src/Db/Db.c @@ -0,0 +1,499 @@ +/* + * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net> + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include + +#include "Db/Internal.h" + +void +StringArrayFree(Array * arr) +{ + size_t i; + + for (i = 0; i < ArraySize(arr); i++) + { + Free(ArrayGet(arr, i)); + } + + ArrayFree(arr); +} + +static ssize_t DbComputeSize(HashMap *); + +static ssize_t +DbComputeSizeOfValue(JsonValue * val) +{ + MemoryInfo *a; + ssize_t total = 0; + + size_t i; + + union + { + char *str; + Array *arr; + } u; + + if (!val) + { + return -1; + } + + a = MemoryInfoGet(val); + if (a) + { + total += MemoryInfoGetSize(a); + } + + switch (JsonValueType(val)) + { + case JSON_OBJECT: + total += DbComputeSize(JsonValueAsObject(val)); + break; + case JSON_ARRAY: + u.arr = JsonValueAsArray(val); + a = MemoryInfoGet(u.arr); + + if (a) + { + total += MemoryInfoGetSize(a); + } + + for (i = 0; i < ArraySize(u.arr); i++) + { + total += DbComputeSizeOfValue(ArrayGet(u.arr, i)); + } + break; + case JSON_STRING: + u.str = JsonValueAsString(val); + a = MemoryInfoGet(u.str); + if (a) + { + total += MemoryInfoGetSize(a); + } + break; + case JSON_NULL: + case JSON_INTEGER: + case JSON_FLOAT: + case JSON_BOOLEAN: + default: + /* These don't use any extra heap space */ + break; + } + return total; +} + +static ssize_t +DbComputeSize(HashMap * json) +{ + char *key; + JsonValue *val; + MemoryInfo *a; + size_t total; + + if (!json) + { + return -1; + } + + total = 0; + + a = MemoryInfoGet(json); + if (a) + { + total += MemoryInfoGetSize(a); + } + + while (HashMapIterate(json, &key, (void **) &val)) + { + a = MemoryInfoGet(key); + if (a) + { + total += MemoryInfoGetSize(a); + } + + total += DbComputeSizeOfValue(val); + } + + return total; +} + +static char * +DbHashKey(Array * args) +{ + size_t i; + char *str = NULL; + + for (i = 0; i < ArraySize(args); i++) + { + char *tmp = StrConcat(2, str, ArrayGet(args, i)); + + Free(str); + str = tmp; + } + + return str; +} + +static void +DbCacheEvict(Db * db) +{ + DbRef *ref = db->leastRecent; + DbRef *tmp; + + while (ref && db->cacheSize > db->maxCache) + { + char *hash; + + JsonFree(ref->json); + + hash = DbHashKey(ref->name); + HashMapDelete(db->cache, hash); + Free(hash); + + StringArrayFree(ref->name); + + db->cacheSize -= ref->size; + + if (ref->next) + { + ref->next->prev = ref->prev; + } + else + { + db->mostRecent = ref->prev; + } + + if (ref->prev) + { + ref->prev->next = ref->next; + } + else + { + db->leastRecent = ref->next; + } + + tmp = ref->next; + Free(ref); + + ref = tmp; + } +} + +void +DbMaxCacheSet(Db * db, size_t cache) +{ + if (!db) + { + return; + } + + pthread_mutex_lock(&db->lock); + + db->maxCache = cache; + if (db->maxCache && !db->cache) + { + db->cache = HashMapCreate(); + db->cacheSize = 0; + } + + DbCacheEvict(db); + + pthread_mutex_unlock(&db->lock); +} + +void +DbClose(Db * db) +{ + if (!db) + { + return; + } + + pthread_mutex_lock(&db->lock); + if (db->close) + { + db->close(db); + } + DbMaxCacheSet(db, 0); + DbCacheEvict(db); + HashMapFree(db->cache); + + pthread_mutex_unlock(&db->lock); + pthread_mutex_destroy(&db->lock); + + Free(db); +} + +DbRef * +DbCreate(Db * db, size_t nArgs,...) +{ + va_list ap; + Array *args; + DbRef *ret; + + if (!db || !db->create) + { + return NULL; + } + + va_start(ap, nArgs); + args = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + if (!args) + { + return NULL; + } + + ret = db->create(db, args); + ArrayFree(args); + + return ret; +} + +bool +DbDelete(Db * db, size_t nArgs,...) +{ + va_list ap; + Array *args; + bool ret = true; + + if (!db) + { + return false; + } + + va_start(ap, nArgs); + args = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + ret = db->delete(db, args); + + ArrayFree(args); + return ret; +} + +DbRef * +DbLock(Db * db, size_t nArgs,...) +{ + va_list ap; + Array *args; + DbRef *ret; + + va_start(ap, nArgs); + args = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + if (!args || !db->lockFunc) + { + return NULL; + } + + ret = db->lockFunc(db, DB_HINT_WRITE, args); + + ArrayFree(args); + + return ret; +} + +DbRef * +DbLockIntent(Db * db, DbHint hint, size_t nArgs,...) +{ + va_list ap; + Array *args; + DbRef *ret; + + va_start(ap, nArgs); + args = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + if (!args || !db->lockFunc) + { + return NULL; + } + + ret = db->lockFunc(db, hint, args); + + ArrayFree(args); + + return ret; +} + +bool +DbUnlock(Db * db, DbRef * ref) +{ + if (!db || !ref || !db->unlock) + { + return false; + } + + return db->unlock(db, ref); +} + +bool +DbExists(Db * db, size_t nArgs,...) +{ + va_list ap; + Array *args; + bool ret; + if (!db || !nArgs || !db->exists) + { + return false; + } + + va_start(ap, nArgs); + args = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + if (!args) + { + return false; + } + + ret = db->exists(db, args); + ArrayFree(args); + + return ret; +} + +Array * +DbList(Db * db, size_t nArgs,...) +{ + Array *result; + Array *path; + va_list ap; + + if (!db || !nArgs || !db->list) + { + return NULL; + } + + va_start(ap, nArgs); + path = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + result = db->list(db, path); + + ArrayFree(path); + return result; +} + +void +DbListFree(Array * arr) +{ + StringArrayFree(arr); +} + +HashMap * +DbJson(DbRef * ref) +{ + return ref ? ref->json : NULL; +} + +bool +DbJsonSet(DbRef * ref, HashMap * json) +{ + if (!ref || !json) + { + return false; + } + + JsonFree(ref->json); + ref->json = JsonDuplicate(json); + return true; +} +void +DbInit(Db *db) +{ + pthread_mutexattr_t attr; + if (!db) + { + return; + } + + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&db->lock, &attr); + pthread_mutexattr_destroy(&attr); + + db->mostRecent = NULL; + db->leastRecent = NULL; + db->cacheSize = 0; + db->maxCache = 0; + + if (db->maxCache) + { + db->cache = HashMapCreate(); + } + else + { + db->cache = NULL; + } +} +void +DbRefInit(Db *db, DbRef *ref) +{ + if (!db || !ref) + { + return; + } + ref->prev = db->mostRecent; + ref->next = NULL; + ref->json = NULL; + ref->name = NULL; + + /* As default values to be overwritten by impls */ + ref->ts = UtilTsMillis(); + ref->size = 0; + + /* TODO: Append the ref to the cache list. + * I removed it because it broke everything and crashed all the time. + * My bad! */ +} +void +StringArrayAppend(Array *arr, char *str) +{ + if (!arr || !str) + { + return; + } + ArrayAdd(arr, StrDuplicate(str)); +} diff --git a/src/Db/Flat.c b/src/Db/Flat.c new file mode 100644 index 0000000..df5685c --- /dev/null +++ b/src/Db/Flat.c @@ -0,0 +1,374 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include + +#include "Db/Internal.h" + +typedef struct FlatDb { + Db base; + char *dir; + /* Theres not much to do here. */ +} FlatDb; +typedef struct FlatDbRef { + DbRef base; + + Stream *stream; + int fd; +} FlatDbRef; + +static char +DbSanitiseChar(char input) +{ + switch (input) + { + case '/': + return '_'; + case '.': + return '-'; + } + return input; +} + +static char * +DbDirName(FlatDb * db, Array * args, size_t strip) +{ + size_t i, j; + char *str = StrConcat(2, db->dir, "/"); + + for (i = 0; i < ArraySize(args) - strip; i++) + { + char *tmp; + char *sanitise = StrDuplicate(ArrayGet(args, i)); + for (j = 0; j < strlen(sanitise); j++) + { + sanitise[j] = DbSanitiseChar(sanitise[j]); + } + + tmp = StrConcat(3, str, sanitise, "/"); + + Free(str); + Free(sanitise); + + str = tmp; + } + + return str; +} +static char * +DbFileName(FlatDb * db, Array * args) +{ + size_t i; + char *str = StrConcat(2, db->dir, "/"); + + for (i = 0; i < ArraySize(args); i++) + { + char *tmp; + char *arg = StrDuplicate(ArrayGet(args, i)); + size_t j = 0; + + /* Sanitize name to prevent directory traversal attacks */ + while (arg[j]) + { + arg[j] = DbSanitiseChar(arg[j]); + j++; + } + + tmp = StrConcat(3, str, arg, + (i < ArraySize(args) - 1) ? "/" : ".json"); + + Free(arg); + Free(str); + + str = tmp; + } + + return str; +} + +static DbRef * +FlatLock(Db *d, DbHint hint, Array *dir) +{ + FlatDb *db = (FlatDb *) d; + FlatDbRef *ref = NULL; + size_t i; + char *path = NULL; + if (!d || !dir) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + path = DbFileName(db, dir); + /* TODO: Caching */ + { + int fd = open(path, O_RDWR); + Stream *stream; + struct flock lock; + if (fd == -1) + { + ref = NULL; + goto end; + } + + stream = StreamFd(fd); + if (!stream) + { + ref = NULL; + goto end; + } + + lock.l_start = 0; + lock.l_len = 0; + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + + if (fcntl(fd, F_SETLK, &lock) < 0) + { + StreamClose(stream); + ref = NULL; + goto end; + } + + ref = Malloc(sizeof(*ref)); + DbRefInit(d, (DbRef *) ref); + /* TODO: Hints */ + ref->base.hint = hint; + ref->base.ts = UtilLastModified(path); + ref->base.json = JsonDecode(stream); + ref->stream = stream; + ref->fd = fd; + if (!ref->base.json) + { + Free(ref); + StreamClose(stream); + ref = NULL; + goto end; + } + + + ref->base.name = ArrayCreate(); + for (i = 0; i < ArraySize(dir); i++) + { + StringArrayAppend(ref->base.name, ArrayGet(dir, i)); + } + } +end: + Free(path); + if (!ref) + { + pthread_mutex_unlock(&d->lock); + } + return (DbRef *) ref; +} + +static bool +FlatUnlock(Db *d, DbRef *r) +{ + FlatDbRef *ref = (FlatDbRef *) r; + + if (!d || !r) + { + return false; + } + + lseek(ref->fd, 0L, SEEK_SET); + if (ftruncate(ref->fd, 0) < 0) + { + pthread_mutex_unlock(&d->lock); + Log(LOG_ERR, "Failed to truncate file on disk."); + Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno)); + return false; + } + + JsonEncode(ref->base.json, ref->stream, JSON_DEFAULT); + StreamClose(ref->stream); + + JsonFree(ref->base.json); + StringArrayFree(ref->base.name); + Free(ref); + + pthread_mutex_unlock(&d->lock); + return true; +} +static DbRef * +FlatCreate(Db *d, Array *dir) +{ + FlatDb *db = (FlatDb *) d; + char *path, *dirPath; + Stream *fp; + DbRef *ret; + + if (!d || !dir) + { + return NULL; + } + pthread_mutex_lock(&d->lock); + + path = DbFileName(db, dir); + if (UtilLastModified(path)) + { + Free(path); + pthread_mutex_unlock(&d->lock); + return NULL; + } + + dirPath = DbDirName(db, dir, 1); + if (UtilMkdir(dirPath, 0750) < 0) + { + Free(path); + Free(dirPath); + pthread_mutex_unlock(&d->lock); + return NULL; + } + Free(dirPath); + + fp = StreamOpen(path, "w"); + Free(path); + if (!fp) + { + pthread_mutex_unlock(&d->lock); + return NULL; + } + + StreamPuts(fp, "{}"); + StreamClose(fp); + + /* FlatLock() will lock again for us */ + pthread_mutex_unlock(&d->lock); + ret = FlatLock(d, DB_HINT_WRITE, dir); + + return ret; +} + +static bool +FlatDelete(Db *d, Array *dir) +{ + bool ret = false; + char *file; + FlatDb *db = (FlatDb *) d; + if (!d || !dir) + { + return false; + } + + pthread_mutex_lock(&d->lock); + file = DbFileName(db, dir); + + /* TODO: Unlink the entry from the linkedlist */ + if (UtilLastModified(file)) + { + ret = remove(file) == 0; + } + + Free(file); + pthread_mutex_unlock(&d->lock); + return ret; +} + +static Array * +FlatList(Db *d, Array *dir) +{ + FlatDb *db = (FlatDb *) d; + struct dirent *file; + Array *ret; + DIR *files; + char *path; + + if (!d || !dir) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + + path = DbDirName(db, dir, 0); + files = opendir(path); + if (!files) + { + Free(path); + pthread_mutex_unlock(&d->lock); + return NULL; + } + + ret = ArrayCreate(); + while ((file = readdir(files))) + { + size_t namlen = strlen(file->d_name); + + if (namlen > 5) + { + int nameOffset = namlen - 5; + + if (StrEquals(file->d_name + nameOffset, ".json")) + { + file->d_name[nameOffset] = '\0'; + StringArrayAppend(ret, file->d_name); + } + } + } + closedir(files); + Free(path); + + pthread_mutex_unlock(&d->lock); + return ret; +} +static bool +FlatExists(Db *d, Array *dir) +{ + FlatDb *db = (FlatDb *) d; + char *path; + bool ret; + if (!d || !dir) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + + path = DbFileName(db, dir); + ret = UtilLastModified(path) != 0; + Free(path); + + pthread_mutex_unlock(&d->lock); + + return ret; +} + +Db * +DbOpen(char *dir, size_t cache) +{ + FlatDb *db; + if (!dir) + { + return NULL; + } + db = Malloc(sizeof(*db)); + DbInit((Db *) db); + db->dir = dir; + db->base.cacheSize = cache; + + db->base.lockFunc = FlatLock; + db->base.unlock = FlatUnlock; + db->base.create = FlatCreate; + db->base.delete = FlatDelete; + db->base.exists = FlatExists; + db->base.list = FlatList; + db->base.close = NULL; + + return (Db *) db; +} diff --git a/src/Db/Internal.h b/src/Db/Internal.h new file mode 100644 index 0000000..1f1014a --- /dev/null +++ b/src/Db/Internal.h @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net> + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef CYTOPLASM_DB_INTERNAL_H +#define CYTOPLASM_DB_INTERNAL_H + +#include +#include + +#include + +/* TODO: Define the base structures to define a database internally. + * All "implementations" shall have them as a first element, so that + * basic, general functions can work on them properly. */ + +/* The base structure of a database */ +struct Db +{ + pthread_mutex_t lock; + + size_t cacheSize; + size_t maxCache; + HashMap *cache; + + /* + * The cache uses a double linked list (see DbRef + * below) to know which objects are most and least + * recently used. The following diagram helps me + * know what way all the pointers go, because it + * can get very confusing sometimes. For example, + * there's nothing stopping "next" from pointing to + * least recent, and "prev" from pointing to most + * recent, so hopefully this clarifies the pointer + * terminology used when dealing with the linked + * list: + * + * mostRecent leastRecent + * | prev prev | prev + * +---+ ---> +---+ ---> +---+ ---> NULL + * |ref| |ref| |ref| + * NULL <--- +---+ <--- +---+ <--- +---+ + * next next next + */ + DbRef *mostRecent; + DbRef *leastRecent; + + /* Functions for implementation-specific operations + * (opening a ref, closing a db, removing an entry, ...) */ + DbRef * (*lockFunc)(Db *, DbHint, Array *); + DbRef * (*create)(Db *, Array *); + Array * (*list)(Db *, Array *); + bool (*unlock)(Db *, DbRef *); + bool (*delete)(Db *, Array *); + bool (*exists)(Db *, Array *); + void (*close)(Db *); + + /* Implementation-specific constructs */ +}; + +struct DbRef +{ + HashMap *json; + + uint64_t ts; + size_t size; + + Array *name; + + DbRef *prev; + DbRef *next; + + DbHint hint; + /* Implementation-specific constructs */ +}; + +extern void DbInit(Db *); +extern void DbRefInit(Db *, DbRef *); +extern void StringArrayFree(Array *); +extern void StringArrayAppend(Array *, char *); + +#endif diff --git a/src/Db/LMDB.c b/src/Db/LMDB.c new file mode 100644 index 0000000..044c439 --- /dev/null +++ b/src/Db/LMDB.c @@ -0,0 +1,590 @@ +#include +#include +#include +#include + +#include "Db/Internal.h" + +#include +#include +#include + +#ifdef EDB_LMDB + +#include + +typedef struct LMDB { + Db base; /* The base implementation required to pass */ + + MDB_env *environ; + MDB_dbi dbi; +} LMDB; +typedef struct LMDBRef { + DbRef base; + + /* TODO: LMDB-dependent stuff */ + MDB_txn *transaction; + MDB_dbi dbi; +} LMDBRef; + +/* Helper functions */ +static MDB_val +LMDBTranslateKey(Array *key) +{ + MDB_val ret = { 0 }; + char *data = NULL; + size_t length = 0; + size_t i; + if (!key || ArraySize(key) > 255) + { + return ret; + } + + data = Realloc(data, ++length); + data[0] = ArraySize(key); + + /* Now, let's push every item */ + for (i = 0; i < ArraySize(key); i++) + { + char *entry = ArrayGet(key, i); + size_t offset = length; + + data = Realloc(data, (length += strlen(entry) + 1)); + memcpy(data + offset, entry, strlen(entry)); + data[length - 1] = '\0'; + } + + /* We now have every key */ + ret.mv_size = length; + ret.mv_data = data; + return ret; +} +static void +LMDBKillKey(MDB_val key) +{ + if (!key.mv_data || !key.mv_size) + { + return; + } + + Free(key.mv_data); +} +static HashMap * +LMDBDecode(MDB_val val) +{ + FILE *fakefile; + Stream *fakestream; + HashMap *ret; + if (!val.mv_data || !val.mv_size) + { + return NULL; + } + + fakefile = fmemopen(val.mv_data, val.mv_size, "r"); + fakestream = StreamFile(fakefile); + ret = JsonDecode(fakestream); + StreamClose(fakestream); + + return ret; +} +static bool +LMDBKeyStartsWith(MDB_val key, MDB_val starts) +{ + size_t i; + if (!key.mv_size || !starts.mv_size) + { + return false; + } + if (key.mv_size < starts.mv_size) + { + return false; + } + + for (i = 0; i < starts.mv_size; i++) + { + char keyC = ((char *) key.mv_data)[i]; + char startC = ((char *) starts.mv_data)[i]; + + if (keyC != startC) + { + return false; + } + } + return true; +} +static char * +LMDBKeyHead(MDB_val key) +{ + char *end; + if (!key.mv_size || !key.mv_data) + { + return NULL; + } + /* -2 because we have a NULL byte in there */ + end = ((char *) key.mv_data) + key.mv_size - 1; + if ((void *) end < key.mv_data) + { + /* Uh oh. */ + return NULL; + } + + while ((void *) (end - 1) >= key.mv_data && *(end - 1)) + { + end--; + } + return end; +} + +static DbRef * +LMDBLock(Db *d, DbHint hint, Array *k) +{ + LMDB *db = (LMDB *) d; + MDB_txn *transaction; + LMDBRef *ret = NULL; + MDB_val key, json_val; + int code, flags; + if (!d || !k) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + key = LMDBTranslateKey(k); + + /* Create a transaction, honoring hints. */ + /* TODO: Do we want to create a "main" transaction that everyone inherits + * from? */ + flags = hint == DB_HINT_READONLY ? MDB_RDONLY : 0; + if ((code = mdb_txn_begin(db->environ, NULL, flags, &transaction)) != 0) + { + /* Very bad! */ + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + goto end; + } + + json_val.mv_size = 0; + json_val.mv_data = NULL; + code = mdb_get(transaction, db->dbi, &key, &json_val); + if (code == MDB_NOTFOUND) + { + mdb_txn_abort(transaction); + goto end; + } + else if (code != 0) + { + Log(LOG_ERR, + "%s: mdb_get failure: %s", + __func__, mdb_strerror(code) + ); + mdb_txn_abort(transaction); + goto end; + } + + ret = Malloc(sizeof(*ret)); + DbRefInit(d, (DbRef *) ret); + /* TODO: Timestamp */ + { + size_t i; + ret->base.name = ArrayCreate(); + for (i = 0; i < ArraySize(k); i++) + { + char *ent = ArrayGet(k, i); + StringArrayAppend(ret->base.name, ent); + } + } + ret->base.json = LMDBDecode(json_val); + ret->base.hint = hint; + ret->transaction = NULL; + + if (hint == DB_HINT_WRITE) + { + ret->transaction = transaction; + } + else + { + mdb_txn_abort(transaction); + } +end: + if (!ret || hint == DB_HINT_READONLY) + { + pthread_mutex_unlock(&d->lock); + } + LMDBKillKey(key); + return (DbRef *) ret; +} +static bool +LMDBExists(Db *d, Array *k) +{ + MDB_val key, empty; + LMDB *db = (LMDB *) d; + MDB_txn *transaction; + int code; + bool ret = false; + if (!d || !k) + { + return false; + } + + pthread_mutex_lock(&d->lock); + key = LMDBTranslateKey(k); + + /* create a txn */ + if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0) + { + /* Very bad! */ + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + goto end; + } + + ret = mdb_get(transaction, db->dbi, &key, &empty) == 0; + mdb_txn_abort(transaction); +end: + LMDBKillKey(key); + pthread_mutex_unlock(&d->lock); + return ret; +} +static bool +LMDBDelete(Db *d, Array *k) +{ + MDB_val key, empty; + LMDB *db = (LMDB *) d; + MDB_txn *transaction; + int code; + bool ret = false; + if (!d || !k) + { + return false; + } + + pthread_mutex_lock(&d->lock); + key = LMDBTranslateKey(k); + + /* create a txn */ + if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0) + { + /* Very bad! */ + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + goto end; + } + + ret = mdb_del(transaction, db->dbi, &key, &empty) == 0; + mdb_txn_commit(transaction); +end: + LMDBKillKey(key); + pthread_mutex_unlock(&d->lock); + return ret; +} + +static bool +LMDBUnlock(Db *d, DbRef *r) +{ + LMDBRef *ref = (LMDBRef *) r; + LMDB *db = (LMDB *) d; + FILE *fakestream; + Stream *stream; + MDB_val key, val; + bool ret = true; + + if (!d || !r) + { + return false; + } + + val.mv_data = NULL; + val.mv_size = 0; + if (ref->transaction && r->hint == DB_HINT_WRITE) + { + key = LMDBTranslateKey(r->name); + + fakestream = open_memstream((char **) &val.mv_data, &val.mv_size); + stream = StreamFile(fakestream); + JsonEncode(r->json, stream, JSON_DEFAULT); + StreamFlush(stream); + StreamClose(stream); + + ret = mdb_put(ref->transaction, db->dbi, &key, &val, 0) == 0; + + mdb_txn_commit(ref->transaction); + LMDBKillKey(key); + } + StringArrayFree(ref->base.name); + JsonFree(ref->base.json); + Free(ref); + + if (val.mv_data) + { + free(val.mv_data); + } + if (ret && r->hint == DB_HINT_WRITE) + { + pthread_mutex_unlock(&d->lock); + } + return ret; +} +static DbRef * +LMDBCreate(Db *d, Array *k) +{ + LMDB *db = (LMDB *) d; + MDB_txn *transaction; + LMDBRef *ret = NULL; + MDB_val key, empty_json; + int code; + if (!d || !k) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + key = LMDBTranslateKey(k); + + /* create a txn */ + if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0) + { + /* Very bad! */ + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + goto end; + } + + empty_json.mv_size = 2; + empty_json.mv_data = "{}"; + /* put data in it */ + code = mdb_put(transaction, db->dbi, &key, &empty_json, MDB_NOOVERWRITE); + if (code == MDB_KEYEXIST) + { + mdb_txn_abort(transaction); + goto end; + } + else if (code == MDB_MAP_FULL) + { + Log(LOG_ERR, "%s: db is full", __func__); + mdb_txn_abort(transaction); + goto end; + } + else if (code != 0) + { + Log(LOG_ERR, "%s: mdb_put failure: %s", __func__, mdb_strerror(code)); + mdb_txn_abort(transaction); + goto end; + } + + ret = Malloc(sizeof(*ret)); + DbRefInit(d, (DbRef *) ret); + /* TODO: Timestamp */ + { + size_t i; + ret->base.name = ArrayCreate(); + for (i = 0; i < ArraySize(k); i++) + { + char *ent = ArrayGet(k, i); + StringArrayAppend(ret->base.name, ent); + } + } + ret->base.hint = DB_HINT_WRITE; + ret->base.json = HashMapCreate(); + ret->transaction = transaction; +end: + if (!ret) + { + pthread_mutex_unlock(&d->lock); + } + LMDBKillKey(key); + return (DbRef *) ret; +} + +static Array * +LMDBList(Db *d, Array *k) +{ + LMDB *db = (LMDB *) d; + MDB_val key = { 0 }; + MDB_val subKey; + MDB_val val; + Array *ret = NULL; + + MDB_cursor *cursor; + MDB_cursor_op op = MDB_SET_RANGE; + MDB_txn *txn; + int code; + + if (!d || !k) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + + /* Marked as read-only, as we just don't need to write anything + * when listing */ + if ((code = mdb_txn_begin(db->environ, NULL, MDB_RDONLY, &txn)) != 0) + { + /* Very bad! */ + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + goto end; + } + if ((code = mdb_cursor_open(txn, db->dbi, &cursor)) != 0) + { + Log(LOG_ERR, + "%s: could not get cursor: %s", + __func__, mdb_strerror(code) + ); + mdb_txn_abort(txn); + goto end; + } + + key = LMDBTranslateKey(k); + + /* Small hack to get it to list subitems */ + ((char *) key.mv_data)[0]++; + + ret = ArrayCreate(); + subKey = key; + while (mdb_cursor_get(cursor, &subKey, &val, op) == 0) + { + /* This searches by *increasing* order. The problem is that it may + * extend to unwanted points. Since the values are sorted, we can + * just exit if the subkey is incorrect. */ + char *head = LMDBKeyHead(subKey); + if (!LMDBKeyStartsWith(subKey, key)) + { + break; + } + + StringArrayAppend(ret, head); + op = MDB_NEXT; + } + + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + +end: + LMDBKillKey(key); + pthread_mutex_unlock(&d->lock); + return ret; +} + +/* Implementation functions */ +static void +LMDBClose(Db *d) +{ + LMDB *db = (LMDB *) d; + if (!d) + { + return; + } + + mdb_dbi_close(db->environ, db->dbi); + mdb_env_close(db->environ); +} + +Db * +DbOpenLMDB(char *dir, size_t size) +{ + /* TODO */ + MDB_env *env = NULL; + MDB_txn *txn; + MDB_dbi dbi; + int code; + LMDB *db; + if (!dir || !size) + { + return NULL; + } + + /* Try initialising LMDB */ + if ((code = mdb_env_create(&env)) != 0) + { + Log(LOG_ERR, + "%s: could not create LMDB env: %s", + __func__, mdb_strerror(code) + ); + return NULL; + } + if ((code = mdb_env_set_mapsize(env, size) != 0)) + { + Log(LOG_ERR, + "%s: could not set mapsize to %lu: %s", + __func__, (unsigned long) size, + mdb_strerror(code) + ); + mdb_env_close(env); + return NULL; + } + mdb_env_set_maxdbs(env, 4); + if ((code = mdb_env_open(env, dir, MDB_NOTLS, 0644)) != 0) + { + Log(LOG_ERR, + "%s: could not open LMDB env: %s", + __func__, mdb_strerror(code) + ); + mdb_env_close(env); + return NULL; + } + + /* Initialise a DBI */ + { + if ((code = mdb_txn_begin(env, NULL, 0, &txn)) != 0) + { + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + mdb_env_close(env); + return NULL; + } + if ((code = mdb_dbi_open(txn, "db", MDB_CREATE, &dbi)) != 0) + { + Log(LOG_ERR, + "%s: could not get transaction dbi: %s", + __func__, mdb_strerror(code) + ); + mdb_txn_abort(txn); + mdb_env_close(env); + return NULL; + } + mdb_txn_commit(txn); + } + + + db = Malloc(sizeof(*db)); + DbInit((Db *) db); + db->environ = env; + db->dbi = dbi; + + db->base.lockFunc = LMDBLock; + db->base.create = LMDBCreate; + db->base.unlock = LMDBUnlock; + db->base.delete = LMDBDelete; + db->base.exists = LMDBExists; + db->base.close = LMDBClose; + db->base.list = LMDBList; + + return (Db *) db; +} + +#else +Db * +DbOpenLMDB(char *dir, size_t size) +{ + /* Unimplemented function */ + Log(LOG_ERR, + "LMDB support is not enabled. Please compile with --use-lmdb" + ); + (void) size; + (void) dir; + return NULL; +} +#endif