diff --git a/configure b/configure index 1afdd91..3fb7bc6 100755 --- a/configure +++ b/configure @@ -15,7 +15,7 @@ TOOLS="tools" # Default compiler flags. These must be supported by all POSIX C compilers. # "Fancy" compilers that have additional options must be detected and set below. -CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE}" +CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE} -I${SRC}"; LIBS="-lm -lpthread" # Default args for all platforms. @@ -24,10 +24,10 @@ SCRIPT_ARGS="--prefix=/usr/local --lib-name=Cytoplasm" # Set SSL flags depending on the platform. case "$(uname)" in OpenBSD) - SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl" + SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl --disable-lmdb" ;; *) - SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl" + SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl --disable-lmdb" ;; esac @@ -80,6 +80,14 @@ for arg in $SCRIPT_ARGS; do TLS_IMPL="" TLS_LIBS="" ;; + --with-lmdb) + EDB_IMPL="EDB_LMDB" + EDB_LIBS="-llmdb" + ;; + --disable-lmdb) + EDB_IMPL="" + EDB_LIBS="" + ;; --prefix=*) PREFIX=$(echo "$arg" | cut -d '=' -f 2-) ;; @@ -104,6 +112,11 @@ if [ -n "$TLS_IMPL" ]; then LIBS="${LIBS} ${TLS_LIBS}" fi +if [ -n "$EDB_IMPL" ]; then + CFLAGS="${CFLAGS} -D${EDB_IMPL}" + LIBS="${LIBS} ${EDB_LIBS}" +fi + CFLAGS="${CFLAGS} '-DLIB_NAME=\"${LIB_NAME}\"' ${DEBUG}" LDFLAGS="${LIBS} ${LDFLAGS}" @@ -140,7 +153,7 @@ print_obj() { get_deps() { src="$1" - ${CC} -I${INCLUDE} -E "$src" \ + ${CC} -I${SRC} -I${INCLUDE} -E "$src" \ | grep '^#' \ | awk '{print $3}' \ | cut -d '"' -f 2 \ @@ -256,8 +269,9 @@ ${TAB}done ${LIB_NAME}: ${OUT}/lib/lib${LIB_NAME}.a ${OUT}/lib/lib${LIB_NAME}.so install: ${LIB_NAME} -${TAB}install -D ${OUT}/lib/lib${LIB_NAME}.a \$(PREFIX)/lib/lib${LIB_NAME}.a -${TAB}install -D ${OUT}/lib/lib${LIB_NAME}.so \$(PREFIX)/lib/lib${LIB_NAME}.so +${TAB}mkdir -p ${OUT}/lib +${TAB}cp ${OUT}/lib/lib${LIB_NAME}.a \$(PREFIX)/lib/lib${LIB_NAME}.a +${TAB}cp ${OUT}/lib/lib${LIB_NAME}.so \$(PREFIX)/lib/lib${LIB_NAME}.so $(collect ${INCLUDE}/ '' '' \$\(PREFIX\)/include/${LIB_NAME}/ install_out) $(collect ${INCLUDE}/ .h .3 \$\(PREFIX\)/man/man3/${LIB_NAME}- install_man) $(collect ${TOOLS}/ '.c' '' \$\(PREFIX\)/bin/ install_tool) diff --git a/include/Cytoplasm/Db.h b/include/Cytoplasm/Db.h index f4c41d2..14a1064 100644 --- a/include/Cytoplasm/Db.h +++ b/include/Cytoplasm/Db.h @@ -48,6 +48,17 @@ */ typedef struct Db Db; +/** + * Some "hints" for the database backend for operations. + * Hints are a way for the program to declare what to except of it + * (and the program MUST adhere to these hints, but the backend + * MAY adhere). + */ +typedef enum DbHint { + DB_HINT_READONLY, /* The database reference is treated as read-only */ + DB_HINT_WRITE /* The database reference is treated as read/write */ +} DbHint; + /** * When an object is locked, a reference is returned. This reference * is owned by the current thread, and the database is inaccessible to @@ -68,6 +79,15 @@ typedef struct DbRef DbRef; */ extern Db * DbOpen(char *, size_t); +/** + * Open a LMDB data directory. This function is similar to + * .Fn DbOpen , + * but works with a LMDB-based backend, with the second argument + * being the maximum filesize. This function fails with NULL if ever + * called without LMDB enabled at compiletime. + */ +extern Db * DbOpenLMDB(char *, size_t); + /** * Close the database. This function will flush anything in the cache * to the disk, and then close the data directory. It assumes that @@ -109,6 +129,15 @@ extern DbRef * DbCreate(Db *, size_t,...); */ extern DbRef * DbLock(Db *, size_t,...); +/** + * Behaves like + * .Fn DbLock , + * but with hints on the reference itself, as + * .Fn DbLock + * itself is read/write. + */ +extern DbRef * DbLockIntent(Db *, DbHint, size_t,...); + /** * Immediately and permanently remove an object from the database. * This function assumes the object is not locked, otherwise undefined diff --git a/src/Db.c b/src/Db.c deleted file mode 100644 index 4128a89..0000000 --- a/src/Db.c +++ /dev/null @@ -1,977 +0,0 @@ -/* - * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net> - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation files - * (the "Software"), to deal in the Software without restriction, - * including without limitation the rights to use, copy, modify, merge, - * publish, distribute, sublicense, and/or sell copies of the Software, - * and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include - -#include -#include - -struct Db -{ - char *dir; - pthread_mutex_t lock; - - size_t cacheSize; - size_t maxCache; - HashMap *cache; - - /* - * The cache uses a double linked list (see DbRef - * below) to know which objects are most and least - * recently used. The following diagram helps me - * know what way all the pointers go, because it - * can get very confusing sometimes. For example, - * there's nothing stopping "next" from pointing to - * least recent, and "prev" from pointing to most - * recent, so hopefully this clarifies the pointer - * terminology used when dealing with the linked - * list: - * - * mostRecent leastRecent - * | prev prev | prev - * +---+ ---> +---+ ---> +---+ ---> NULL - * |ref| |ref| |ref| - * NULL <--- +---+ <--- +---+ <--- +---+ - * next next next - */ - DbRef *mostRecent; - DbRef *leastRecent; -}; - -struct DbRef -{ - HashMap *json; - - uint64_t ts; - size_t size; - - Array *name; - - DbRef *prev; - DbRef *next; - - int fd; - Stream *stream; -}; - -static void -StringArrayFree(Array * arr) -{ - size_t i; - - for (i = 0; i < ArraySize(arr); i++) - { - Free(ArrayGet(arr, i)); - } - - ArrayFree(arr); -} - -static ssize_t DbComputeSize(HashMap *); - -static ssize_t -DbComputeSizeOfValue(JsonValue * val) -{ - MemoryInfo *a; - ssize_t total = 0; - - size_t i; - - union - { - char *str; - Array *arr; - } u; - - if (!val) - { - return -1; - } - - a = MemoryInfoGet(val); - if (a) - { - total += MemoryInfoGetSize(a); - } - - switch (JsonValueType(val)) - { - case JSON_OBJECT: - total += DbComputeSize(JsonValueAsObject(val)); - break; - case JSON_ARRAY: - u.arr = JsonValueAsArray(val); - a = MemoryInfoGet(u.arr); - - if (a) - { - total += MemoryInfoGetSize(a); - } - - for (i = 0; i < ArraySize(u.arr); i++) - { - total += DbComputeSizeOfValue(ArrayGet(u.arr, i)); - } - break; - case JSON_STRING: - u.str = JsonValueAsString(val); - a = MemoryInfoGet(u.str); - if (a) - { - total += MemoryInfoGetSize(a); - } - break; - case JSON_NULL: - case JSON_INTEGER: - case JSON_FLOAT: - case JSON_BOOLEAN: - default: - /* These don't use any extra heap space */ - break; - } - return total; -} - -static ssize_t -DbComputeSize(HashMap * json) -{ - char *key; - JsonValue *val; - MemoryInfo *a; - size_t total; - - if (!json) - { - return -1; - } - - total = 0; - - a = MemoryInfoGet(json); - if (a) - { - total += MemoryInfoGetSize(a); - } - - while (HashMapIterate(json, &key, (void **) &val)) - { - a = MemoryInfoGet(key); - if (a) - { - total += MemoryInfoGetSize(a); - } - - total += DbComputeSizeOfValue(val); - } - - return total; -} - -static char * -DbHashKey(Array * args) -{ - size_t i; - char *str = NULL; - - for (i = 0; i < ArraySize(args); i++) - { - char *tmp = StrConcat(2, str, ArrayGet(args, i)); - - Free(str); - str = tmp; - } - - return str; -} - -static char -DbSanitiseChar(char input) -{ - switch (input) - { - case '/': - return '_'; - case '.': - return '-'; - } - return input; -} - -static char * -DbDirName(Db * db, Array * args, size_t strip) -{ - size_t i, j; - char *str = StrConcat(2, db->dir, "/"); - - for (i = 0; i < ArraySize(args) - strip; i++) - { - char *tmp; - char *sanitise = StrDuplicate(ArrayGet(args, i)); - for (j = 0; j < strlen(sanitise); j++) - { - sanitise[j] = DbSanitiseChar(sanitise[j]); - } - - tmp = StrConcat(3, str, sanitise, "/"); - - Free(str); - Free(sanitise); - - str = tmp; - } - - return str; -} - -static char * -DbFileName(Db * db, Array * args) -{ - size_t i; - char *str = StrConcat(2, db->dir, "/"); - - for (i = 0; i < ArraySize(args); i++) - { - char *tmp; - char *arg = StrDuplicate(ArrayGet(args, i)); - size_t j = 0; - - /* Sanitize name to prevent directory traversal attacks */ - while (arg[j]) - { - arg[j] = DbSanitiseChar(arg[j]); - j++; - } - - tmp = StrConcat(3, str, arg, - (i < ArraySize(args) - 1) ? "/" : ".json"); - - Free(arg); - Free(str); - - str = tmp; - } - - return str; -} - -static void -DbCacheEvict(Db * db) -{ - DbRef *ref = db->leastRecent; - DbRef *tmp; - - while (ref && db->cacheSize > db->maxCache) - { - char *hash; - - JsonFree(ref->json); - - hash = DbHashKey(ref->name); - HashMapDelete(db->cache, hash); - Free(hash); - - StringArrayFree(ref->name); - - db->cacheSize -= ref->size; - - if (ref->next) - { - ref->next->prev = ref->prev; - } - else - { - db->mostRecent = ref->prev; - } - - if (ref->prev) - { - ref->prev->next = ref->next; - } - else - { - db->leastRecent = ref->next; - } - - tmp = ref->next; - Free(ref); - - ref = tmp; - } -} - -Db * -DbOpen(char *dir, size_t cache) -{ - Db *db; - pthread_mutexattr_t attr; - - if (!dir) - { - return NULL; - } - - db = Malloc(sizeof(Db)); - if (!db) - { - return NULL; - } - - db->dir = dir; - db->maxCache = cache; - - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutex_init(&db->lock, &attr); - pthread_mutexattr_destroy(&attr); - - db->mostRecent = NULL; - db->leastRecent = NULL; - db->cacheSize = 0; - - if (db->maxCache) - { - db->cache = HashMapCreate(); - if (!db->cache) - { - return NULL; - } - } - else - { - db->cache = NULL; - } - - return db; -} - -void -DbMaxCacheSet(Db * db, size_t cache) -{ - if (!db) - { - return; - } - - pthread_mutex_lock(&db->lock); - - db->maxCache = cache; - if (db->maxCache && !db->cache) - { - db->cache = HashMapCreate(); - db->cacheSize = 0; - } - - DbCacheEvict(db); - - pthread_mutex_unlock(&db->lock); -} - -void -DbClose(Db * db) -{ - if (!db) - { - return; - } - - pthread_mutex_lock(&db->lock); - - DbMaxCacheSet(db, 0); - DbCacheEvict(db); - HashMapFree(db->cache); - - pthread_mutex_unlock(&db->lock); - pthread_mutex_destroy(&db->lock); - - Free(db); -} - -static DbRef * -DbLockFromArr(Db * db, Array * args) -{ - char *file; - char *hash; - DbRef *ref; - struct flock lock; - - int fd; - Stream *stream; - - if (!db || !args) - { - return NULL; - } - - ref = NULL; - hash = NULL; - - pthread_mutex_lock(&db->lock); - - /* Check if the item is in the cache */ - hash = DbHashKey(args); - ref = HashMapGet(db->cache, hash); - file = DbFileName(db, args); - - fd = open(file, O_RDWR); - if (fd == -1) - { - if (ref) - { - HashMapDelete(db->cache, hash); - JsonFree(ref->json); - StringArrayFree(ref->name); - - db->cacheSize -= ref->size; - - if (ref->next) - { - ref->next->prev = ref->prev; - } - else - { - db->mostRecent = ref->prev; - } - - if (ref->prev) - { - ref->prev->next = ref->next; - } - else - { - db->leastRecent = ref->next; - } - - if (!db->leastRecent) - { - db->leastRecent = db->mostRecent; - } - - Free(ref); - } - ref = NULL; - goto finish; - } - - stream = StreamFd(fd); - - lock.l_start = 0; - lock.l_len = 0; - lock.l_type = F_WRLCK; - lock.l_whence = SEEK_SET; - - /* Lock the file on the disk */ - if (fcntl(fd, F_SETLK, &lock) < 0) - { - StreamClose(stream); - ref = NULL; - goto finish; - } - - if (ref) /* In cache */ - { - uint64_t diskTs = UtilLastModified(file); - - ref->fd = fd; - ref->stream = stream; - - if (diskTs > ref->ts) - { - /* File was modified on disk since it was cached */ - HashMap *json = JsonDecode(ref->stream); - - if (!json) - { - StreamClose(ref->stream); - ref = NULL; - goto finish; - } - - JsonFree(ref->json); - ref->json = json; - ref->ts = diskTs; - ref->size = DbComputeSize(ref->json); - } - - /* Float this ref to mostRecent */ - if (ref->next) - { - ref->next->prev = ref->prev; - - if (!ref->prev) - { - db->leastRecent = ref->next; - } - else - { - ref->prev->next = ref->next; - } - - ref->prev = db->mostRecent; - ref->next = NULL; - if (db->mostRecent) - { - db->mostRecent->next = ref; - } - db->mostRecent = ref; - } - - /* If there is no least recent, this is the only thing in the - * cache, so it is also least recent. */ - if (!db->leastRecent) - { - db->leastRecent = ref; - } - - /* The file on disk may be larger than what we have in memory, - * which may require items in cache to be evicted. */ - DbCacheEvict(db); - } - else - { - Array *name; - size_t i; - - /* Not in cache; load from disk */ - - ref = Malloc(sizeof(DbRef)); - if (!ref) - { - StreamClose(stream); - goto finish; - } - - ref->json = JsonDecode(stream); - - if (!ref->json) - { - Free(ref); - StreamClose(stream); - ref = NULL; - goto finish; - } - - ref->fd = fd; - ref->stream = stream; - - name = ArrayCreate(); - for (i = 0; i < ArraySize(args); i++) - { - ArrayAdd(name, StrDuplicate(ArrayGet(args, i))); - } - ref->name = name; - - if (db->cache) - { - ref->ts = UtilTsMillis(); - ref->size = DbComputeSize(ref->json); - HashMapSet(db->cache, hash, ref); - db->cacheSize += ref->size; - - ref->next = NULL; - ref->prev = db->mostRecent; - if (db->mostRecent) - { - db->mostRecent->next = ref; - } - db->mostRecent = ref; - - if (!db->leastRecent) - { - db->leastRecent = ref; - } - - /* Adding this item to the cache may case it to grow too - * large, requiring some items to be evicted */ - DbCacheEvict(db); - } - } - -finish: - if (!ref) - { - pthread_mutex_unlock(&db->lock); - } - - Free(file); - Free(hash); - - return ref; -} - -DbRef * -DbCreate(Db * db, size_t nArgs,...) -{ - Stream *fp; - char *file; - char *dir; - va_list ap; - Array *args; - DbRef *ret; - - if (!db) - { - return NULL; - } - - va_start(ap, nArgs); - args = ArrayFromVarArgs(nArgs, ap); - va_end(ap); - - if (!args) - { - return NULL; - } - - pthread_mutex_lock(&db->lock); - - file = DbFileName(db, args); - - if (UtilLastModified(file)) - { - Free(file); - ArrayFree(args); - pthread_mutex_unlock(&db->lock); - return NULL; - } - - dir = DbDirName(db, args, 1); - if (UtilMkdir(dir, 0750) < 0) - { - Free(file); - ArrayFree(args); - Free(dir); - pthread_mutex_unlock(&db->lock); - return NULL; - } - - Free(dir); - - fp = StreamOpen(file, "w"); - Free(file); - if (!fp) - { - ArrayFree(args); - pthread_mutex_unlock(&db->lock); - return NULL; - } - - StreamPuts(fp, "{}"); - StreamClose(fp); - - /* DbLockFromArr() will lock again for us */ - pthread_mutex_unlock(&db->lock); - - ret = DbLockFromArr(db, args); - - ArrayFree(args); - - return ret; -} - -bool -DbDelete(Db * db, size_t nArgs,...) -{ - va_list ap; - Array *args; - char *file; - char *hash; - bool ret = true; - DbRef *ref; - - if (!db) - { - return false; - } - - va_start(ap, nArgs); - args = ArrayFromVarArgs(nArgs, ap); - va_end(ap); - - pthread_mutex_lock(&db->lock); - - hash = DbHashKey(args); - file = DbFileName(db, args); - - ref = HashMapGet(db->cache, hash); - if (ref) - { - HashMapDelete(db->cache, hash); - JsonFree(ref->json); - StringArrayFree(ref->name); - - db->cacheSize -= ref->size; - - if (ref->next) - { - ref->next->prev = ref->prev; - } - else - { - db->mostRecent = ref->prev; - } - - if (ref->prev) - { - ref->prev->next = ref->next; - } - else - { - db->leastRecent = ref->next; - } - - if (!db->leastRecent) - { - db->leastRecent = db->mostRecent; - } - - Free(ref); - } - - Free(hash); - - if (UtilLastModified(file)) - { - ret = (remove(file) == 0); - } - - pthread_mutex_unlock(&db->lock); - - ArrayFree(args); - Free(file); - return ret; -} - -DbRef * -DbLock(Db * db, size_t nArgs,...) -{ - va_list ap; - Array *args; - DbRef *ret; - - va_start(ap, nArgs); - args = ArrayFromVarArgs(nArgs, ap); - va_end(ap); - - if (!args) - { - return NULL; - } - - ret = DbLockFromArr(db, args); - - ArrayFree(args); - - return ret; -} - -bool -DbUnlock(Db * db, DbRef * ref) -{ - bool destroy; - - if (!db || !ref) - { - return false; - } - - lseek(ref->fd, 0L, SEEK_SET); - if (ftruncate(ref->fd, 0) < 0) - { - pthread_mutex_unlock(&db->lock); - Log(LOG_ERR, "Failed to truncate file on disk."); - Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno)); - return false; - } - - JsonEncode(ref->json, ref->stream, JSON_DEFAULT); - StreamClose(ref->stream); - - if (db->cache) - { - char *key = DbHashKey(ref->name); - - if (HashMapGet(db->cache, key)) - { - db->cacheSize -= ref->size; - ref->size = DbComputeSize(ref->json); - db->cacheSize += ref->size; - - /* If this ref has grown significantly since we last - * computed its size, it may have filled the cache and - * require some items to be evicted. */ - DbCacheEvict(db); - - destroy = false; - } - else - { - destroy = true; - } - - Free(key); - } - else - { - destroy = true; - } - - if (destroy) - { - JsonFree(ref->json); - StringArrayFree(ref->name); - - Free(ref); - } - - pthread_mutex_unlock(&db->lock); - return true; -} - -bool -DbExists(Db * db, size_t nArgs,...) -{ - va_list ap; - Array *args; - char *file; - bool ret; - - va_start(ap, nArgs); - args = ArrayFromVarArgs(nArgs, ap); - va_end(ap); - - if (!args) - { - return false; - } - - pthread_mutex_lock(&db->lock); - - file = DbFileName(db, args); - ret = (UtilLastModified(file) != 0); - - pthread_mutex_unlock(&db->lock); - - Free(file); - ArrayFree(args); - - return ret; -} - -Array * -DbList(Db * db, size_t nArgs,...) -{ - Array *result; - Array *path; - DIR *files; - struct dirent *file; - char *dir; - va_list ap; - - if (!db || !nArgs) - { - return NULL; - } - - result = ArrayCreate(); - if (!result) - { - return NULL; - } - - va_start(ap, nArgs); - path = ArrayFromVarArgs(nArgs, ap); - dir = DbDirName(db, path, 0); - - pthread_mutex_lock(&db->lock); - - files = opendir(dir); - if (!files) - { - ArrayFree(path); - ArrayFree(result); - Free(dir); - pthread_mutex_unlock(&db->lock); - return NULL; - } - while ((file = readdir(files))) - { - size_t namlen = strlen(file->d_name); - - if (namlen > 5) - { - int nameOffset = namlen - 5; - - if (StrEquals(file->d_name + nameOffset, ".json")) - { - file->d_name[nameOffset] = '\0'; - ArrayAdd(result, StrDuplicate(file->d_name)); - } - } - } - closedir(files); - - ArrayFree(path); - Free(dir); - pthread_mutex_unlock(&db->lock); - - return result; -} - -void -DbListFree(Array * arr) -{ - StringArrayFree(arr); -} - -HashMap * -DbJson(DbRef * ref) -{ - return ref ? ref->json : NULL; -} - -bool -DbJsonSet(DbRef * ref, HashMap * json) -{ - if (!ref || !json) - { - return false; - } - - JsonFree(ref->json); - ref->json = JsonDuplicate(json); - return true; -} diff --git a/src/Db/Db.c b/src/Db/Db.c new file mode 100644 index 0000000..1015bbf --- /dev/null +++ b/src/Db/Db.c @@ -0,0 +1,499 @@ +/* + * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net> + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include + +#include "Db/Internal.h" + +void +StringArrayFree(Array * arr) +{ + size_t i; + + for (i = 0; i < ArraySize(arr); i++) + { + Free(ArrayGet(arr, i)); + } + + ArrayFree(arr); +} + +static ssize_t DbComputeSize(HashMap *); + +static ssize_t +DbComputeSizeOfValue(JsonValue * val) +{ + MemoryInfo *a; + ssize_t total = 0; + + size_t i; + + union + { + char *str; + Array *arr; + } u; + + if (!val) + { + return -1; + } + + a = MemoryInfoGet(val); + if (a) + { + total += MemoryInfoGetSize(a); + } + + switch (JsonValueType(val)) + { + case JSON_OBJECT: + total += DbComputeSize(JsonValueAsObject(val)); + break; + case JSON_ARRAY: + u.arr = JsonValueAsArray(val); + a = MemoryInfoGet(u.arr); + + if (a) + { + total += MemoryInfoGetSize(a); + } + + for (i = 0; i < ArraySize(u.arr); i++) + { + total += DbComputeSizeOfValue(ArrayGet(u.arr, i)); + } + break; + case JSON_STRING: + u.str = JsonValueAsString(val); + a = MemoryInfoGet(u.str); + if (a) + { + total += MemoryInfoGetSize(a); + } + break; + case JSON_NULL: + case JSON_INTEGER: + case JSON_FLOAT: + case JSON_BOOLEAN: + default: + /* These don't use any extra heap space */ + break; + } + return total; +} + +static ssize_t +DbComputeSize(HashMap * json) +{ + char *key; + JsonValue *val; + MemoryInfo *a; + size_t total; + + if (!json) + { + return -1; + } + + total = 0; + + a = MemoryInfoGet(json); + if (a) + { + total += MemoryInfoGetSize(a); + } + + while (HashMapIterate(json, &key, (void **) &val)) + { + a = MemoryInfoGet(key); + if (a) + { + total += MemoryInfoGetSize(a); + } + + total += DbComputeSizeOfValue(val); + } + + return total; +} + +static char * +DbHashKey(Array * args) +{ + size_t i; + char *str = NULL; + + for (i = 0; i < ArraySize(args); i++) + { + char *tmp = StrConcat(2, str, ArrayGet(args, i)); + + Free(str); + str = tmp; + } + + return str; +} + +static void +DbCacheEvict(Db * db) +{ + DbRef *ref = db->leastRecent; + DbRef *tmp; + + while (ref && db->cacheSize > db->maxCache) + { + char *hash; + + JsonFree(ref->json); + + hash = DbHashKey(ref->name); + HashMapDelete(db->cache, hash); + Free(hash); + + StringArrayFree(ref->name); + + db->cacheSize -= ref->size; + + if (ref->next) + { + ref->next->prev = ref->prev; + } + else + { + db->mostRecent = ref->prev; + } + + if (ref->prev) + { + ref->prev->next = ref->next; + } + else + { + db->leastRecent = ref->next; + } + + tmp = ref->next; + Free(ref); + + ref = tmp; + } +} + +void +DbMaxCacheSet(Db * db, size_t cache) +{ + if (!db) + { + return; + } + + pthread_mutex_lock(&db->lock); + + db->maxCache = cache; + if (db->maxCache && !db->cache) + { + db->cache = HashMapCreate(); + db->cacheSize = 0; + } + + DbCacheEvict(db); + + pthread_mutex_unlock(&db->lock); +} + +void +DbClose(Db * db) +{ + if (!db) + { + return; + } + + pthread_mutex_lock(&db->lock); + if (db->close) + { + db->close(db); + } + DbMaxCacheSet(db, 0); + DbCacheEvict(db); + HashMapFree(db->cache); + + pthread_mutex_unlock(&db->lock); + pthread_mutex_destroy(&db->lock); + + Free(db); +} + +DbRef * +DbCreate(Db * db, size_t nArgs,...) +{ + va_list ap; + Array *args; + DbRef *ret; + + if (!db || !db->create) + { + return NULL; + } + + va_start(ap, nArgs); + args = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + if (!args) + { + return NULL; + } + + ret = db->create(db, args); + ArrayFree(args); + + return ret; +} + +bool +DbDelete(Db * db, size_t nArgs,...) +{ + va_list ap; + Array *args; + bool ret = true; + + if (!db) + { + return false; + } + + va_start(ap, nArgs); + args = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + ret = db->delete(db, args); + + ArrayFree(args); + return ret; +} + +DbRef * +DbLock(Db * db, size_t nArgs,...) +{ + va_list ap; + Array *args; + DbRef *ret; + + va_start(ap, nArgs); + args = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + if (!args || !db->lockFunc) + { + return NULL; + } + + ret = db->lockFunc(db, DB_HINT_WRITE, args); + + ArrayFree(args); + + return ret; +} + +DbRef * +DbLockIntent(Db * db, DbHint hint, size_t nArgs,...) +{ + va_list ap; + Array *args; + DbRef *ret; + + va_start(ap, nArgs); + args = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + if (!args || !db->lockFunc) + { + return NULL; + } + + ret = db->lockFunc(db, hint, args); + + ArrayFree(args); + + return ret; +} + +bool +DbUnlock(Db * db, DbRef * ref) +{ + if (!db || !ref || !db->unlock) + { + return false; + } + + return db->unlock(db, ref); +} + +bool +DbExists(Db * db, size_t nArgs,...) +{ + va_list ap; + Array *args; + bool ret; + if (!db || !nArgs || !db->exists) + { + return false; + } + + va_start(ap, nArgs); + args = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + if (!args) + { + return false; + } + + ret = db->exists(db, args); + ArrayFree(args); + + return ret; +} + +Array * +DbList(Db * db, size_t nArgs,...) +{ + Array *result; + Array *path; + va_list ap; + + if (!db || !nArgs || !db->list) + { + return NULL; + } + + va_start(ap, nArgs); + path = ArrayFromVarArgs(nArgs, ap); + va_end(ap); + + result = db->list(db, path); + + ArrayFree(path); + return result; +} + +void +DbListFree(Array * arr) +{ + StringArrayFree(arr); +} + +HashMap * +DbJson(DbRef * ref) +{ + return ref ? ref->json : NULL; +} + +bool +DbJsonSet(DbRef * ref, HashMap * json) +{ + if (!ref || !json) + { + return false; + } + + JsonFree(ref->json); + ref->json = JsonDuplicate(json); + return true; +} +void +DbInit(Db *db) +{ + pthread_mutexattr_t attr; + if (!db) + { + return; + } + + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&db->lock, &attr); + pthread_mutexattr_destroy(&attr); + + db->mostRecent = NULL; + db->leastRecent = NULL; + db->cacheSize = 0; + db->maxCache = 0; + + if (db->maxCache) + { + db->cache = HashMapCreate(); + } + else + { + db->cache = NULL; + } +} +void +DbRefInit(Db *db, DbRef *ref) +{ + if (!db || !ref) + { + return; + } + ref->prev = db->mostRecent; + ref->next = NULL; + ref->json = NULL; + ref->name = NULL; + + /* As default values to be overwritten by impls */ + ref->ts = UtilTsMillis(); + ref->size = 0; + + /* TODO: Append the ref to the cache list. + * I removed it because it broke everything and crashed all the time. + * My bad! */ +} +void +StringArrayAppend(Array *arr, char *str) +{ + if (!arr || !str) + { + return; + } + ArrayAdd(arr, StrDuplicate(str)); +} diff --git a/src/Db/Flat.c b/src/Db/Flat.c new file mode 100644 index 0000000..df5685c --- /dev/null +++ b/src/Db/Flat.c @@ -0,0 +1,374 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include + +#include "Db/Internal.h" + +typedef struct FlatDb { + Db base; + char *dir; + /* Theres not much to do here. */ +} FlatDb; +typedef struct FlatDbRef { + DbRef base; + + Stream *stream; + int fd; +} FlatDbRef; + +static char +DbSanitiseChar(char input) +{ + switch (input) + { + case '/': + return '_'; + case '.': + return '-'; + } + return input; +} + +static char * +DbDirName(FlatDb * db, Array * args, size_t strip) +{ + size_t i, j; + char *str = StrConcat(2, db->dir, "/"); + + for (i = 0; i < ArraySize(args) - strip; i++) + { + char *tmp; + char *sanitise = StrDuplicate(ArrayGet(args, i)); + for (j = 0; j < strlen(sanitise); j++) + { + sanitise[j] = DbSanitiseChar(sanitise[j]); + } + + tmp = StrConcat(3, str, sanitise, "/"); + + Free(str); + Free(sanitise); + + str = tmp; + } + + return str; +} +static char * +DbFileName(FlatDb * db, Array * args) +{ + size_t i; + char *str = StrConcat(2, db->dir, "/"); + + for (i = 0; i < ArraySize(args); i++) + { + char *tmp; + char *arg = StrDuplicate(ArrayGet(args, i)); + size_t j = 0; + + /* Sanitize name to prevent directory traversal attacks */ + while (arg[j]) + { + arg[j] = DbSanitiseChar(arg[j]); + j++; + } + + tmp = StrConcat(3, str, arg, + (i < ArraySize(args) - 1) ? "/" : ".json"); + + Free(arg); + Free(str); + + str = tmp; + } + + return str; +} + +static DbRef * +FlatLock(Db *d, DbHint hint, Array *dir) +{ + FlatDb *db = (FlatDb *) d; + FlatDbRef *ref = NULL; + size_t i; + char *path = NULL; + if (!d || !dir) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + path = DbFileName(db, dir); + /* TODO: Caching */ + { + int fd = open(path, O_RDWR); + Stream *stream; + struct flock lock; + if (fd == -1) + { + ref = NULL; + goto end; + } + + stream = StreamFd(fd); + if (!stream) + { + ref = NULL; + goto end; + } + + lock.l_start = 0; + lock.l_len = 0; + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + + if (fcntl(fd, F_SETLK, &lock) < 0) + { + StreamClose(stream); + ref = NULL; + goto end; + } + + ref = Malloc(sizeof(*ref)); + DbRefInit(d, (DbRef *) ref); + /* TODO: Hints */ + ref->base.hint = hint; + ref->base.ts = UtilLastModified(path); + ref->base.json = JsonDecode(stream); + ref->stream = stream; + ref->fd = fd; + if (!ref->base.json) + { + Free(ref); + StreamClose(stream); + ref = NULL; + goto end; + } + + + ref->base.name = ArrayCreate(); + for (i = 0; i < ArraySize(dir); i++) + { + StringArrayAppend(ref->base.name, ArrayGet(dir, i)); + } + } +end: + Free(path); + if (!ref) + { + pthread_mutex_unlock(&d->lock); + } + return (DbRef *) ref; +} + +static bool +FlatUnlock(Db *d, DbRef *r) +{ + FlatDbRef *ref = (FlatDbRef *) r; + + if (!d || !r) + { + return false; + } + + lseek(ref->fd, 0L, SEEK_SET); + if (ftruncate(ref->fd, 0) < 0) + { + pthread_mutex_unlock(&d->lock); + Log(LOG_ERR, "Failed to truncate file on disk."); + Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno)); + return false; + } + + JsonEncode(ref->base.json, ref->stream, JSON_DEFAULT); + StreamClose(ref->stream); + + JsonFree(ref->base.json); + StringArrayFree(ref->base.name); + Free(ref); + + pthread_mutex_unlock(&d->lock); + return true; +} +static DbRef * +FlatCreate(Db *d, Array *dir) +{ + FlatDb *db = (FlatDb *) d; + char *path, *dirPath; + Stream *fp; + DbRef *ret; + + if (!d || !dir) + { + return NULL; + } + pthread_mutex_lock(&d->lock); + + path = DbFileName(db, dir); + if (UtilLastModified(path)) + { + Free(path); + pthread_mutex_unlock(&d->lock); + return NULL; + } + + dirPath = DbDirName(db, dir, 1); + if (UtilMkdir(dirPath, 0750) < 0) + { + Free(path); + Free(dirPath); + pthread_mutex_unlock(&d->lock); + return NULL; + } + Free(dirPath); + + fp = StreamOpen(path, "w"); + Free(path); + if (!fp) + { + pthread_mutex_unlock(&d->lock); + return NULL; + } + + StreamPuts(fp, "{}"); + StreamClose(fp); + + /* FlatLock() will lock again for us */ + pthread_mutex_unlock(&d->lock); + ret = FlatLock(d, DB_HINT_WRITE, dir); + + return ret; +} + +static bool +FlatDelete(Db *d, Array *dir) +{ + bool ret = false; + char *file; + FlatDb *db = (FlatDb *) d; + if (!d || !dir) + { + return false; + } + + pthread_mutex_lock(&d->lock); + file = DbFileName(db, dir); + + /* TODO: Unlink the entry from the linkedlist */ + if (UtilLastModified(file)) + { + ret = remove(file) == 0; + } + + Free(file); + pthread_mutex_unlock(&d->lock); + return ret; +} + +static Array * +FlatList(Db *d, Array *dir) +{ + FlatDb *db = (FlatDb *) d; + struct dirent *file; + Array *ret; + DIR *files; + char *path; + + if (!d || !dir) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + + path = DbDirName(db, dir, 0); + files = opendir(path); + if (!files) + { + Free(path); + pthread_mutex_unlock(&d->lock); + return NULL; + } + + ret = ArrayCreate(); + while ((file = readdir(files))) + { + size_t namlen = strlen(file->d_name); + + if (namlen > 5) + { + int nameOffset = namlen - 5; + + if (StrEquals(file->d_name + nameOffset, ".json")) + { + file->d_name[nameOffset] = '\0'; + StringArrayAppend(ret, file->d_name); + } + } + } + closedir(files); + Free(path); + + pthread_mutex_unlock(&d->lock); + return ret; +} +static bool +FlatExists(Db *d, Array *dir) +{ + FlatDb *db = (FlatDb *) d; + char *path; + bool ret; + if (!d || !dir) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + + path = DbFileName(db, dir); + ret = UtilLastModified(path) != 0; + Free(path); + + pthread_mutex_unlock(&d->lock); + + return ret; +} + +Db * +DbOpen(char *dir, size_t cache) +{ + FlatDb *db; + if (!dir) + { + return NULL; + } + db = Malloc(sizeof(*db)); + DbInit((Db *) db); + db->dir = dir; + db->base.cacheSize = cache; + + db->base.lockFunc = FlatLock; + db->base.unlock = FlatUnlock; + db->base.create = FlatCreate; + db->base.delete = FlatDelete; + db->base.exists = FlatExists; + db->base.list = FlatList; + db->base.close = NULL; + + return (Db *) db; +} diff --git a/src/Db/Internal.h b/src/Db/Internal.h new file mode 100644 index 0000000..1f1014a --- /dev/null +++ b/src/Db/Internal.h @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net> + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef CYTOPLASM_DB_INTERNAL_H +#define CYTOPLASM_DB_INTERNAL_H + +#include +#include + +#include + +/* TODO: Define the base structures to define a database internally. + * All "implementations" shall have them as a first element, so that + * basic, general functions can work on them properly. */ + +/* The base structure of a database */ +struct Db +{ + pthread_mutex_t lock; + + size_t cacheSize; + size_t maxCache; + HashMap *cache; + + /* + * The cache uses a double linked list (see DbRef + * below) to know which objects are most and least + * recently used. The following diagram helps me + * know what way all the pointers go, because it + * can get very confusing sometimes. For example, + * there's nothing stopping "next" from pointing to + * least recent, and "prev" from pointing to most + * recent, so hopefully this clarifies the pointer + * terminology used when dealing with the linked + * list: + * + * mostRecent leastRecent + * | prev prev | prev + * +---+ ---> +---+ ---> +---+ ---> NULL + * |ref| |ref| |ref| + * NULL <--- +---+ <--- +---+ <--- +---+ + * next next next + */ + DbRef *mostRecent; + DbRef *leastRecent; + + /* Functions for implementation-specific operations + * (opening a ref, closing a db, removing an entry, ...) */ + DbRef * (*lockFunc)(Db *, DbHint, Array *); + DbRef * (*create)(Db *, Array *); + Array * (*list)(Db *, Array *); + bool (*unlock)(Db *, DbRef *); + bool (*delete)(Db *, Array *); + bool (*exists)(Db *, Array *); + void (*close)(Db *); + + /* Implementation-specific constructs */ +}; + +struct DbRef +{ + HashMap *json; + + uint64_t ts; + size_t size; + + Array *name; + + DbRef *prev; + DbRef *next; + + DbHint hint; + /* Implementation-specific constructs */ +}; + +extern void DbInit(Db *); +extern void DbRefInit(Db *, DbRef *); +extern void StringArrayFree(Array *); +extern void StringArrayAppend(Array *, char *); + +#endif diff --git a/src/Db/LMDB.c b/src/Db/LMDB.c new file mode 100644 index 0000000..044c439 --- /dev/null +++ b/src/Db/LMDB.c @@ -0,0 +1,590 @@ +#include +#include +#include +#include + +#include "Db/Internal.h" + +#include +#include +#include + +#ifdef EDB_LMDB + +#include + +typedef struct LMDB { + Db base; /* The base implementation required to pass */ + + MDB_env *environ; + MDB_dbi dbi; +} LMDB; +typedef struct LMDBRef { + DbRef base; + + /* TODO: LMDB-dependent stuff */ + MDB_txn *transaction; + MDB_dbi dbi; +} LMDBRef; + +/* Helper functions */ +static MDB_val +LMDBTranslateKey(Array *key) +{ + MDB_val ret = { 0 }; + char *data = NULL; + size_t length = 0; + size_t i; + if (!key || ArraySize(key) > 255) + { + return ret; + } + + data = Realloc(data, ++length); + data[0] = ArraySize(key); + + /* Now, let's push every item */ + for (i = 0; i < ArraySize(key); i++) + { + char *entry = ArrayGet(key, i); + size_t offset = length; + + data = Realloc(data, (length += strlen(entry) + 1)); + memcpy(data + offset, entry, strlen(entry)); + data[length - 1] = '\0'; + } + + /* We now have every key */ + ret.mv_size = length; + ret.mv_data = data; + return ret; +} +static void +LMDBKillKey(MDB_val key) +{ + if (!key.mv_data || !key.mv_size) + { + return; + } + + Free(key.mv_data); +} +static HashMap * +LMDBDecode(MDB_val val) +{ + FILE *fakefile; + Stream *fakestream; + HashMap *ret; + if (!val.mv_data || !val.mv_size) + { + return NULL; + } + + fakefile = fmemopen(val.mv_data, val.mv_size, "r"); + fakestream = StreamFile(fakefile); + ret = JsonDecode(fakestream); + StreamClose(fakestream); + + return ret; +} +static bool +LMDBKeyStartsWith(MDB_val key, MDB_val starts) +{ + size_t i; + if (!key.mv_size || !starts.mv_size) + { + return false; + } + if (key.mv_size < starts.mv_size) + { + return false; + } + + for (i = 0; i < starts.mv_size; i++) + { + char keyC = ((char *) key.mv_data)[i]; + char startC = ((char *) starts.mv_data)[i]; + + if (keyC != startC) + { + return false; + } + } + return true; +} +static char * +LMDBKeyHead(MDB_val key) +{ + char *end; + if (!key.mv_size || !key.mv_data) + { + return NULL; + } + /* -2 because we have a NULL byte in there */ + end = ((char *) key.mv_data) + key.mv_size - 1; + if ((void *) end < key.mv_data) + { + /* Uh oh. */ + return NULL; + } + + while ((void *) (end - 1) >= key.mv_data && *(end - 1)) + { + end--; + } + return end; +} + +static DbRef * +LMDBLock(Db *d, DbHint hint, Array *k) +{ + LMDB *db = (LMDB *) d; + MDB_txn *transaction; + LMDBRef *ret = NULL; + MDB_val key, json_val; + int code, flags; + if (!d || !k) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + key = LMDBTranslateKey(k); + + /* Create a transaction, honoring hints. */ + /* TODO: Do we want to create a "main" transaction that everyone inherits + * from? */ + flags = hint == DB_HINT_READONLY ? MDB_RDONLY : 0; + if ((code = mdb_txn_begin(db->environ, NULL, flags, &transaction)) != 0) + { + /* Very bad! */ + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + goto end; + } + + json_val.mv_size = 0; + json_val.mv_data = NULL; + code = mdb_get(transaction, db->dbi, &key, &json_val); + if (code == MDB_NOTFOUND) + { + mdb_txn_abort(transaction); + goto end; + } + else if (code != 0) + { + Log(LOG_ERR, + "%s: mdb_get failure: %s", + __func__, mdb_strerror(code) + ); + mdb_txn_abort(transaction); + goto end; + } + + ret = Malloc(sizeof(*ret)); + DbRefInit(d, (DbRef *) ret); + /* TODO: Timestamp */ + { + size_t i; + ret->base.name = ArrayCreate(); + for (i = 0; i < ArraySize(k); i++) + { + char *ent = ArrayGet(k, i); + StringArrayAppend(ret->base.name, ent); + } + } + ret->base.json = LMDBDecode(json_val); + ret->base.hint = hint; + ret->transaction = NULL; + + if (hint == DB_HINT_WRITE) + { + ret->transaction = transaction; + } + else + { + mdb_txn_abort(transaction); + } +end: + if (!ret || hint == DB_HINT_READONLY) + { + pthread_mutex_unlock(&d->lock); + } + LMDBKillKey(key); + return (DbRef *) ret; +} +static bool +LMDBExists(Db *d, Array *k) +{ + MDB_val key, empty; + LMDB *db = (LMDB *) d; + MDB_txn *transaction; + int code; + bool ret = false; + if (!d || !k) + { + return false; + } + + pthread_mutex_lock(&d->lock); + key = LMDBTranslateKey(k); + + /* create a txn */ + if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0) + { + /* Very bad! */ + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + goto end; + } + + ret = mdb_get(transaction, db->dbi, &key, &empty) == 0; + mdb_txn_abort(transaction); +end: + LMDBKillKey(key); + pthread_mutex_unlock(&d->lock); + return ret; +} +static bool +LMDBDelete(Db *d, Array *k) +{ + MDB_val key, empty; + LMDB *db = (LMDB *) d; + MDB_txn *transaction; + int code; + bool ret = false; + if (!d || !k) + { + return false; + } + + pthread_mutex_lock(&d->lock); + key = LMDBTranslateKey(k); + + /* create a txn */ + if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0) + { + /* Very bad! */ + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + goto end; + } + + ret = mdb_del(transaction, db->dbi, &key, &empty) == 0; + mdb_txn_commit(transaction); +end: + LMDBKillKey(key); + pthread_mutex_unlock(&d->lock); + return ret; +} + +static bool +LMDBUnlock(Db *d, DbRef *r) +{ + LMDBRef *ref = (LMDBRef *) r; + LMDB *db = (LMDB *) d; + FILE *fakestream; + Stream *stream; + MDB_val key, val; + bool ret = true; + + if (!d || !r) + { + return false; + } + + val.mv_data = NULL; + val.mv_size = 0; + if (ref->transaction && r->hint == DB_HINT_WRITE) + { + key = LMDBTranslateKey(r->name); + + fakestream = open_memstream((char **) &val.mv_data, &val.mv_size); + stream = StreamFile(fakestream); + JsonEncode(r->json, stream, JSON_DEFAULT); + StreamFlush(stream); + StreamClose(stream); + + ret = mdb_put(ref->transaction, db->dbi, &key, &val, 0) == 0; + + mdb_txn_commit(ref->transaction); + LMDBKillKey(key); + } + StringArrayFree(ref->base.name); + JsonFree(ref->base.json); + Free(ref); + + if (val.mv_data) + { + free(val.mv_data); + } + if (ret && r->hint == DB_HINT_WRITE) + { + pthread_mutex_unlock(&d->lock); + } + return ret; +} +static DbRef * +LMDBCreate(Db *d, Array *k) +{ + LMDB *db = (LMDB *) d; + MDB_txn *transaction; + LMDBRef *ret = NULL; + MDB_val key, empty_json; + int code; + if (!d || !k) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + key = LMDBTranslateKey(k); + + /* create a txn */ + if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0) + { + /* Very bad! */ + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + goto end; + } + + empty_json.mv_size = 2; + empty_json.mv_data = "{}"; + /* put data in it */ + code = mdb_put(transaction, db->dbi, &key, &empty_json, MDB_NOOVERWRITE); + if (code == MDB_KEYEXIST) + { + mdb_txn_abort(transaction); + goto end; + } + else if (code == MDB_MAP_FULL) + { + Log(LOG_ERR, "%s: db is full", __func__); + mdb_txn_abort(transaction); + goto end; + } + else if (code != 0) + { + Log(LOG_ERR, "%s: mdb_put failure: %s", __func__, mdb_strerror(code)); + mdb_txn_abort(transaction); + goto end; + } + + ret = Malloc(sizeof(*ret)); + DbRefInit(d, (DbRef *) ret); + /* TODO: Timestamp */ + { + size_t i; + ret->base.name = ArrayCreate(); + for (i = 0; i < ArraySize(k); i++) + { + char *ent = ArrayGet(k, i); + StringArrayAppend(ret->base.name, ent); + } + } + ret->base.hint = DB_HINT_WRITE; + ret->base.json = HashMapCreate(); + ret->transaction = transaction; +end: + if (!ret) + { + pthread_mutex_unlock(&d->lock); + } + LMDBKillKey(key); + return (DbRef *) ret; +} + +static Array * +LMDBList(Db *d, Array *k) +{ + LMDB *db = (LMDB *) d; + MDB_val key = { 0 }; + MDB_val subKey; + MDB_val val; + Array *ret = NULL; + + MDB_cursor *cursor; + MDB_cursor_op op = MDB_SET_RANGE; + MDB_txn *txn; + int code; + + if (!d || !k) + { + return NULL; + } + + pthread_mutex_lock(&d->lock); + + /* Marked as read-only, as we just don't need to write anything + * when listing */ + if ((code = mdb_txn_begin(db->environ, NULL, MDB_RDONLY, &txn)) != 0) + { + /* Very bad! */ + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + goto end; + } + if ((code = mdb_cursor_open(txn, db->dbi, &cursor)) != 0) + { + Log(LOG_ERR, + "%s: could not get cursor: %s", + __func__, mdb_strerror(code) + ); + mdb_txn_abort(txn); + goto end; + } + + key = LMDBTranslateKey(k); + + /* Small hack to get it to list subitems */ + ((char *) key.mv_data)[0]++; + + ret = ArrayCreate(); + subKey = key; + while (mdb_cursor_get(cursor, &subKey, &val, op) == 0) + { + /* This searches by *increasing* order. The problem is that it may + * extend to unwanted points. Since the values are sorted, we can + * just exit if the subkey is incorrect. */ + char *head = LMDBKeyHead(subKey); + if (!LMDBKeyStartsWith(subKey, key)) + { + break; + } + + StringArrayAppend(ret, head); + op = MDB_NEXT; + } + + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + +end: + LMDBKillKey(key); + pthread_mutex_unlock(&d->lock); + return ret; +} + +/* Implementation functions */ +static void +LMDBClose(Db *d) +{ + LMDB *db = (LMDB *) d; + if (!d) + { + return; + } + + mdb_dbi_close(db->environ, db->dbi); + mdb_env_close(db->environ); +} + +Db * +DbOpenLMDB(char *dir, size_t size) +{ + /* TODO */ + MDB_env *env = NULL; + MDB_txn *txn; + MDB_dbi dbi; + int code; + LMDB *db; + if (!dir || !size) + { + return NULL; + } + + /* Try initialising LMDB */ + if ((code = mdb_env_create(&env)) != 0) + { + Log(LOG_ERR, + "%s: could not create LMDB env: %s", + __func__, mdb_strerror(code) + ); + return NULL; + } + if ((code = mdb_env_set_mapsize(env, size) != 0)) + { + Log(LOG_ERR, + "%s: could not set mapsize to %lu: %s", + __func__, (unsigned long) size, + mdb_strerror(code) + ); + mdb_env_close(env); + return NULL; + } + mdb_env_set_maxdbs(env, 4); + if ((code = mdb_env_open(env, dir, MDB_NOTLS, 0644)) != 0) + { + Log(LOG_ERR, + "%s: could not open LMDB env: %s", + __func__, mdb_strerror(code) + ); + mdb_env_close(env); + return NULL; + } + + /* Initialise a DBI */ + { + if ((code = mdb_txn_begin(env, NULL, 0, &txn)) != 0) + { + Log(LOG_ERR, + "%s: could not begin transaction: %s", + __func__, mdb_strerror(code) + ); + mdb_env_close(env); + return NULL; + } + if ((code = mdb_dbi_open(txn, "db", MDB_CREATE, &dbi)) != 0) + { + Log(LOG_ERR, + "%s: could not get transaction dbi: %s", + __func__, mdb_strerror(code) + ); + mdb_txn_abort(txn); + mdb_env_close(env); + return NULL; + } + mdb_txn_commit(txn); + } + + + db = Malloc(sizeof(*db)); + DbInit((Db *) db); + db->environ = env; + db->dbi = dbi; + + db->base.lockFunc = LMDBLock; + db->base.create = LMDBCreate; + db->base.unlock = LMDBUnlock; + db->base.delete = LMDBDelete; + db->base.exists = LMDBExists; + db->base.close = LMDBClose; + db->base.list = LMDBList; + + return (Db *) db; +} + +#else +Db * +DbOpenLMDB(char *dir, size_t size) +{ + /* Unimplemented function */ + Log(LOG_ERR, + "LMDB support is not enabled. Please compile with --use-lmdb" + ); + (void) size; + (void) dir; + return NULL; +} +#endif diff --git a/src/Memory.c b/src/Memory.c index 4083143..31d8050 100644 --- a/src/Memory.c +++ b/src/Memory.c @@ -40,16 +40,24 @@ #define MEMORY_HEXDUMP_WIDTH 16 #endif +#define MEMORY_FILE_SIZE 256 + struct MemoryInfo { + uint64_t magic; + size_t size; - const char *file; + char file[MEMORY_FILE_SIZE]; int line; void *pointer; + + MemoryInfo *prev; + MemoryInfo *next; }; #define MEM_BOUND_TYPE uint32_t #define MEM_BOUND 0xDEADBEEF +#define MEM_MAGIC 0xDEADBEEFDEADBEEF #define MEM_BOUND_LOWER(p) *((MEM_BOUND_TYPE *) p) #define MEM_BOUND_UPPER(p, x) *(((MEM_BOUND_TYPE *) (((uint8_t *) p) + x)) + 1) @@ -59,10 +67,10 @@ static pthread_mutex_t lock; static void (*hook) (MemoryAction, MemoryInfo *, void *) = MemoryDefaultHook; static void *hookArgs = NULL; -static MemoryInfo **allocations = NULL; -static size_t allocationsSize = 0; static size_t allocationsLen = 0; +static MemoryInfo *allocationTail = NULL; + int MemoryRuntimeInit(void) { @@ -91,70 +99,18 @@ MemoryRuntimeDestroy(void) return pthread_mutex_destroy(&lock) == 0; } -static size_t -MemoryHash(void *p) -{ - return (((size_t) p) >> 2 * 7) % allocationsSize; -} - static int MemoryInsert(MemoryInfo * a) { - size_t hash; - - if (!allocations) + if (allocationTail) { - allocationsSize = MEMORY_TABLE_CHUNK; - allocations = calloc(allocationsSize, sizeof(void *)); - if (!allocations) - { - return 0; - } + allocationTail->next = a; } + a->next = NULL; + a->prev = allocationTail; + a->magic = MEM_MAGIC; - /* If the next insertion would cause the table to be at least 3/4 - * full, re-allocate and re-hash. */ - if ((allocationsLen + 1) >= ((allocationsSize * 3) >> 2)) - { - size_t i; - size_t tmpAllocationsSize = allocationsSize; - MemoryInfo **tmpAllocations; - - allocationsSize += MEMORY_TABLE_CHUNK; - tmpAllocations = calloc(allocationsSize, sizeof(void *)); - - if (!tmpAllocations) - { - return 0; - } - - for (i = 0; i < tmpAllocationsSize; i++) - { - if (allocations[i]) - { - hash = MemoryHash(allocations[i]->pointer); - - while (tmpAllocations[hash]) - { - hash = (hash + 1) % allocationsSize; - } - - tmpAllocations[hash] = allocations[i]; - } - } - - free(allocations); - allocations = tmpAllocations; - } - - hash = MemoryHash(a->pointer); - - while (allocations[hash]) - { - hash = (hash + 1) % allocationsSize; - } - - allocations[hash] = a; + allocationTail = a; allocationsLen++; return 1; @@ -163,23 +119,24 @@ MemoryInsert(MemoryInfo * a) static void MemoryDelete(MemoryInfo * a) { - size_t hash = MemoryHash(a->pointer); - size_t count = 0; + MemoryInfo *aPrev = a->prev; + MemoryInfo *aNext = a->next; - while (count <= allocationsSize) + if (aPrev) { - if (allocations[hash] && allocations[hash] == a) - { - allocations[hash] = NULL; - allocationsLen--; - return; - } - else - { - hash = (hash + 1) % allocationsSize; - count++; - } + aPrev->next = aNext; } + if (aNext) + { + aNext->prev = aPrev; + } + + if (a == allocationTail) + { + allocationTail = aPrev; + } + + a->magic = ~MEM_MAGIC; } static int @@ -203,38 +160,31 @@ MemoryAllocate(size_t size, const char *file, int line) void *p; MemoryInfo *a; - MemoryIterate(NULL, NULL); + //MemoryIterate(NULL, NULL); pthread_mutex_lock(&lock); - a = malloc(sizeof(MemoryInfo)); + a = malloc(sizeof(MemoryInfo) + MEM_SIZE_ACTUAL(size)); if (!a) { pthread_mutex_unlock(&lock); return NULL; } - p = malloc(MEM_SIZE_ACTUAL(size)); - if (!p) - { - free(a); - pthread_mutex_unlock(&lock); - return NULL; - } + p = a + 1; memset(p, 0, MEM_SIZE_ACTUAL(size)); MEM_BOUND_LOWER(p) = MEM_BOUND; MEM_BOUND_UPPER(p, size) = MEM_BOUND; a->size = MEM_SIZE_ACTUAL(size); - a->file = file; + strncpy(a->file, file, MEMORY_FILE_SIZE - 1); a->line = line; a->pointer = p; if (!MemoryInsert(a)) { free(a); - free(p); pthread_mutex_unlock(&lock); return NULL; } @@ -254,7 +204,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line) MemoryInfo *a; void *new = NULL; - MemoryIterate(NULL, NULL); + //MemoryIterate(NULL, NULL); if (!p) { @@ -265,15 +215,17 @@ MemoryReallocate(void *p, size_t size, const char *file, int line) if (a) { pthread_mutex_lock(&lock); - new = realloc(a->pointer, MEM_SIZE_ACTUAL(size)); + MemoryDelete(a); + new = realloc(a, sizeof(MemoryInfo) + MEM_SIZE_ACTUAL(size)); if (new) { - MemoryDelete(a); + a = new; a->size = MEM_SIZE_ACTUAL(size); - a->file = file; + strncpy(a->file, file, MEMORY_FILE_SIZE - 1); a->line = line; + a->magic = MEM_MAGIC; - a->pointer = new; + a->pointer = a + 1; MemoryInsert(a); MEM_BOUND_LOWER(a->pointer) = MEM_BOUND; @@ -283,7 +235,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line) { hook(MEMORY_REALLOCATE, a, hookArgs); } - + } pthread_mutex_unlock(&lock); } @@ -293,7 +245,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line) if (a) { a->size = 0; - a->file = file; + strncpy(a->file, file, MEMORY_FILE_SIZE - 1); a->line = line; a->pointer = p; hook(MEMORY_BAD_POINTER, a, hookArgs); @@ -301,7 +253,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line) } } - return ((MEM_BOUND_TYPE *) new) + 1; + return ((MEM_BOUND_TYPE *) a->pointer) + 1; } void @@ -309,7 +261,7 @@ MemoryFree(void *p, const char *file, int line) { MemoryInfo *a; - MemoryIterate(NULL, NULL); + //MemoryIterate(NULL, NULL); if (!p) { @@ -322,12 +274,11 @@ MemoryFree(void *p, const char *file, int line) pthread_mutex_lock(&lock); if (hook) { - a->file = file; + strncpy(a->file, file, MEMORY_FILE_SIZE - 1); a->line = line; hook(MEMORY_FREE, a, hookArgs); } MemoryDelete(a); - free(a->pointer); free(a); pthread_mutex_unlock(&lock); @@ -337,7 +288,7 @@ MemoryFree(void *p, const char *file, int line) a = malloc(sizeof(MemoryInfo)); if (a) { - a->file = file; + strncpy(a->file, file, MEMORY_FILE_SIZE - 1); a->line = line; a->size = 0; a->pointer = p; @@ -350,17 +301,15 @@ MemoryFree(void *p, const char *file, int line) size_t MemoryAllocated(void) { - size_t i; size_t total = 0; + MemoryInfo *cur; pthread_mutex_lock(&lock); - for (i = 0; i < allocationsSize; i++) + /* TODO */ + for (cur = allocationTail; cur; cur = cur->prev) { - if (allocations[i]) - { - total += MemoryInfoGetSize(allocations[i]); - } + total += MemoryInfoGetSize(cur); } pthread_mutex_unlock(&lock); @@ -371,55 +320,40 @@ MemoryAllocated(void) void MemoryFreeAll(void) { - size_t i; + MemoryInfo *cur; + MemoryInfo *prev; pthread_mutex_lock(&lock); - for (i = 0; i < allocationsSize; i++) + /* TODO */ + for (cur = allocationTail; cur; cur = prev) { - if (allocations[i]) - { - free(allocations[i]->pointer); - free(allocations[i]); - } + prev = cur->prev; + free(cur); } - free(allocations); - allocations = NULL; - allocationsSize = 0; allocationsLen = 0; pthread_mutex_unlock(&lock); } MemoryInfo * -MemoryInfoGet(void *p) +MemoryInfoGet(void *po) { - size_t hash, count; + void *p = po; pthread_mutex_lock(&lock); - p = ((MEM_BOUND_TYPE *) p) - 1; - hash = MemoryHash(p); + p = ((MEM_BOUND_TYPE *) po) - 1; + p = ((MemoryInfo *) p) - 1; - count = 0; - while (count <= allocationsSize) + if (((MemoryInfo *)p)->magic != MEM_MAGIC) { - if (!allocations[hash] || allocations[hash]->pointer != p) - { - hash = (hash + 1) % allocationsSize; - count++; - } - else - { - MemoryCheck(allocations[hash]); - pthread_mutex_unlock(&lock); - return allocations[hash]; - } + p = NULL; } pthread_mutex_unlock(&lock); - return NULL; + return p; } size_t @@ -467,21 +401,18 @@ MemoryInfoGetPointer(MemoryInfo * a) } void - MemoryIterate(void (*iterFunc) (MemoryInfo *, void *), void *args) +MemoryIterate(void (*iterFunc) (MemoryInfo *, void *), void *args) { - size_t i; + MemoryInfo *cur; pthread_mutex_lock(&lock); - for (i = 0; i < allocationsSize; i++) + for (cur = allocationTail; cur; cur = cur->prev) { - if (allocations[i]) + MemoryCheck(cur); + if (iterFunc) { - MemoryCheck(allocations[i]); - if (iterFunc) - { - iterFunc(allocations[i], args); - } + iterFunc(cur, args); } }