lda/telodendria
lda/telodendria
1
1
Fork 1
forked from Telodendria/Telodendria
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: lda:d0b5c441dd3061c45e7bd5f25020070b1ea66f6b
Branches Tags
lda:master
lda:roomwerk
lda:fix-configure
lda:fix-doublefree
lda:fix-mistake
lda:implement-22-9
lda:contribute
lda:replace-config-j2s
lda:setup-ci-workflows
lda:refactor-matrix-error-create
lda:implement-22
lda:refactor-6
lda:implement-26
lda:implement-27
lda:fix-33
lda:jordan
lda:initial
Telodendria:master
lda:Telodendria-0_3_0
lda:Telodendria-0_2_1
lda:Telodendria-0_2_0
lda:Telodendria-0_1_0
Telodendria:Telodendria-0_1_0
Telodendria:Telodendria-0_2_0
Telodendria:Telodendria-0_2_1
Telodendria:Telodendria-0_3_0
..
compare: lda:e8543bdb2a1965298d3b5a6590c85b3b58f2c4f0
Branches Tags
lda:roomwerk
lda:master
lda:fix-configure
lda:fix-doublefree
lda:fix-mistake
lda:implement-22-9
lda:contribute
lda:replace-config-j2s
lda:setup-ci-workflows
lda:refactor-matrix-error-create
lda:implement-22
lda:refactor-6
lda:implement-26
lda:implement-27
lda:fix-33
lda:jordan
lda:initial
Telodendria:master
lda:Telodendria-0_3_0
lda:Telodendria-0_2_1
lda:Telodendria-0_2_0
lda:Telodendria-0_1_0
Telodendria:Telodendria-0_1_0
Telodendria:Telodendria-0_2_0
Telodendria:Telodendria-0_2_1
Telodendria:Telodendria-0_3_0

No commits in common. "d0b5c441dd3061c45e7bd5f25020070b1ea66f6b" and "e8543bdb2a1965298d3b5a6590c85b3b58f2c4f0" have entirely different histories.
d0b5c441dd ... e8543bdb2a

11 changed files with 1128 additions and 1694 deletions
Show stats Expand all files Collapse all files

27
configure vendored
View file

@ -15,7 +15,7 @@ TOOLS="tools"
# Default compiler flags. These must be supported by all POSIX C compilers.
# "Fancy" compilers that have additional options must be detected and set below.
CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE} -I${SRC}";
CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE}"
LIBS="-lm -lpthread"
# Default args for all platforms.
@ -24,10 +24,10 @@ SCRIPT_ARGS="--prefix=/usr/local --lib-name=Cytoplasm"
# Set SSL flags depending on the platform.
case "$(uname)" in
OpenBSD)
SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl --disable-lmdb"
SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl"
;;
*)
SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl --disable-lmdb"
SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl"
;;
esac
@ -80,14 +80,6 @@ for arg in $SCRIPT_ARGS; do
TLS_IMPL=""
TLS_LIBS=""
;;
--with-lmdb)
EDB_IMPL="EDB_LMDB"
EDB_LIBS="-llmdb"
;;
--disable-lmdb)
EDB_IMPL=""
EDB_LIBS=""
;;
--prefix=*)
PREFIX=$(echo "$arg" | cut -d '=' -f 2-)
;;
@ -112,11 +104,6 @@ if [ -n "$TLS_IMPL" ]; then
LIBS="${LIBS} ${TLS_LIBS}"
fi
if [ -n "$EDB_IMPL" ]; then
CFLAGS="${CFLAGS} -D${EDB_IMPL}"
LIBS="${LIBS} ${EDB_LIBS}"
fi
CFLAGS="${CFLAGS} '-DLIB_NAME=\"${LIB_NAME}\"' ${DEBUG}"
LDFLAGS="${LIBS} ${LDFLAGS}"
@ -153,7 +140,7 @@ print_obj() {
get_deps() {
src="$1"
${CC} -I${SRC} -I${INCLUDE} -E "$src" \
${CC} -I${INCLUDE} -E "$src" \
| grep '^#' \
| awk '{print $3}' \
| cut -d '"' -f 2 \
@ -269,10 +256,8 @@ ${TAB}done
${LIB_NAME}: ${OUT}/lib/lib${LIB_NAME}.a ${OUT}/lib/lib${LIB_NAME}.so
install: ${LIB_NAME}
${TAB}mkdir -p \$(PREFIX)/${OUT}/lib
${TAB}mkdir -p \$(PREFIX)/lib
${TAB}cp ${OUT}/lib/lib${LIB_NAME}.a \$(PREFIX)/lib/lib${LIB_NAME}.a
${TAB}cp ${OUT}/lib/lib${LIB_NAME}.so \$(PREFIX)/lib/lib${LIB_NAME}.so
${TAB}install -D ${OUT}/lib/lib${LIB_NAME}.a \$(PREFIX)/lib/lib${LIB_NAME}.a
${TAB}install -D ${OUT}/lib/lib${LIB_NAME}.so \$(PREFIX)/lib/lib${LIB_NAME}.so
$(collect ${INCLUDE}/ '' '' \$\(PREFIX\)/include/${LIB_NAME}/ install_out)
$(collect ${INCLUDE}/ .h .3 \$\(PREFIX\)/man/man3/${LIB_NAME}- install_man)
$(collect ${TOOLS}/ '.c' '' \$\(PREFIX\)/bin/ install_tool)

29
include/Cytoplasm/Db.h
View file

@ -48,17 +48,6 @@
*/
typedef struct Db Db;
/**
* Some "hints" for the database backend for operations.
* Hints are a way for the program to declare what to except of it
* (and the program MUST adhere to these hints, but the backend
* MAY adhere).
*/
typedef enum DbHint {
DB_HINT_READONLY, /* The database reference is treated as read-only */
DB_HINT_WRITE /* The database reference is treated as read/write */
} DbHint;
/**
* When an object is locked, a reference is returned. This reference
* is owned by the current thread, and the database is inaccessible to
@ -79,15 +68,6 @@ typedef struct DbRef DbRef;
*/
extern Db * DbOpen(char *, size_t);
/**
* Open a LMDB data directory. This function is similar to
* .Fn DbOpen ,
* but works with a LMDB-based backend, with the second argument
* being the maximum filesize. This function fails with NULL if ever
* called without LMDB enabled at compiletime.
*/
extern Db * DbOpenLMDB(char *, size_t);
/**
* Close the database. This function will flush anything in the cache
* to the disk, and then close the data directory. It assumes that
@ -129,15 +109,6 @@ extern DbRef * DbCreate(Db *, size_t,...);
*/
extern DbRef * DbLock(Db *, size_t,...);
/**
* Behaves like
* .Fn DbLock ,
* but with hints on the reference itself, as
* .Fn DbLock
* itself is read/write.
*/
extern DbRef * DbLockIntent(Db *, DbHint, size_t,...);
/**
* Immediately and permanently remove an object from the database.
* This function assumes the object is not locked, otherwise undefined

977
src/Db.c Normal file
View file

@ -0,0 +1,977 @@
/*
* Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation files
* (the "Software"), to deal in the Software without restriction,
* including without limitation the rights to use, copy, modify, merge,
* publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <Db.h>
#include <Memory.h>
#include <Json.h>
#include <Util.h>
#include <Str.h>
#include <Stream.h>
#include <Log.h>
#include <sys/types.h>
#include <dirent.h>
#include <pthread.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
struct Db
{
char *dir;
pthread_mutex_t lock;
size_t cacheSize;
size_t maxCache;
HashMap *cache;
/*
* The cache uses a double linked list (see DbRef
* below) to know which objects are most and least
* recently used. The following diagram helps me
* know what way all the pointers go, because it
* can get very confusing sometimes. For example,
* there's nothing stopping "next" from pointing to
* least recent, and "prev" from pointing to most
* recent, so hopefully this clarifies the pointer
* terminology used when dealing with the linked
* list:
*
* mostRecent leastRecent
* | prev prev | prev
* +---+ ---> +---+ ---> +---+ ---> NULL
* |ref| |ref| |ref|
* NULL <--- +---+ <--- +---+ <--- +---+
* next next next
*/
DbRef *mostRecent;
DbRef *leastRecent;
};
struct DbRef
{
HashMap *json;
uint64_t ts;
size_t size;
Array *name;
DbRef *prev;
DbRef *next;
int fd;
Stream *stream;
};
static void
StringArrayFree(Array * arr)
{
size_t i;
for (i = 0; i < ArraySize(arr); i++)
{
Free(ArrayGet(arr, i));
}
ArrayFree(arr);
}
static ssize_t DbComputeSize(HashMap *);
static ssize_t
DbComputeSizeOfValue(JsonValue * val)
{
MemoryInfo *a;
ssize_t total = 0;
size_t i;
union
{
char *str;
Array *arr;
} u;
if (!val)
{
return -1;
}
a = MemoryInfoGet(val);
if (a)
{
total += MemoryInfoGetSize(a);
}
switch (JsonValueType(val))
{
case JSON_OBJECT:
total += DbComputeSize(JsonValueAsObject(val));
break;
case JSON_ARRAY:
u.arr = JsonValueAsArray(val);
a = MemoryInfoGet(u.arr);
if (a)
{
total += MemoryInfoGetSize(a);
}
for (i = 0; i < ArraySize(u.arr); i++)
{
total += DbComputeSizeOfValue(ArrayGet(u.arr, i));
}
break;
case JSON_STRING:
u.str = JsonValueAsString(val);
a = MemoryInfoGet(u.str);
if (a)
{
total += MemoryInfoGetSize(a);
}
break;
case JSON_NULL:
case JSON_INTEGER:
case JSON_FLOAT:
case JSON_BOOLEAN:
default:
/* These don't use any extra heap space */
break;
}
return total;
}
static ssize_t
DbComputeSize(HashMap * json)
{
char *key;
JsonValue *val;
MemoryInfo *a;
size_t total;
if (!json)
{
return -1;
}
total = 0;
a = MemoryInfoGet(json);
if (a)
{
total += MemoryInfoGetSize(a);
}
while (HashMapIterate(json, &key, (void **) &val))
{
a = MemoryInfoGet(key);
if (a)
{
total += MemoryInfoGetSize(a);
}
total += DbComputeSizeOfValue(val);
}
return total;
}
static char *
DbHashKey(Array * args)
{
size_t i;
char *str = NULL;
for (i = 0; i < ArraySize(args); i++)
{
char *tmp = StrConcat(2, str, ArrayGet(args, i));
Free(str);
str = tmp;
}
return str;
}
static char
DbSanitiseChar(char input)
{
switch (input)
{
case '/':
return '_';
case '.':
return '-';
}
return input;
}
static char *
DbDirName(Db * db, Array * args, size_t strip)
{
size_t i, j;
char *str = StrConcat(2, db->dir, "/");
for (i = 0; i < ArraySize(args) - strip; i++)
{
char *tmp;
char *sanitise = StrDuplicate(ArrayGet(args, i));
for (j = 0; j < strlen(sanitise); j++)
{
sanitise[j] = DbSanitiseChar(sanitise[j]);
}
tmp = StrConcat(3, str, sanitise, "/");
Free(str);
Free(sanitise);
str = tmp;
}
return str;
}
static char *
DbFileName(Db * db, Array * args)
{
size_t i;
char *str = StrConcat(2, db->dir, "/");
for (i = 0; i < ArraySize(args); i++)
{
char *tmp;
char *arg = StrDuplicate(ArrayGet(args, i));
size_t j = 0;
/* Sanitize name to prevent directory traversal attacks */
while (arg[j])
{
arg[j] = DbSanitiseChar(arg[j]);
j++;
}
tmp = StrConcat(3, str, arg,
(i < ArraySize(args) - 1) ? "/" : ".json");
Free(arg);
Free(str);
str = tmp;
}
return str;
}
static void
DbCacheEvict(Db * db)
{
DbRef *ref = db->leastRecent;
DbRef *tmp;
while (ref && db->cacheSize > db->maxCache)
{
char *hash;
JsonFree(ref->json);
hash = DbHashKey(ref->name);
HashMapDelete(db->cache, hash);
Free(hash);
StringArrayFree(ref->name);
db->cacheSize -= ref->size;
if (ref->next)
{
ref->next->prev = ref->prev;
}
else
{
db->mostRecent = ref->prev;
}
if (ref->prev)
{
ref->prev->next = ref->next;
}
else
{
db->leastRecent = ref->next;
}
tmp = ref->next;
Free(ref);
ref = tmp;
}
}
Db *
DbOpen(char *dir, size_t cache)
{
Db *db;
pthread_mutexattr_t attr;
if (!dir)
{
return NULL;
}
db = Malloc(sizeof(Db));
if (!db)
{
return NULL;
}
db->dir = dir;
db->maxCache = cache;
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&db->lock, &attr);
pthread_mutexattr_destroy(&attr);
db->mostRecent = NULL;
db->leastRecent = NULL;
db->cacheSize = 0;
if (db->maxCache)
{
db->cache = HashMapCreate();
if (!db->cache)
{
return NULL;
}
}
else
{
db->cache = NULL;
}
return db;
}
void
DbMaxCacheSet(Db * db, size_t cache)
{
if (!db)
{
return;
}
pthread_mutex_lock(&db->lock);
db->maxCache = cache;
if (db->maxCache && !db->cache)
{
db->cache = HashMapCreate();
db->cacheSize = 0;
}
DbCacheEvict(db);
pthread_mutex_unlock(&db->lock);
}
void
DbClose(Db * db)
{
if (!db)
{
return;
}
pthread_mutex_lock(&db->lock);
DbMaxCacheSet(db, 0);
DbCacheEvict(db);
HashMapFree(db->cache);
pthread_mutex_unlock(&db->lock);
pthread_mutex_destroy(&db->lock);
Free(db);
}
static DbRef *
DbLockFromArr(Db * db, Array * args)
{
char *file;
char *hash;
DbRef *ref;
struct flock lock;
int fd;
Stream *stream;
if (!db || !args)
{
return NULL;
}
ref = NULL;
hash = NULL;
pthread_mutex_lock(&db->lock);
/* Check if the item is in the cache */
hash = DbHashKey(args);
ref = HashMapGet(db->cache, hash);
file = DbFileName(db, args);
fd = open(file, O_RDWR);
if (fd == -1)
{
if (ref)
{
HashMapDelete(db->cache, hash);
JsonFree(ref->json);
StringArrayFree(ref->name);
db->cacheSize -= ref->size;
if (ref->next)
{
ref->next->prev = ref->prev;
}
else
{
db->mostRecent = ref->prev;
}
if (ref->prev)
{
ref->prev->next = ref->next;
}
else
{
db->leastRecent = ref->next;
}
if (!db->leastRecent)
{
db->leastRecent = db->mostRecent;
}
Free(ref);
}
ref = NULL;
goto finish;
}
stream = StreamFd(fd);
lock.l_start = 0;
lock.l_len = 0;
lock.l_type = F_WRLCK;
lock.l_whence = SEEK_SET;
/* Lock the file on the disk */
if (fcntl(fd, F_SETLK, &lock) < 0)
{
StreamClose(stream);
ref = NULL;
goto finish;
}
if (ref) /* In cache */
{
uint64_t diskTs = UtilLastModified(file);
ref->fd = fd;
ref->stream = stream;
if (diskTs > ref->ts)
{
/* File was modified on disk since it was cached */
HashMap *json = JsonDecode(ref->stream);
if (!json)
{
StreamClose(ref->stream);
ref = NULL;
goto finish;
}
JsonFree(ref->json);
ref->json = json;
ref->ts = diskTs;
ref->size = DbComputeSize(ref->json);
}
/* Float this ref to mostRecent */
if (ref->next)
{
ref->next->prev = ref->prev;
if (!ref->prev)
{
db->leastRecent = ref->next;
}
else
{
ref->prev->next = ref->next;
}
ref->prev = db->mostRecent;
ref->next = NULL;
if (db->mostRecent)
{
db->mostRecent->next = ref;
}
db->mostRecent = ref;
}
/* If there is no least recent, this is the only thing in the
* cache, so it is also least recent. */
if (!db->leastRecent)
{
db->leastRecent = ref;
}
/* The file on disk may be larger than what we have in memory,
* which may require items in cache to be evicted. */
DbCacheEvict(db);
}
else
{
Array *name;
size_t i;
/* Not in cache; load from disk */
ref = Malloc(sizeof(DbRef));
if (!ref)
{
StreamClose(stream);
goto finish;
}
ref->json = JsonDecode(stream);
if (!ref->json)
{
Free(ref);
StreamClose(stream);
ref = NULL;
goto finish;
}
ref->fd = fd;
ref->stream = stream;
name = ArrayCreate();
for (i = 0; i < ArraySize(args); i++)
{
ArrayAdd(name, StrDuplicate(ArrayGet(args, i)));
}
ref->name = name;
if (db->cache)
{
ref->ts = UtilTsMillis();
ref->size = DbComputeSize(ref->json);
HashMapSet(db->cache, hash, ref);
db->cacheSize += ref->size;
ref->next = NULL;
ref->prev = db->mostRecent;
if (db->mostRecent)
{
db->mostRecent->next = ref;
}
db->mostRecent = ref;
if (!db->leastRecent)
{
db->leastRecent = ref;
}
/* Adding this item to the cache may case it to grow too
* large, requiring some items to be evicted */
DbCacheEvict(db);
}
}
finish:
if (!ref)
{
pthread_mutex_unlock(&db->lock);
}
Free(file);
Free(hash);
return ref;
}
DbRef *
DbCreate(Db * db, size_t nArgs,...)
{
Stream *fp;
char *file;
char *dir;
va_list ap;
Array *args;
DbRef *ret;
if (!db)
{
return NULL;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return NULL;
}
pthread_mutex_lock(&db->lock);
file = DbFileName(db, args);
if (UtilLastModified(file))
{
Free(file);
ArrayFree(args);
pthread_mutex_unlock(&db->lock);
return NULL;
}
dir = DbDirName(db, args, 1);
if (UtilMkdir(dir, 0750) < 0)
{
Free(file);
ArrayFree(args);
Free(dir);
pthread_mutex_unlock(&db->lock);
return NULL;
}
Free(dir);
fp = StreamOpen(file, "w");
Free(file);
if (!fp)
{
ArrayFree(args);
pthread_mutex_unlock(&db->lock);
return NULL;
}
StreamPuts(fp, "{}");
StreamClose(fp);
/* DbLockFromArr() will lock again for us */
pthread_mutex_unlock(&db->lock);
ret = DbLockFromArr(db, args);
ArrayFree(args);
return ret;
}
bool
DbDelete(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
char *file;
char *hash;
bool ret = true;
DbRef *ref;
if (!db)
{
return false;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
pthread_mutex_lock(&db->lock);
hash = DbHashKey(args);
file = DbFileName(db, args);
ref = HashMapGet(db->cache, hash);
if (ref)
{
HashMapDelete(db->cache, hash);
JsonFree(ref->json);
StringArrayFree(ref->name);
db->cacheSize -= ref->size;
if (ref->next)
{
ref->next->prev = ref->prev;
}
else
{
db->mostRecent = ref->prev;
}
if (ref->prev)
{
ref->prev->next = ref->next;
}
else
{
db->leastRecent = ref->next;
}
if (!db->leastRecent)
{
db->leastRecent = db->mostRecent;
}
Free(ref);
}
Free(hash);
if (UtilLastModified(file))
{
ret = (remove(file) == 0);
}
pthread_mutex_unlock(&db->lock);
ArrayFree(args);
Free(file);
return ret;
}
DbRef *
DbLock(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
DbRef *ret;
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return NULL;
}
ret = DbLockFromArr(db, args);
ArrayFree(args);
return ret;
}
bool
DbUnlock(Db * db, DbRef * ref)
{
bool destroy;
if (!db || !ref)
{
return false;
}
lseek(ref->fd, 0L, SEEK_SET);
if (ftruncate(ref->fd, 0) < 0)
{
pthread_mutex_unlock(&db->lock);
Log(LOG_ERR, "Failed to truncate file on disk.");
Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno));
return false;
}
JsonEncode(ref->json, ref->stream, JSON_DEFAULT);
StreamClose(ref->stream);
if (db->cache)
{
char *key = DbHashKey(ref->name);
if (HashMapGet(db->cache, key))
{
db->cacheSize -= ref->size;
ref->size = DbComputeSize(ref->json);
db->cacheSize += ref->size;
/* If this ref has grown significantly since we last
* computed its size, it may have filled the cache and
* require some items to be evicted. */
DbCacheEvict(db);
destroy = false;
}
else
{
destroy = true;
}
Free(key);
}
else
{
destroy = true;
}
if (destroy)
{
JsonFree(ref->json);
StringArrayFree(ref->name);
Free(ref);
}
pthread_mutex_unlock(&db->lock);
return true;
}
bool
DbExists(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
char *file;
bool ret;
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return false;
}
pthread_mutex_lock(&db->lock);
file = DbFileName(db, args);
ret = (UtilLastModified(file) != 0);
pthread_mutex_unlock(&db->lock);
Free(file);
ArrayFree(args);
return ret;
}
Array *
DbList(Db * db, size_t nArgs,...)
{
Array *result;
Array *path;
DIR *files;
struct dirent *file;
char *dir;
va_list ap;
if (!db || !nArgs)
{
return NULL;
}
result = ArrayCreate();
if (!result)
{
return NULL;
}
va_start(ap, nArgs);
path = ArrayFromVarArgs(nArgs, ap);
dir = DbDirName(db, path, 0);
pthread_mutex_lock(&db->lock);
files = opendir(dir);
if (!files)
{
ArrayFree(path);
ArrayFree(result);
Free(dir);
pthread_mutex_unlock(&db->lock);
return NULL;
}
while ((file = readdir(files)))
{
size_t namlen = strlen(file->d_name);
if (namlen > 5)
{
int nameOffset = namlen - 5;
if (StrEquals(file->d_name + nameOffset, ".json"))
{
file->d_name[nameOffset] = '\0';
ArrayAdd(result, StrDuplicate(file->d_name));
}
}
}
closedir(files);
ArrayFree(path);
Free(dir);
pthread_mutex_unlock(&db->lock);
return result;
}
void
DbListFree(Array * arr)
{
StringArrayFree(arr);
}
HashMap *
DbJson(DbRef * ref)
{
return ref ? ref->json : NULL;
}
bool
DbJsonSet(DbRef * ref, HashMap * json)
{
if (!ref || !json)
{
return false;
}
JsonFree(ref->json);
ref->json = JsonDuplicate(json);
return true;
}

499
src/Db/Db.c
View file

@ -1,499 +0,0 @@
/*
* Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation files
* (the "Software"), to deal in the Software without restriction,
* including without limitation the rights to use, copy, modify, merge,
* publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <Db.h>
#include <Memory.h>
#include <Json.h>
#include <Util.h>
#include <Str.h>
#include <Stream.h>
#include <Log.h>
#include <sys/types.h>
#include <dirent.h>
#include <pthread.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include "Db/Internal.h"
void
StringArrayFree(Array * arr)
{
size_t i;
for (i = 0; i < ArraySize(arr); i++)
{
Free(ArrayGet(arr, i));
}
ArrayFree(arr);
}
static ssize_t DbComputeSize(HashMap *);
static ssize_t
DbComputeSizeOfValue(JsonValue * val)
{
MemoryInfo *a;
ssize_t total = 0;
size_t i;
union
{
char *str;
Array *arr;
} u;
if (!val)
{
return -1;
}
a = MemoryInfoGet(val);
if (a)
{
total += MemoryInfoGetSize(a);
}
switch (JsonValueType(val))
{
case JSON_OBJECT:
total += DbComputeSize(JsonValueAsObject(val));
break;
case JSON_ARRAY:
u.arr = JsonValueAsArray(val);
a = MemoryInfoGet(u.arr);
if (a)
{
total += MemoryInfoGetSize(a);
}
for (i = 0; i < ArraySize(u.arr); i++)
{
total += DbComputeSizeOfValue(ArrayGet(u.arr, i));
}
break;
case JSON_STRING:
u.str = JsonValueAsString(val);
a = MemoryInfoGet(u.str);
if (a)
{
total += MemoryInfoGetSize(a);
}
break;
case JSON_NULL:
case JSON_INTEGER:
case JSON_FLOAT:
case JSON_BOOLEAN:
default:
/* These don't use any extra heap space */
break;
}
return total;
}
static ssize_t
DbComputeSize(HashMap * json)
{
char *key;
JsonValue *val;
MemoryInfo *a;
size_t total;
if (!json)
{
return -1;
}
total = 0;
a = MemoryInfoGet(json);
if (a)
{
total += MemoryInfoGetSize(a);
}
while (HashMapIterate(json, &key, (void **) &val))
{
a = MemoryInfoGet(key);
if (a)
{
total += MemoryInfoGetSize(a);
}
total += DbComputeSizeOfValue(val);
}
return total;
}
static char *
DbHashKey(Array * args)
{
size_t i;
char *str = NULL;
for (i = 0; i < ArraySize(args); i++)
{
char *tmp = StrConcat(2, str, ArrayGet(args, i));
Free(str);
str = tmp;
}
return str;
}
static void
DbCacheEvict(Db * db)
{
DbRef *ref = db->leastRecent;
DbRef *tmp;
while (ref && db->cacheSize > db->maxCache)
{
char *hash;
JsonFree(ref->json);
hash = DbHashKey(ref->name);
HashMapDelete(db->cache, hash);
Free(hash);
StringArrayFree(ref->name);
db->cacheSize -= ref->size;
if (ref->next)
{
ref->next->prev = ref->prev;
}
else
{
db->mostRecent = ref->prev;
}
if (ref->prev)
{
ref->prev->next = ref->next;
}
else
{
db->leastRecent = ref->next;
}
tmp = ref->next;
Free(ref);
ref = tmp;
}
}
void
DbMaxCacheSet(Db * db, size_t cache)
{
if (!db)
{
return;
}
pthread_mutex_lock(&db->lock);
db->maxCache = cache;
if (db->maxCache && !db->cache)
{
db->cache = HashMapCreate();
db->cacheSize = 0;
}
DbCacheEvict(db);
pthread_mutex_unlock(&db->lock);
}
void
DbClose(Db * db)
{
if (!db)
{
return;
}
pthread_mutex_lock(&db->lock);
if (db->close)
{
db->close(db);
}
DbMaxCacheSet(db, 0);
DbCacheEvict(db);
HashMapFree(db->cache);
pthread_mutex_unlock(&db->lock);
pthread_mutex_destroy(&db->lock);
Free(db);
}
DbRef *
DbCreate(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
DbRef *ret;
if (!db || !db->create)
{
return NULL;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return NULL;
}
ret = db->create(db, args);
ArrayFree(args);
return ret;
}
bool
DbDelete(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
bool ret = true;
if (!db)
{
return false;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
ret = db->delete(db, args);
ArrayFree(args);
return ret;
}
DbRef *
DbLock(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
DbRef *ret;
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args || !db->lockFunc)
{
return NULL;
}
ret = db->lockFunc(db, DB_HINT_WRITE, args);
ArrayFree(args);
return ret;
}
DbRef *
DbLockIntent(Db * db, DbHint hint, size_t nArgs,...)
{
va_list ap;
Array *args;
DbRef *ret;
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args || !db->lockFunc)
{
return NULL;
}
ret = db->lockFunc(db, hint, args);
ArrayFree(args);
return ret;
}
bool
DbUnlock(Db * db, DbRef * ref)
{
if (!db || !ref || !db->unlock)
{
return false;
}
return db->unlock(db, ref);
}
bool
DbExists(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
bool ret;
if (!db || !nArgs || !db->exists)
{
return false;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return false;
}
ret = db->exists(db, args);
ArrayFree(args);
return ret;
}
Array *
DbList(Db * db, size_t nArgs,...)
{
Array *result;
Array *path;
va_list ap;
if (!db || !nArgs || !db->list)
{
return NULL;
}
va_start(ap, nArgs);
path = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
result = db->list(db, path);
ArrayFree(path);
return result;
}
void
DbListFree(Array * arr)
{
StringArrayFree(arr);
}
HashMap *
DbJson(DbRef * ref)
{
return ref ? ref->json : NULL;
}
bool
DbJsonSet(DbRef * ref, HashMap * json)
{
if (!ref || !json)
{
return false;
}
JsonFree(ref->json);
ref->json = JsonDuplicate(json);
return true;
}
void
DbInit(Db *db)
{
pthread_mutexattr_t attr;
if (!db)
{
return;
}
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&db->lock, &attr);
pthread_mutexattr_destroy(&attr);
db->mostRecent = NULL;
db->leastRecent = NULL;
db->cacheSize = 0;
db->maxCache = 0;
if (db->maxCache)
{
db->cache = HashMapCreate();
}
else
{
db->cache = NULL;
}
}
void
DbRefInit(Db *db, DbRef *ref)
{
if (!db || !ref)
{
return;
}
ref->prev = db->mostRecent;
ref->next = NULL;
ref->json = NULL;
ref->name = NULL;
/* As default values to be overwritten by impls */
ref->ts = UtilTsMillis();
ref->size = 0;
/* TODO: Append the ref to the cache list.
* I removed it because it broke everything and crashed all the time.
* My bad! */
}
void
StringArrayAppend(Array *arr, char *str)
{
if (!arr || !str)
{
return;
}
ArrayAdd(arr, StrDuplicate(str));
}

374
src/Db/Flat.c
View file

@ -1,374 +0,0 @@
#include <Db.h>
#include <Memory.h>
#include <Json.h>
#include <Util.h>
#include <Str.h>
#include <Stream.h>
#include <Log.h>
#include <sys/types.h>
#include <dirent.h>
#include <pthread.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include "Db/Internal.h"
typedef struct FlatDb {
Db base;
char *dir;
/* Theres not much to do here. */
} FlatDb;
typedef struct FlatDbRef {
DbRef base;
Stream *stream;
int fd;
} FlatDbRef;
static char
DbSanitiseChar(char input)
{
switch (input)
{
case '/':
return '_';
case '.':
return '-';
}
return input;
}
static char *
DbDirName(FlatDb * db, Array * args, size_t strip)
{
size_t i, j;
char *str = StrConcat(2, db->dir, "/");
for (i = 0; i < ArraySize(args) - strip; i++)
{
char *tmp;
char *sanitise = StrDuplicate(ArrayGet(args, i));
for (j = 0; j < strlen(sanitise); j++)
{
sanitise[j] = DbSanitiseChar(sanitise[j]);
}
tmp = StrConcat(3, str, sanitise, "/");
Free(str);
Free(sanitise);
str = tmp;
}
return str;
}
static char *
DbFileName(FlatDb * db, Array * args)
{
size_t i;
char *str = StrConcat(2, db->dir, "/");
for (i = 0; i < ArraySize(args); i++)
{
char *tmp;
char *arg = StrDuplicate(ArrayGet(args, i));
size_t j = 0;
/* Sanitize name to prevent directory traversal attacks */
while (arg[j])
{
arg[j] = DbSanitiseChar(arg[j]);
j++;
}
tmp = StrConcat(3, str, arg,
(i < ArraySize(args) - 1) ? "/" : ".json");
Free(arg);
Free(str);
str = tmp;
}
return str;
}
static DbRef *
FlatLock(Db *d, DbHint hint, Array *dir)
{
FlatDb *db = (FlatDb *) d;
FlatDbRef *ref = NULL;
size_t i;
char *path = NULL;
if (!d || !dir)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
path = DbFileName(db, dir);
/* TODO: Caching */
{
int fd = open(path, O_RDWR);
Stream *stream;
struct flock lock;
if (fd == -1)
{
ref = NULL;
goto end;
}
stream = StreamFd(fd);
if (!stream)
{
ref = NULL;
goto end;
}
lock.l_start = 0;
lock.l_len = 0;
lock.l_type = F_WRLCK;
lock.l_whence = SEEK_SET;
if (fcntl(fd, F_SETLK, &lock) < 0)
{
StreamClose(stream);
ref = NULL;
goto end;
}
ref = Malloc(sizeof(*ref));
DbRefInit(d, (DbRef *) ref);
/* TODO: Hints */
ref->base.hint = hint;
ref->base.ts = UtilLastModified(path);
ref->base.json = JsonDecode(stream);
ref->stream = stream;
ref->fd = fd;
if (!ref->base.json)
{
Free(ref);
StreamClose(stream);
ref = NULL;
goto end;
}
ref->base.name = ArrayCreate();
for (i = 0; i < ArraySize(dir); i++)
{
StringArrayAppend(ref->base.name, ArrayGet(dir, i));
}
}
end:
Free(path);
if (!ref)
{
pthread_mutex_unlock(&d->lock);
}
return (DbRef *) ref;
}
static bool
FlatUnlock(Db *d, DbRef *r)
{
FlatDbRef *ref = (FlatDbRef *) r;
if (!d || !r)
{
return false;
}
lseek(ref->fd, 0L, SEEK_SET);
if (ftruncate(ref->fd, 0) < 0)
{
pthread_mutex_unlock(&d->lock);
Log(LOG_ERR, "Failed to truncate file on disk.");
Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno));
return false;
}
JsonEncode(ref->base.json, ref->stream, JSON_DEFAULT);
StreamClose(ref->stream);
JsonFree(ref->base.json);
StringArrayFree(ref->base.name);
Free(ref);
pthread_mutex_unlock(&d->lock);
return true;
}
static DbRef *
FlatCreate(Db *d, Array *dir)
{
FlatDb *db = (FlatDb *) d;
char *path, *dirPath;
Stream *fp;
DbRef *ret;
if (!d || !dir)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
path = DbFileName(db, dir);
if (UtilLastModified(path))
{
Free(path);
pthread_mutex_unlock(&d->lock);
return NULL;
}
dirPath = DbDirName(db, dir, 1);
if (UtilMkdir(dirPath, 0750) < 0)
{
Free(path);
Free(dirPath);
pthread_mutex_unlock(&d->lock);
return NULL;
}
Free(dirPath);
fp = StreamOpen(path, "w");
Free(path);
if (!fp)
{
pthread_mutex_unlock(&d->lock);
return NULL;
}
StreamPuts(fp, "{}");
StreamClose(fp);
/* FlatLock() will lock again for us */
pthread_mutex_unlock(&d->lock);
ret = FlatLock(d, DB_HINT_WRITE, dir);
return ret;
}
static bool
FlatDelete(Db *d, Array *dir)
{
bool ret = false;
char *file;
FlatDb *db = (FlatDb *) d;
if (!d || !dir)
{
return false;
}
pthread_mutex_lock(&d->lock);
file = DbFileName(db, dir);
/* TODO: Unlink the entry from the linkedlist */
if (UtilLastModified(file))
{
ret = remove(file) == 0;
}
Free(file);
pthread_mutex_unlock(&d->lock);
return ret;
}
static Array *
FlatList(Db *d, Array *dir)
{
FlatDb *db = (FlatDb *) d;
struct dirent *file;
Array *ret;
DIR *files;
char *path;
if (!d || !dir)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
path = DbDirName(db, dir, 0);
files = opendir(path);
if (!files)
{
Free(path);
pthread_mutex_unlock(&d->lock);
return NULL;
}
ret = ArrayCreate();
while ((file = readdir(files)))
{
size_t namlen = strlen(file->d_name);
if (namlen > 5)
{
int nameOffset = namlen - 5;
if (StrEquals(file->d_name + nameOffset, ".json"))
{
file->d_name[nameOffset] = '\0';
StringArrayAppend(ret, file->d_name);
}
}
}
closedir(files);
Free(path);
pthread_mutex_unlock(&d->lock);
return ret;
}
static bool
FlatExists(Db *d, Array *dir)
{
FlatDb *db = (FlatDb *) d;
char *path;
bool ret;
if (!d || !dir)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
path = DbFileName(db, dir);
ret = UtilLastModified(path) != 0;
Free(path);
pthread_mutex_unlock(&d->lock);
return ret;
}
Db *
DbOpen(char *dir, size_t cache)
{
FlatDb *db;
if (!dir)
{
return NULL;
}
db = Malloc(sizeof(*db));
DbInit((Db *) db);
db->dir = dir;
db->base.cacheSize = cache;
db->base.lockFunc = FlatLock;
db->base.unlock = FlatUnlock;
db->base.create = FlatCreate;
db->base.delete = FlatDelete;
db->base.exists = FlatExists;
db->base.list = FlatList;
db->base.close = NULL;
return (Db *) db;
}

101
src/Db/Internal.h
View file

@ -1,101 +0,0 @@
/*
* Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation files
* (the "Software"), to deal in the Software without restriction,
* including without limitation the rights to use, copy, modify, merge,
* publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef CYTOPLASM_DB_INTERNAL_H
#define CYTOPLASM_DB_INTERNAL_H
#include <HashMap.h>
#include <Db.h>
#include <pthread.h>
/* TODO: Define the base structures to define a database internally.
* All "implementations" shall have them as a first element, so that
* basic, general functions can work on them properly. */
/* The base structure of a database */
struct Db
{
pthread_mutex_t lock;
size_t cacheSize;
size_t maxCache;
HashMap *cache;
/*
* The cache uses a double linked list (see DbRef
* below) to know which objects are most and least
* recently used. The following diagram helps me
* know what way all the pointers go, because it
* can get very confusing sometimes. For example,
* there's nothing stopping "next" from pointing to
* least recent, and "prev" from pointing to most
* recent, so hopefully this clarifies the pointer
* terminology used when dealing with the linked
* list:
*
* mostRecent leastRecent
* | prev prev | prev
* +---+ ---> +---+ ---> +---+ ---> NULL
* |ref| |ref| |ref|
* NULL <--- +---+ <--- +---+ <--- +---+
* next next next
*/
DbRef *mostRecent;
DbRef *leastRecent;
/* Functions for implementation-specific operations
* (opening a ref, closing a db, removing an entry, ...) */
DbRef * (*lockFunc)(Db *, DbHint, Array *);
DbRef * (*create)(Db *, Array *);
Array * (*list)(Db *, Array *);
bool (*unlock)(Db *, DbRef *);
bool (*delete)(Db *, Array *);
bool (*exists)(Db *, Array *);
void (*close)(Db *);
/* Implementation-specific constructs */
};
struct DbRef
{
HashMap *json;
uint64_t ts;
size_t size;
Array *name;
DbRef *prev;
DbRef *next;
DbHint hint;
/* Implementation-specific constructs */
};
extern void DbInit(Db *);
extern void DbRefInit(Db *, DbRef *);
extern void StringArrayFree(Array *);
extern void StringArrayAppend(Array *, char *);
#endif

590
src/Db/LMDB.c
View file

@ -1,590 +0,0 @@
#include <Memory.h>
#include <Json.h>
#include <Log.h>
#include <Db.h>
#include "Db/Internal.h"
#include <stdint.h>
#include <string.h>
#include <stdlib.h>
#ifdef EDB_LMDB
#include <lmdb.h>
typedef struct LMDB {
Db base; /* The base implementation required to pass */
MDB_env *environ;
MDB_dbi dbi;
} LMDB;
typedef struct LMDBRef {
DbRef base;
/* TODO: LMDB-dependent stuff */
MDB_txn *transaction;
MDB_dbi dbi;
} LMDBRef;
/* Helper functions */
static MDB_val
LMDBTranslateKey(Array *key)
{
MDB_val ret = { 0 };
char *data = NULL;
size_t length = 0;
size_t i;
if (!key || ArraySize(key) > 255)
{
return ret;
}
data = Realloc(data, ++length);
data[0] = ArraySize(key);
/* Now, let's push every item */
for (i = 0; i < ArraySize(key); i++)
{
char *entry = ArrayGet(key, i);
size_t offset = length;
data = Realloc(data, (length += strlen(entry) + 1));
memcpy(data + offset, entry, strlen(entry));
data[length - 1] = '\0';
}
/* We now have every key */
ret.mv_size = length;
ret.mv_data = data;
return ret;
}
static void
LMDBKillKey(MDB_val key)
{
if (!key.mv_data || !key.mv_size)
{
return;
}
Free(key.mv_data);
}
static HashMap *
LMDBDecode(MDB_val val)
{
FILE *fakefile;
Stream *fakestream;
HashMap *ret;
if (!val.mv_data || !val.mv_size)
{
return NULL;
}
fakefile = fmemopen(val.mv_data, val.mv_size, "r");
fakestream = StreamFile(fakefile);
ret = JsonDecode(fakestream);
StreamClose(fakestream);
return ret;
}
static bool
LMDBKeyStartsWith(MDB_val key, MDB_val starts)
{
size_t i;
if (!key.mv_size || !starts.mv_size)
{
return false;
}
if (key.mv_size < starts.mv_size)
{
return false;
}
for (i = 0; i < starts.mv_size; i++)
{
char keyC = ((char *) key.mv_data)[i];
char startC = ((char *) starts.mv_data)[i];
if (keyC != startC)
{
return false;
}
}
return true;
}
static char *
LMDBKeyHead(MDB_val key)
{
char *end;
if (!key.mv_size || !key.mv_data)
{
return NULL;
}
/* -2 because we have a NULL byte in there */
end = ((char *) key.mv_data) + key.mv_size - 1;
if ((void *) end < key.mv_data)
{
/* Uh oh. */
return NULL;
}
while ((void *) (end - 1) >= key.mv_data && *(end - 1))
{
end--;
}
return end;
}
static DbRef *
LMDBLock(Db *d, DbHint hint, Array *k)
{
LMDB *db = (LMDB *) d;
MDB_txn *transaction;
LMDBRef *ret = NULL;
MDB_val key, json_val;
int code, flags;
if (!d || !k)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
key = LMDBTranslateKey(k);
/* Create a transaction, honoring hints. */
/* TODO: Do we want to create a "main" transaction that everyone inherits
* from? */
flags = hint == DB_HINT_READONLY ? MDB_RDONLY : 0;
if ((code = mdb_txn_begin(db->environ, NULL, flags, &transaction)) != 0)
{
/* Very bad! */
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
goto end;
}
json_val.mv_size = 0;
json_val.mv_data = NULL;
code = mdb_get(transaction, db->dbi, &key, &json_val);
if (code == MDB_NOTFOUND)
{
mdb_txn_abort(transaction);
goto end;
}
else if (code != 0)
{
Log(LOG_ERR,
"%s: mdb_get failure: %s",
__func__, mdb_strerror(code)
);
mdb_txn_abort(transaction);
goto end;
}
ret = Malloc(sizeof(*ret));
DbRefInit(d, (DbRef *) ret);
/* TODO: Timestamp */
{
size_t i;
ret->base.name = ArrayCreate();
for (i = 0; i < ArraySize(k); i++)
{
char *ent = ArrayGet(k, i);
StringArrayAppend(ret->base.name, ent);
}
}
ret->base.json = LMDBDecode(json_val);
ret->base.hint = hint;
ret->transaction = NULL;
if (hint == DB_HINT_WRITE)
{
ret->transaction = transaction;
}
else
{
mdb_txn_abort(transaction);
}
end:
if (!ret || hint == DB_HINT_READONLY)
{
pthread_mutex_unlock(&d->lock);
}
LMDBKillKey(key);
return (DbRef *) ret;
}
static bool
LMDBExists(Db *d, Array *k)
{
MDB_val key, empty;
LMDB *db = (LMDB *) d;
MDB_txn *transaction;
int code;
bool ret = false;
if (!d || !k)
{
return false;
}
pthread_mutex_lock(&d->lock);
key = LMDBTranslateKey(k);
/* create a txn */
if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0)
{
/* Very bad! */
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
goto end;
}
ret = mdb_get(transaction, db->dbi, &key, &empty) == 0;
mdb_txn_abort(transaction);
end:
LMDBKillKey(key);
pthread_mutex_unlock(&d->lock);
return ret;
}
static bool
LMDBDelete(Db *d, Array *k)
{
MDB_val key, empty;
LMDB *db = (LMDB *) d;
MDB_txn *transaction;
int code;
bool ret = false;
if (!d || !k)
{
return false;
}
pthread_mutex_lock(&d->lock);
key = LMDBTranslateKey(k);
/* create a txn */
if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0)
{
/* Very bad! */
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
goto end;
}
ret = mdb_del(transaction, db->dbi, &key, &empty) == 0;
mdb_txn_commit(transaction);
end:
LMDBKillKey(key);
pthread_mutex_unlock(&d->lock);
return ret;
}
static bool
LMDBUnlock(Db *d, DbRef *r)
{
LMDBRef *ref = (LMDBRef *) r;
LMDB *db = (LMDB *) d;
FILE *fakestream;
Stream *stream;
MDB_val key, val;
bool ret = true;
if (!d || !r)
{
return false;
}
val.mv_data = NULL;
val.mv_size = 0;
if (ref->transaction && r->hint == DB_HINT_WRITE)
{
key = LMDBTranslateKey(r->name);
fakestream = open_memstream((char **) &val.mv_data, &val.mv_size);
stream = StreamFile(fakestream);
JsonEncode(r->json, stream, JSON_DEFAULT);
StreamFlush(stream);
StreamClose(stream);
ret = mdb_put(ref->transaction, db->dbi, &key, &val, 0) == 0;
mdb_txn_commit(ref->transaction);
LMDBKillKey(key);
}
StringArrayFree(ref->base.name);
JsonFree(ref->base.json);
Free(ref);
if (val.mv_data)
{
free(val.mv_data);
}
if (ret && r->hint == DB_HINT_WRITE)
{
pthread_mutex_unlock(&d->lock);
}
return ret;
}
static DbRef *
LMDBCreate(Db *d, Array *k)
{
LMDB *db = (LMDB *) d;
MDB_txn *transaction;
LMDBRef *ret = NULL;
MDB_val key, empty_json;
int code;
if (!d || !k)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
key = LMDBTranslateKey(k);
/* create a txn */
if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0)
{
/* Very bad! */
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
goto end;
}
empty_json.mv_size = 2;
empty_json.mv_data = "{}";
/* put data in it */
code = mdb_put(transaction, db->dbi, &key, &empty_json, MDB_NOOVERWRITE);
if (code == MDB_KEYEXIST)
{
mdb_txn_abort(transaction);
goto end;
}
else if (code == MDB_MAP_FULL)
{
Log(LOG_ERR, "%s: db is full", __func__);
mdb_txn_abort(transaction);
goto end;
}
else if (code != 0)
{
Log(LOG_ERR, "%s: mdb_put failure: %s", __func__, mdb_strerror(code));
mdb_txn_abort(transaction);
goto end;
}
ret = Malloc(sizeof(*ret));
DbRefInit(d, (DbRef *) ret);
/* TODO: Timestamp */
{
size_t i;
ret->base.name = ArrayCreate();
for (i = 0; i < ArraySize(k); i++)
{
char *ent = ArrayGet(k, i);
StringArrayAppend(ret->base.name, ent);
}
}
ret->base.hint = DB_HINT_WRITE;
ret->base.json = HashMapCreate();
ret->transaction = transaction;
end:
if (!ret)
{
pthread_mutex_unlock(&d->lock);
}
LMDBKillKey(key);
return (DbRef *) ret;
}
static Array *
LMDBList(Db *d, Array *k)
{
LMDB *db = (LMDB *) d;
MDB_val key = { 0 };
MDB_val subKey;
MDB_val val;
Array *ret = NULL;
MDB_cursor *cursor;
MDB_cursor_op op = MDB_SET_RANGE;
MDB_txn *txn;
int code;
if (!d || !k)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
/* Marked as read-only, as we just don't need to write anything
* when listing */
if ((code = mdb_txn_begin(db->environ, NULL, MDB_RDONLY, &txn)) != 0)
{
/* Very bad! */
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
goto end;
}
if ((code = mdb_cursor_open(txn, db->dbi, &cursor)) != 0)
{
Log(LOG_ERR,
"%s: could not get cursor: %s",
__func__, mdb_strerror(code)
);
mdb_txn_abort(txn);
goto end;
}
key = LMDBTranslateKey(k);
/* Small hack to get it to list subitems */
((char *) key.mv_data)[0]++;
ret = ArrayCreate();
subKey = key;
while (mdb_cursor_get(cursor, &subKey, &val, op) == 0)
{
/* This searches by *increasing* order. The problem is that it may
* extend to unwanted points. Since the values are sorted, we can
* just exit if the subkey is incorrect. */
char *head = LMDBKeyHead(subKey);
if (!LMDBKeyStartsWith(subKey, key))
{
break;
}
StringArrayAppend(ret, head);
op = MDB_NEXT;
}
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
end:
LMDBKillKey(key);
pthread_mutex_unlock(&d->lock);
return ret;
}
/* Implementation functions */
static void
LMDBClose(Db *d)
{
LMDB *db = (LMDB *) d;
if (!d)
{
return;
}
mdb_dbi_close(db->environ, db->dbi);
mdb_env_close(db->environ);
}
Db *
DbOpenLMDB(char *dir, size_t size)
{
/* TODO */
MDB_env *env = NULL;
MDB_txn *txn;
MDB_dbi dbi;
int code;
LMDB *db;
if (!dir || !size)
{
return NULL;
}
/* Try initialising LMDB */
if ((code = mdb_env_create(&env)) != 0)
{
Log(LOG_ERR,
"%s: could not create LMDB env: %s",
__func__, mdb_strerror(code)
);
return NULL;
}
if ((code = mdb_env_set_mapsize(env, size) != 0))
{
Log(LOG_ERR,
"%s: could not set mapsize to %lu: %s",
__func__, (unsigned long) size,
mdb_strerror(code)
);
mdb_env_close(env);
return NULL;
}
mdb_env_set_maxdbs(env, 4);
if ((code = mdb_env_open(env, dir, MDB_NOTLS, 0644)) != 0)
{
Log(LOG_ERR,
"%s: could not open LMDB env: %s",
__func__, mdb_strerror(code)
);
mdb_env_close(env);
return NULL;
}
/* Initialise a DBI */
{
if ((code = mdb_txn_begin(env, NULL, 0, &txn)) != 0)
{
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
mdb_env_close(env);
return NULL;
}
if ((code = mdb_dbi_open(txn, "db", MDB_CREATE, &dbi)) != 0)
{
Log(LOG_ERR,
"%s: could not get transaction dbi: %s",
__func__, mdb_strerror(code)
);
mdb_txn_abort(txn);
mdb_env_close(env);
return NULL;
}
mdb_txn_commit(txn);
}
db = Malloc(sizeof(*db));
DbInit((Db *) db);
db->environ = env;
db->dbi = dbi;
db->base.lockFunc = LMDBLock;
db->base.create = LMDBCreate;
db->base.unlock = LMDBUnlock;
db->base.delete = LMDBDelete;
db->base.exists = LMDBExists;
db->base.close = LMDBClose;
db->base.list = LMDBList;
return (Db *) db;
}
#else
Db *
DbOpenLMDB(char *dir, size_t size)
{
/* Unimplemented function */
Log(LOG_ERR,
"LMDB support is not enabled. Please compile with --use-lmdb"
);
(void) size;
(void) dir;
return NULL;
}
#endif

2
src/HashMap.c
View file

@ -184,8 +184,6 @@ HashMapDelete(HashMap * map, const char *key)
if (bucket->hash == hash)
{
bucket->hash = 0;
Free(bucket->key);
bucket->key = NULL;
return bucket->value;
}

207
src/Memory.c
View file

@ -40,24 +40,16 @@
#define MEMORY_HEXDUMP_WIDTH 16
#endif
#define MEMORY_FILE_SIZE 256
struct MemoryInfo
{
uint64_t magic;
size_t size;
char file[MEMORY_FILE_SIZE];
const char *file;
int line;
void *pointer;
MemoryInfo *prev;
MemoryInfo *next;
};
#define MEM_BOUND_TYPE uint32_t
#define MEM_BOUND 0xDEADBEEF
#define MEM_MAGIC 0xDEADBEEFDEADBEEF
#define MEM_BOUND_LOWER(p) *((MEM_BOUND_TYPE *) p)
#define MEM_BOUND_UPPER(p, x) *(((MEM_BOUND_TYPE *) (((uint8_t *) p) + x)) + 1)
@ -67,10 +59,10 @@ static pthread_mutex_t lock;
static void (*hook) (MemoryAction, MemoryInfo *, void *) = MemoryDefaultHook;
static void *hookArgs = NULL;
static MemoryInfo **allocations = NULL;
static size_t allocationsSize = 0;
static size_t allocationsLen = 0;
static MemoryInfo *allocationTail = NULL;
int
MemoryRuntimeInit(void)
{
@ -99,18 +91,70 @@ MemoryRuntimeDestroy(void)
return pthread_mutex_destroy(&lock) == 0;
}
static size_t
MemoryHash(void *p)
{
return (((size_t) p) >> 2 * 7) % allocationsSize;
}
static int
MemoryInsert(MemoryInfo * a)
{
if (allocationTail)
{
allocationTail->next = a;
}
a->next = NULL;
a->prev = allocationTail;
a->magic = MEM_MAGIC;
size_t hash;
allocationTail = a;
if (!allocations)
{
allocationsSize = MEMORY_TABLE_CHUNK;
allocations = calloc(allocationsSize, sizeof(void *));
if (!allocations)
{
return 0;
}
}
/* If the next insertion would cause the table to be at least 3/4
* full, re-allocate and re-hash. */
if ((allocationsLen + 1) >= ((allocationsSize * 3) >> 2))
{
size_t i;
size_t tmpAllocationsSize = allocationsSize;
MemoryInfo **tmpAllocations;
allocationsSize += MEMORY_TABLE_CHUNK;
tmpAllocations = calloc(allocationsSize, sizeof(void *));
if (!tmpAllocations)
{
return 0;
}
for (i = 0; i < tmpAllocationsSize; i++)
{
if (allocations[i])
{
hash = MemoryHash(allocations[i]->pointer);
while (tmpAllocations[hash])
{
hash = (hash + 1) % allocationsSize;
}
tmpAllocations[hash] = allocations[i];
}
}
free(allocations);
allocations = tmpAllocations;
}
hash = MemoryHash(a->pointer);
while (allocations[hash])
{
hash = (hash + 1) % allocationsSize;
}
allocations[hash] = a;
allocationsLen++;
return 1;
@ -119,24 +163,23 @@ MemoryInsert(MemoryInfo * a)
static void
MemoryDelete(MemoryInfo * a)
{
MemoryInfo *aPrev = a->prev;
MemoryInfo *aNext = a->next;
size_t hash = MemoryHash(a->pointer);
size_t count = 0;
if (aPrev)
while (count <= allocationsSize)
{
aPrev->next = aNext;
}
if (aNext)
if (allocations[hash] && allocations[hash] == a)
{
aNext->prev = aPrev;
allocations[hash] = NULL;
allocationsLen--;
return;
}
if (a == allocationTail)
else
{
allocationTail = aPrev;
hash = (hash + 1) % allocationsSize;
count++;
}
}
a->magic = ~MEM_MAGIC;
}
static int
@ -160,31 +203,38 @@ MemoryAllocate(size_t size, const char *file, int line)
void *p;
MemoryInfo *a;
//MemoryIterate(NULL, NULL);
MemoryIterate(NULL, NULL);
pthread_mutex_lock(&lock);
a = malloc(sizeof(MemoryInfo) + MEM_SIZE_ACTUAL(size));
a = malloc(sizeof(MemoryInfo));
if (!a)
{
pthread_mutex_unlock(&lock);
return NULL;
}
p = a + 1;
p = malloc(MEM_SIZE_ACTUAL(size));
if (!p)
{
free(a);
pthread_mutex_unlock(&lock);
return NULL;
}
memset(p, 0, MEM_SIZE_ACTUAL(size));
MEM_BOUND_LOWER(p) = MEM_BOUND;
MEM_BOUND_UPPER(p, size) = MEM_BOUND;
a->size = MEM_SIZE_ACTUAL(size);
strncpy(a->file, file, MEMORY_FILE_SIZE - 1);
a->file = file;
a->line = line;
a->pointer = p;
if (!MemoryInsert(a))
{
free(a);
free(p);
pthread_mutex_unlock(&lock);
return NULL;
}
@ -204,7 +254,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line)
MemoryInfo *a;
void *new = NULL;
//MemoryIterate(NULL, NULL);
MemoryIterate(NULL, NULL);
if (!p)
{
@ -215,17 +265,15 @@ MemoryReallocate(void *p, size_t size, const char *file, int line)
if (a)
{
pthread_mutex_lock(&lock);
MemoryDelete(a);
new = realloc(a, sizeof(MemoryInfo) + MEM_SIZE_ACTUAL(size));
new = realloc(a->pointer, MEM_SIZE_ACTUAL(size));
if (new)
{
a = new;
MemoryDelete(a);
a->size = MEM_SIZE_ACTUAL(size);
strncpy(a->file, file, MEMORY_FILE_SIZE - 1);
a->file = file;
a->line = line;
a->magic = MEM_MAGIC;
a->pointer = a + 1;
a->pointer = new;
MemoryInsert(a);
MEM_BOUND_LOWER(a->pointer) = MEM_BOUND;
@ -245,7 +293,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line)
if (a)
{
a->size = 0;
strncpy(a->file, file, MEMORY_FILE_SIZE - 1);
a->file = file;
a->line = line;
a->pointer = p;
hook(MEMORY_BAD_POINTER, a, hookArgs);
@ -253,7 +301,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line)
}
}
return ((MEM_BOUND_TYPE *) a->pointer) + 1;
return ((MEM_BOUND_TYPE *) new) + 1;
}
void
@ -261,7 +309,7 @@ MemoryFree(void *p, const char *file, int line)
{
MemoryInfo *a;
//MemoryIterate(NULL, NULL);
MemoryIterate(NULL, NULL);
if (!p)
{
@ -274,11 +322,12 @@ MemoryFree(void *p, const char *file, int line)
pthread_mutex_lock(&lock);
if (hook)
{
strncpy(a->file, file, MEMORY_FILE_SIZE - 1);
a->file = file;
a->line = line;
hook(MEMORY_FREE, a, hookArgs);
}
MemoryDelete(a);
free(a->pointer);
free(a);
pthread_mutex_unlock(&lock);
@ -288,7 +337,7 @@ MemoryFree(void *p, const char *file, int line)
a = malloc(sizeof(MemoryInfo));
if (a)
{
strncpy(a->file, file, MEMORY_FILE_SIZE - 1);
a->file = file;
a->line = line;
a->size = 0;
a->pointer = p;
@ -301,15 +350,17 @@ MemoryFree(void *p, const char *file, int line)
size_t
MemoryAllocated(void)
{
size_t i;
size_t total = 0;
MemoryInfo *cur;
pthread_mutex_lock(&lock);
/* TODO */
for (cur = allocationTail; cur; cur = cur->prev)
for (i = 0; i < allocationsSize; i++)
{
total += MemoryInfoGetSize(cur);
if (allocations[i])
{
total += MemoryInfoGetSize(allocations[i]);
}
}
pthread_mutex_unlock(&lock);
@ -320,40 +371,55 @@ MemoryAllocated(void)
void
MemoryFreeAll(void)
{
MemoryInfo *cur;
MemoryInfo *prev;
size_t i;
pthread_mutex_lock(&lock);
/* TODO */
for (cur = allocationTail; cur; cur = prev)
for (i = 0; i < allocationsSize; i++)
{
prev = cur->prev;
free(cur);
if (allocations[i])
{
free(allocations[i]->pointer);
free(allocations[i]);
}
}
free(allocations);
allocations = NULL;
allocationsSize = 0;
allocationsLen = 0;
pthread_mutex_unlock(&lock);
}
MemoryInfo *
MemoryInfoGet(void *po)
MemoryInfoGet(void *p)
{
void *p = po;
size_t hash, count;
pthread_mutex_lock(&lock);
p = ((MEM_BOUND_TYPE *) po) - 1;
p = ((MemoryInfo *) p) - 1;
p = ((MEM_BOUND_TYPE *) p) - 1;
hash = MemoryHash(p);
if (((MemoryInfo *)p)->magic != MEM_MAGIC)
count = 0;
while (count <= allocationsSize)
{
p = NULL;
if (!allocations[hash] || allocations[hash]->pointer != p)
{
hash = (hash + 1) % allocationsSize;
count++;
}
else
{
MemoryCheck(allocations[hash]);
pthread_mutex_unlock(&lock);
return allocations[hash];
}
}
pthread_mutex_unlock(&lock);
return p;
return NULL;
}
size_t
@ -401,18 +467,21 @@ MemoryInfoGetPointer(MemoryInfo * a)
}
void
MemoryIterate(void (*iterFunc) (MemoryInfo *, void *), void *args)
MemoryIterate(void (*iterFunc) (MemoryInfo *, void *), void *args)
{
MemoryInfo *cur;
size_t i;
pthread_mutex_lock(&lock);
for (cur = allocationTail; cur; cur = cur->prev)
for (i = 0; i < allocationsSize; i++)
{
MemoryCheck(cur);
if (allocations[i])
{
MemoryCheck(allocations[i]);
if (iterFunc)
{
iterFunc(cur, args);
iterFunc(allocations[i], args);
}
}
}

3
src/Sha/Sha1.c
View file

@ -28,8 +28,7 @@
#include <limits.h>
/* TODO: Verify LibreSSL support later */
#if TLS_IMPL == TLS_OPENSSL
#if (TLS_IMPL == TLS_OPENSSL) || (TLS_IMPL == TLS_LIBRESSL)
#include <openssl/sha.h>

3
src/Sha/Sha256.c
View file

@ -30,8 +30,7 @@
#include <limits.h>
/* TODO: Verify LibreSSL support later */
#if TLS_IMPL == TLS_OPENSSL
#if (TLS_IMPL == TLS_OPENSSL) || (TLS_IMPL == TLS_LIBRESSL)
#include <openssl/sha.h>