Telodendria/src/Db.c

890 lines
17 KiB
C
Raw Normal View History

/*
2022-12-26 15:52:52 +00:00
* Copyright (C) 2022-2023 Jordan Bancino <@jordan:bancino.net>
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation files
* (the "Software"), to deal in the Software without restriction,
* including without limitation the rights to use, copy, modify, merge,
* publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <Db.h>
#include <Memory.h>
#include <Json.h>
2022-11-15 18:20:05 +00:00
#include <Util.h>
#include <Str.h>
#include <Stream.h>
2023-03-01 19:33:25 +00:00
#include <sys/types.h>
#include <dirent.h>
#include <pthread.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
struct Db
{
2022-11-15 18:20:05 +00:00
char *dir;
pthread_mutex_t lock;
2022-11-18 20:42:08 +00:00
size_t cacheSize;
size_t maxCache;
HashMap *cache;
2022-11-18 20:42:08 +00:00
/*
* The cache uses a double linked list (see DbRef
* below) to know which objects are most and least
* recently used. The following diagram helps me
* know what way all the pointers go, because it
* can get very confusing sometimes. For example,
* there's nothing stopping "next" from pointing to
* least recent, and "prev" from pointing to most
* recent, so hopefully this clarifies the pointer
* terminology used when dealing with the linked
* list:
*
* mostRecent leastRecent
* | prev prev | prev
* +---+ ---> +---+ ---> +---+ ---> NULL
* |ref| |ref| |ref|
* NULL <--- +---+ <--- +---+ <--- +---+
* next next next
*/
2022-11-18 20:42:08 +00:00
DbRef *mostRecent;
DbRef *leastRecent;
};
struct DbRef
{
HashMap *json;
pthread_mutex_t lock;
unsigned long ts;
2022-11-15 18:20:05 +00:00
size_t size;
Array *name;
2022-11-18 20:42:08 +00:00
DbRef *prev;
DbRef *next;
int fd;
Stream *stream;
};
2022-12-15 20:29:19 +00:00
static void
2022-12-15 20:31:51 +00:00
StringArrayFree(Array * arr)
2022-12-15 20:29:19 +00:00
{
size_t i;
for (i = 0; i < ArraySize(arr); i++)
{
Free(ArrayGet(arr, i));
}
ArrayFree(arr);
}
2022-11-12 18:26:31 +00:00
static ssize_t DbComputeSize(HashMap *);
static ssize_t
DbComputeSizeOfValue(JsonValue * val)
{
MemoryInfo *a;
ssize_t total = 0;
size_t i;
union
{
char *str;
Array *arr;
} u;
if (!val)
{
return -1;
}
a = MemoryInfoGet(val);
if (a)
{
total += MemoryInfoGetSize(a);
}
switch (JsonValueType(val))
{
case JSON_OBJECT:
total += DbComputeSize(JsonValueAsObject(val));
break;
case JSON_ARRAY:
u.arr = JsonValueAsArray(val);
a = MemoryInfoGet(u.arr);
if (a)
{
total += MemoryInfoGetSize(a);
}
for (i = 0; i < ArraySize(u.arr); i++)
{
total += DbComputeSizeOfValue(ArrayGet(u.arr, i));
}
break;
case JSON_STRING:
u.str = JsonValueAsString(val);
a = MemoryInfoGet(u.str);
if (a)
{
total += MemoryInfoGetSize(a);
}
break;
case JSON_NULL:
case JSON_INTEGER:
case JSON_FLOAT:
case JSON_BOOLEAN:
default:
/* These don't use any extra heap space */
break;
}
return total;
}
static ssize_t
DbComputeSize(HashMap * json)
{
char *key;
JsonValue *val;
MemoryInfo *a;
size_t total;
if (!json)
{
return -1;
}
total = 0;
a = MemoryInfoGet(json);
if (a)
{
total += MemoryInfoGetSize(a);
}
while (HashMapIterate(json, &key, (void **) &val))
{
a = MemoryInfoGet(key);
if (a)
{
total += MemoryInfoGetSize(a);
}
2022-11-12 18:26:31 +00:00
total += DbComputeSizeOfValue(val);
}
return total;
}
2022-11-15 18:20:05 +00:00
static char *
DbHashKey(Array * args)
2022-11-15 18:20:05 +00:00
{
size_t i;
char *str = NULL;
for (i = 0; i < ArraySize(args); i++)
{
char *tmp = StrConcat(2, str, ArrayGet(args, i));
Free(str);
str = tmp;
}
return str;
2022-11-15 18:20:05 +00:00
}
static char *
2023-03-01 19:33:25 +00:00
DbDirName(Db * db, Array * args, size_t strip)
2022-11-15 18:20:05 +00:00
{
size_t i;
char *str = StrConcat(2, db->dir, "/");
2023-03-01 19:33:25 +00:00
for (i = 0; i < ArraySize(args) - strip; i++)
{
char *tmp;
2022-11-15 18:20:05 +00:00
tmp = StrConcat(3, str, ArrayGet(args, i), "/");
2022-11-15 18:20:05 +00:00
Free(str);
2022-11-15 18:20:05 +00:00
str = tmp;
}
return str;
}
static char *
DbFileName(Db * db, Array * args)
{
size_t i;
char *str = StrConcat(2, db->dir, "/");
for (i = 0; i < ArraySize(args); i++)
{
char *tmp;
char *arg = StrDuplicate(ArrayGet(args, i));
2022-12-15 20:29:19 +00:00
size_t j = 0;
/* Sanitize name to prevent directory traversal attacks */
2022-12-15 20:29:19 +00:00
while (arg[j])
{
2022-12-15 20:29:19 +00:00
switch (arg[j])
{
case '/':
2022-12-15 20:29:19 +00:00
arg[j] = '_';
break;
case '.':
2022-12-15 20:29:19 +00:00
arg[j] = '-';
break;
default:
break;
}
2022-12-15 20:29:19 +00:00
j++;
}
tmp = StrConcat(3, str, arg,
(i < ArraySize(args) - 1) ? "/" : ".json");
Free(arg);
Free(str);
str = tmp;
}
return str;
2022-11-15 18:20:05 +00:00
}
2022-11-18 20:42:08 +00:00
static void
DbCacheEvict(Db * db)
{
DbRef *ref = db->leastRecent;
DbRef *tmp;
2022-11-18 20:42:08 +00:00
while (ref && db->cacheSize > db->maxCache)
{
char *hash = DbHashKey(ref->name);
2022-11-18 20:42:08 +00:00
if (pthread_mutex_trylock(&ref->lock) != 0)
{
/* This ref is locked by another thread, don't evict it. */
ref = ref->next;
continue;
}
2022-11-18 20:42:08 +00:00
JsonFree(ref->json);
pthread_mutex_unlock(&ref->lock);
2022-11-18 20:42:08 +00:00
pthread_mutex_destroy(&ref->lock);
hash = DbHashKey(ref->name);
2022-11-18 20:42:08 +00:00
HashMapDelete(db->cache, hash);
Free(hash);
2022-12-15 20:29:19 +00:00
StringArrayFree(ref->name);
2022-11-18 20:42:08 +00:00
db->cacheSize -= ref->size;
ref->next->prev = ref->prev;
if (!ref->prev)
{
db->leastRecent = ref->next;
}
else
{
ref->prev->next = ref->next;
}
tmp = ref->next;
2022-11-18 20:42:08 +00:00
Free(ref);
ref = tmp;
2022-11-18 20:42:08 +00:00
}
}
Db *
2022-11-15 18:20:05 +00:00
DbOpen(char *dir, size_t cache)
{
Db *db;
if (!dir)
{
return NULL;
}
db = Malloc(sizeof(Db));
if (!db)
{
return NULL;
}
db->dir = dir;
db->maxCache = cache;
pthread_mutex_init(&db->lock, NULL);
if (db->maxCache)
2022-11-15 18:20:05 +00:00
{
db->cache = HashMapCreate();
if (!db->cache)
{
return NULL;
}
db->mostRecent = NULL;
db->leastRecent = NULL;
}
else
{
db->cache = NULL;
}
2022-11-18 20:42:08 +00:00
return db;
}
void
DbClose(Db * db)
{
2022-11-15 18:20:05 +00:00
char *key;
DbRef *val;
if (!db)
{
return;
}
pthread_mutex_destroy(&db->lock);
2022-11-15 18:20:05 +00:00
while (HashMapIterate(db->cache, &key, (void **) &val))
2022-11-15 18:20:05 +00:00
{
JsonFree(val->json);
2022-12-15 20:29:19 +00:00
StringArrayFree(val->name);
pthread_mutex_destroy(&val->lock);
Free(val);
2022-11-15 18:20:05 +00:00
}
HashMapFree(db->cache);
Free(db);
}
static DbRef *
DbLockFromArr(Db * db, Array * args)
{
2022-11-15 18:20:05 +00:00
char *file;
char *hash;
DbRef *ref;
struct flock lock;
2022-11-15 18:20:05 +00:00
int fd;
Stream *stream;
if (!db || !args)
{
return NULL;
}
2022-11-15 18:20:05 +00:00
ref = NULL;
hash = NULL;
pthread_mutex_lock(&db->lock);
2022-11-15 18:20:05 +00:00
/* Check if the item is in the cache */
hash = DbHashKey(args);
ref = HashMapGet(db->cache, hash);
file = DbFileName(db, args);
fd = open(file, O_RDWR);
if (fd == -1)
{
if (ref)
{
pthread_mutex_lock(&ref->lock);
HashMapDelete(db->cache, hash);
JsonFree(ref->json);
StringArrayFree(ref->name);
db->cacheSize -= ref->size;
if (ref->next)
{
ref->next->prev = ref->prev;
}
else
{
db->mostRecent = ref->prev;
}
if (ref->prev)
{
ref->prev->next = ref->next;
}
else
{
db->leastRecent = ref->next;
}
pthread_mutex_unlock(&ref->lock);
pthread_mutex_destroy(&ref->lock);
Free(ref);
}
ref = NULL;
goto finish;
}
stream = StreamFd(fd);
lock.l_start = 0;
lock.l_len = 0;
lock.l_type = F_WRLCK;
lock.l_whence = SEEK_SET;
/* Lock the file on the disk */
if (fcntl(fd, F_SETLK, &lock) < 0)
{
StreamClose(stream);
ref = NULL;
goto finish;
}
2022-11-15 18:20:05 +00:00
if (ref) /* In cache */
{
unsigned long diskTs = UtilLastModified(file);
pthread_mutex_lock(&ref->lock);
2022-11-15 18:20:05 +00:00
if (diskTs > ref->ts)
{
/* File was modified on disk since it was cached */
HashMap *json = JsonDecode(ref->stream);
2022-11-15 18:20:05 +00:00
if (!json)
{
pthread_mutex_unlock(&ref->lock);
StreamClose(ref->stream);
2022-11-15 18:20:05 +00:00
ref = NULL;
goto finish;
}
JsonFree(ref->json);
ref->json = json;
ref->ts = diskTs;
ref->size = DbComputeSize(ref->json);
}
2022-11-18 20:42:08 +00:00
/* Float this ref to mostRecent */
if (ref->next)
{
ref->next->prev = ref->prev;
2022-11-18 20:42:08 +00:00
if (!ref->prev)
{
db->leastRecent = ref->next;
2022-11-18 20:42:08 +00:00
}
2022-11-23 14:56:31 +00:00
else
{
ref->prev->next = ref->next;
}
ref->prev = db->mostRecent;
ref->next = NULL;
db->mostRecent = ref;
2022-11-15 18:20:05 +00:00
}
/* The file on disk may be larger than what we have in memory,
* which may require items in cache to be evicted. */
DbCacheEvict(db);
2022-11-15 18:20:05 +00:00
}
else
{
2022-11-23 14:56:31 +00:00
Array *name = ArrayCreate();
size_t i;
2022-11-15 18:20:05 +00:00
/* Not in cache; load from disk */
ref = Malloc(sizeof(DbRef));
if (!ref)
{
StreamClose(stream);
2022-11-15 18:20:05 +00:00
goto finish;
}
ref->json = JsonDecode(stream);
2022-11-15 18:20:05 +00:00
if (!ref->json)
{
Free(ref);
StreamClose(stream);
2022-11-15 18:20:05 +00:00
ref = NULL;
goto finish;
}
ref->fd = fd;
ref->stream = stream;
2022-11-15 18:20:05 +00:00
pthread_mutex_init(&ref->lock, NULL);
pthread_mutex_lock(&ref->lock);
2022-11-23 14:56:31 +00:00
for (i = 0; i < ArraySize(args); i++)
{
ArrayAdd(name, StrDuplicate(ArrayGet(args, i)));
2022-11-23 14:56:31 +00:00
}
ref->name = name;
2022-11-15 18:20:05 +00:00
if (db->cache)
{
ref->ts = UtilServerTs();
ref->size = DbComputeSize(ref->json);
HashMapSet(db->cache, hash, ref);
db->cacheSize += ref->size;
2022-11-15 18:20:05 +00:00
ref->next = NULL;
ref->prev = db->mostRecent;
db->mostRecent = ref;
2022-11-18 20:42:08 +00:00
/* Adding this item to the cache may case it to grow too
* large, requiring some items to be evicted */
DbCacheEvict(db);
}
2022-11-15 18:20:05 +00:00
}
finish:
pthread_mutex_unlock(&db->lock);
2022-11-15 18:20:05 +00:00
Free(file);
Free(hash);
2022-11-18 19:36:15 +00:00
2022-11-15 18:20:05 +00:00
return ref;
}
DbRef *
DbCreate(Db * db, size_t nArgs,...)
{
Stream *fp;
char *file;
char *dir;
va_list ap;
Array *args;
2022-12-15 20:29:19 +00:00
DbRef *ret;
if (!db)
{
return NULL;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return NULL;
}
file = DbFileName(db, args);
if (UtilLastModified(file))
{
Free(file);
ArrayFree(args);
return NULL;
}
2023-03-01 19:33:25 +00:00
dir = DbDirName(db, args, 1);
if (UtilMkdir(dir, 0750) < 0)
{
Free(file);
ArrayFree(args);
Free(dir);
return NULL;
}
2022-11-23 14:56:31 +00:00
Free(dir);
fp = StreamOpen(file, "w");
Free(file);
if (!fp)
{
ArrayFree(args);
return NULL;
}
StreamPuts(fp, "{}");
StreamClose(fp);
2022-12-15 20:29:19 +00:00
ret = DbLockFromArr(db, args);
ArrayFree(args);
return ret;
}
2022-12-15 02:39:58 +00:00
int
DbDelete(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
char *file;
char *hash;
int ret = 1;
DbRef *ref;
if (!db)
{
return 0;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
pthread_mutex_lock(&db->lock);
hash = DbHashKey(args);
file = DbFileName(db, args);
ref = HashMapGet(db->cache, hash);
if (ref)
{
pthread_mutex_lock(&ref->lock);
HashMapDelete(db->cache, hash);
JsonFree(ref->json);
2022-12-15 20:29:19 +00:00
StringArrayFree(ref->name);
2022-12-15 02:39:58 +00:00
db->cacheSize -= ref->size;
if (ref->next)
{
ref->next->prev = ref->prev;
}
else
{
db->mostRecent = ref->prev;
}
if (ref->prev)
{
ref->prev->next = ref->next;
}
else
{
db->leastRecent = ref->next;
}
pthread_mutex_unlock(&ref->lock);
pthread_mutex_destroy(&ref->lock);
Free(ref);
}
Free(hash);
if (UtilLastModified(file))
{
ret = remove(file) == 0;
}
pthread_mutex_unlock(&db->lock);
2022-12-28 15:44:21 +00:00
ArrayFree(args);
Free(file);
2022-12-15 02:39:58 +00:00
return ret;
}
DbRef *
DbLock(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
DbRef *ret;
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return NULL;
}
ret = DbLockFromArr(db, args);
ArrayFree(args);
return ret;
}
int
DbUnlock(Db * db, DbRef * ref)
{
if (!db || !ref)
{
return 0;
}
pthread_mutex_lock(&db->lock);
lseek(ref->fd, 0L, SEEK_SET);
if (ftruncate(ref->fd, 0) < 0)
2022-11-23 14:56:31 +00:00
{
pthread_mutex_unlock(&db->lock);
return 0;
}
JsonEncode(ref->json, ref->stream, JSON_DEFAULT);
2022-11-15 18:20:05 +00:00
StreamClose(ref->stream);
2022-11-15 18:20:05 +00:00
if (db->cache)
{
db->cacheSize -= ref->size;
ref->size = DbComputeSize(ref->json);
db->cacheSize += ref->size;
/* If this ref has grown significantly since we last computed
* its size, it may have filled the cache and require some
* items to be evicted. */
DbCacheEvict(db);
pthread_mutex_unlock(&ref->lock);
}
else
{
JsonFree(ref->json);
StringArrayFree(ref->name);
2022-11-18 20:42:08 +00:00
pthread_mutex_unlock(&ref->lock);
pthread_mutex_destroy(&ref->lock);
Free(ref);
}
2022-11-18 19:36:15 +00:00
pthread_mutex_unlock(&db->lock);
return 1;
}
2023-01-07 15:51:56 +00:00
int
DbExists(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
char *file;
int ret;
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return 0;
}
/*
* Though it's not explicitly required, we lock the
* database before checking that an object exists to
* prevent any potential race conditions.
*/
pthread_mutex_lock(&db->lock);
2023-01-07 15:51:56 +00:00
file = DbFileName(db, args);
2023-01-07 15:51:56 +00:00
ret = UtilLastModified(file);
pthread_mutex_unlock(&db->lock);
2023-01-07 15:51:56 +00:00
Free(file);
ArrayFree(args);
return ret;
}
Array *
2023-03-01 19:03:42 +00:00
DbList(Db * db, size_t nArgs,...)
{
Array *result;
2023-03-01 19:33:25 +00:00
Array *path;
DIR *files;
struct dirent *file;
char *dir;
2023-03-01 19:33:25 +00:00
va_list ap;
if (!db || !nArgs)
{
return NULL;
}
result = ArrayCreate();
if (!result)
{
return NULL;
}
2023-03-01 19:33:25 +00:00
va_start(ap, nArgs);
path = ArrayFromVarArgs(nArgs, ap);
dir = DbDirName(db, path, 0);
files = opendir(dir);
if (!files)
{
ArrayFree(path);
2023-03-03 03:11:49 +00:00
ArrayFree(result);
2023-03-01 19:33:25 +00:00
Free(dir);
return NULL;
}
while ((file = readdir(files)))
{
2023-03-01 21:39:22 +00:00
size_t namlen = strlen(file->d_name);
2023-03-02 22:06:33 +00:00
2023-03-02 02:53:43 +00:00
if (namlen > 5)
2023-03-01 19:33:25 +00:00
{
2023-03-01 21:39:22 +00:00
int nameOffset = namlen - 5;
2023-03-01 19:33:25 +00:00
if (strcmp(file->d_name + nameOffset, ".json") == 0)
{
file->d_name[nameOffset] = '\0';
ArrayAdd(result, StrDuplicate(file->d_name));
}
}
}
closedir(files);
ArrayFree(path);
Free(dir);
return result;
}
2023-03-01 19:33:25 +00:00
void
DbListFree(Array * arr)
2023-03-01 19:33:25 +00:00
{
StringArrayFree(arr);
}
HashMap *
DbJson(DbRef * ref)
{
return ref ? ref->json : NULL;
}