Telodendria/Cytoplasm
Telodendria/Cytoplasm
3
3
Fork 5
Code Issues 17 Pull requests 2 Projects Releases 2 Packages Activity Actions

Compare commits

base: Telodendria:master
Branches Tags
Telodendria:master
Levitating:log
Levitating:remove-install
Levitating:master
Telodendria:v0.4.1
Telodendria:v0.4.0
Levitating:v0.4.1
Levitating:v0.4.0
..
compare: Levitating:master
Branches Tags
Levitating:log
Levitating:remove-install
Levitating:master
Telodendria:master
Levitating:v0.4.1
Levitating:v0.4.0
Telodendria:v0.4.1
Telodendria:v0.4.0

No commits in common. "master" and "master" have entirely different histories.
master ... master

22 changed files with 1179 additions and 2326 deletions
Show stats Expand all files Collapse all files

18
.forgejo/workflows/compile.yaml
View file

@ -1,18 +0,0 @@
name: Compile Cytoplasm
run-name: Compile Cytoplasm on ${{ forgejo.actor }}
on: [push, pull_request]
jobs:
"Compile Cytoplasm":
strategy:
matrix:
os: [alpine]
arch: [aarch64]
runs-on: ["${{ matrix.os }}", "${{ matrix.arch }}"]
steps:
- name: Check out repository
uses: actions/checkout@v4
- name: Configure Cytoplasm
run: ./configure
- name: Build Cytoplasm
run: make

25
.gitea/workflows/compile.yaml Normal file
View file

@ -0,0 +1,25 @@
name: Compile Cytoplasm
run-name: Compile Cytoplasm on ${{ gitea.actor }}
on: [push]
jobs:
"Compile Cytoplasm":
strategy:
matrix:
os: [debian-v12.4, alpine-v3.19, openbsd-v7.4, freebsd-v14.0, netbsd-v9.3]
arch: [x86, x86_64]
exclude:
# 32-bit OpenBSD does not behave well in QEMU. Even when using
# QEMU to emulate i386, it utilizes 100% of its CPU core and is
# still extremely sluggish. Thus, we don't have a working 32-bit
# OpenBSD runner, so exclude it from the matrix configuration.
- os: openbsd-v7.4
arch: x86
runs-on: ["${{ matrix.os }}", "${{ matrix.arch }}"]
steps:
- name: Check out repository
uses: actions/checkout@v3
- name: Configure Cytoplasm
run: ./configure
- name: Build Cytoplasm
run: make

2
CHANGELOG.md
View file

@ -21,8 +21,6 @@ are not even initialized to a default value.
- Make `HttpRouter` decode path parts before matching them on regular expressions.
- Fixed a bug in `ArraySort()` that would crash programs if the passed array has no
elements.
- Adds `Db[OP]Args` functions that are equivalent to their `Db[OP]` counter parts, but
uses an array of string instead of variadic arguments.
## v0.4.0

29
configure vendored
View file

@ -15,7 +15,7 @@ TOOLS="tools"
# Default compiler flags. These must be supported by all POSIX C compilers.
# "Fancy" compilers that have additional options must be detected and set below.
CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE} -I${SRC}";
CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE}"
LIBS="-lm -lpthread"
# Default args for all platforms.
@ -24,10 +24,10 @@ SCRIPT_ARGS="--prefix=/usr/local --lib-name=Cytoplasm"
# Set SSL flags depending on the platform.
case "$(uname)" in
OpenBSD)
SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl --disable-lmdb"
SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl"
;;
*)
SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl --disable-lmdb"
SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl"
;;
esac
@ -37,7 +37,7 @@ case "$(uname)" in
# These systems typically use GCC.
SCRIPT_ARGS="${SCRIPT_ARGS} --cc=gcc"
;;
OpenBSD|FreeBSD|Darwin)
OpenBSD|FreeBSD)
# These systems typically use Clang.
SCRIPT_ARGS="${SCRIPT_ARGS} --cc=clang"
;;
@ -80,14 +80,6 @@ for arg in $SCRIPT_ARGS; do
TLS_IMPL=""
TLS_LIBS=""
;;
--with-lmdb)
EDB_IMPL="EDB_LMDB"
EDB_LIBS="-llmdb"
;;
--disable-lmdb)
EDB_IMPL=""
EDB_LIBS=""
;;
--prefix=*)
PREFIX=$(echo "$arg" | cut -d '=' -f 2-)
;;
@ -112,11 +104,6 @@ if [ -n "$TLS_IMPL" ]; then
LIBS="${LIBS} ${TLS_LIBS}"
fi
if [ -n "$EDB_IMPL" ]; then
CFLAGS="${CFLAGS} -D${EDB_IMPL}"
LIBS="${LIBS} ${EDB_LIBS}"
fi
CFLAGS="${CFLAGS} '-DLIB_NAME=\"${LIB_NAME}\"' ${DEBUG}"
LDFLAGS="${LIBS} ${LDFLAGS}"
@ -153,7 +140,7 @@ print_obj() {
get_deps() {
src="$1"
${CC} -I${SRC} -I${INCLUDE} -E "$src" \
${CC} -I${INCLUDE} -E "$src" \
| grep '^#' \
| awk '{print $3}' \
| cut -d '"' -f 2 \
@ -269,10 +256,8 @@ ${TAB}done
${LIB_NAME}: ${OUT}/lib/lib${LIB_NAME}.a ${OUT}/lib/lib${LIB_NAME}.so
install: ${LIB_NAME}
${TAB}mkdir -p \$(PREFIX)/${OUT}/lib
${TAB}mkdir -p \$(PREFIX)/lib
${TAB}cp ${OUT}/lib/lib${LIB_NAME}.a \$(PREFIX)/lib/lib${LIB_NAME}.a
${TAB}cp ${OUT}/lib/lib${LIB_NAME}.so \$(PREFIX)/lib/lib${LIB_NAME}.so
${TAB}install -D ${OUT}/lib/lib${LIB_NAME}.a \$(PREFIX)/lib/lib${LIB_NAME}.a
${TAB}install -D ${OUT}/lib/lib${LIB_NAME}.so \$(PREFIX)/lib/lib${LIB_NAME}.so
$(collect ${INCLUDE}/ '' '' \$\(PREFIX\)/include/${LIB_NAME}/ install_out)
$(collect ${INCLUDE}/ .h .3 \$\(PREFIX\)/man/man3/${LIB_NAME}- install_man)
$(collect ${TOOLS}/ '.c' '' \$\(PREFIX\)/bin/ install_tool)

71
include/Cytoplasm/Db.h
View file

@ -48,17 +48,6 @@
*/
typedef struct Db Db;
/**
* Some "hints" for the database backend for operations.
* Hints are a way for the program to declare what to except of it
* (and the program MUST adhere to these hints, but the backend
* MAY adhere).
*/
typedef enum DbHint {
DB_HINT_READONLY, /* The database reference is treated as read-only */
DB_HINT_WRITE /* The database reference is treated as read/write */
} DbHint;
/**
* When an object is locked, a reference is returned. This reference
* is owned by the current thread, and the database is inaccessible to
@ -79,15 +68,6 @@ typedef struct DbRef DbRef;
*/
extern Db * DbOpen(char *, size_t);
/**
* Open a LMDB data directory. This function is similar to
* .Fn DbOpen ,
* but works with a LMDB-based backend, with the second argument
* being the maximum filesize. This function fails with NULL if ever
* called without LMDB enabled at compiletime.
*/
extern Db * DbOpenLMDB(char *, size_t);
/**
* Close the database. This function will flush anything in the cache
* to the disk, and then close the data directory. It assumes that
@ -119,13 +99,6 @@ extern void DbMaxCacheSet(Db *, size_t);
*/
extern DbRef * DbCreate(Db *, size_t,...);
/**
* Behaves like
* .Fn DbCreate ,
* but with an array of strings instead of variadic arguments.
*/
extern DbRef * DbCreateArgs(Db *, Array *);
/**
* Lock an existing object in the database. This function will fail
* if the object does not exist. It takes a variable number of C
@ -136,29 +109,6 @@ extern DbRef * DbCreateArgs(Db *, Array *);
*/
extern DbRef * DbLock(Db *, size_t,...);
/**
* Behaves like
* .Fn DbLock ,
* but with an array of strings instead of variadic arguments.
*/
extern DbRef * DbLockArgs(Db *, Array *);
/**
* Behaves like
* .Fn DbLock ,
* but with hints on the reference itself, as
* .Fn DbLock
* itself is read/write.
*/
extern DbRef * DbLockIntent(Db *, DbHint, size_t,...);
/**
* Behaves like
* .Fn DbLockIntent ,
* but with an array of strings instead of variadic arguments.
*/
extern DbRef * DbLockIntentArgs(Db *, DbHint, Array *);
/**
* Immediately and permanently remove an object from the database.
* This function assumes the object is not locked, otherwise undefined
@ -166,13 +116,6 @@ extern DbRef * DbLockIntentArgs(Db *, DbHint, Array *);
*/
extern bool DbDelete(Db *, size_t,...);
/**
* Behaves like
* .Fn DbDelete ,
* but with an array of strings instead of variadic arguments.
*/
extern bool DbDeleteArgs(Db *, Array *);
/**
* Unlock an object and return it back to the database. This function
* immediately syncs the object to the filesystem. The cache is a
@ -190,13 +133,6 @@ extern bool DbUnlock(Db *, DbRef *);
*/
extern bool DbExists(Db *, size_t,...);
/**
* Behaves like
* .Fn DbExists ,
* but with an array of strings instead of variadic arguments.
*/
extern bool DbExistsArgs(Db *, Array *);
/**
* List all of the objects at a given path. Unlike the other varargs
* functions, this one does not take a path to a specific object; it
@ -209,13 +145,6 @@ extern bool DbExistsArgs(Db *, Array *);
*/
extern Array * DbList(Db *, size_t,...);
/**
* Behaves like
* .Fn DbList ,
* but with an array of strings instead of variadic arguments.
*/
extern Array * DbListArgs(Db *, Array *);
/**
* Free the list returned by
* .Fn DbListFree .

6
include/Cytoplasm/Json.h
View file

@ -297,12 +297,6 @@ extern size_t JsonEncode(HashMap *, Stream *, int);
*/
extern HashMap * JsonDecode(Stream *);
/**
* Decodes a JSON value from thr current input strram and parse it
* into a JsonValue.
*/
extern JsonValue * JsonValueDecode(Stream *);
/**
* A convenience function that allows the caller to retrieve and
* arbitrarily deep keys within a JSON object. It takes a root JSON

69
include/Cytoplasm/Platform.h
View file

@ -1,69 +0,0 @@
/*
* Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation files
* (the "Software"), to deal in the Software without restriction,
* including without limitation the rights to use, copy, modify, merge,
* publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef CYTOPLASM_PLATFORM_H
#define CYTOPLASM_PLATFORM_H
/***
* @Nm Platform
* @Nd A simple macro header that determines what platform the application
* is being built for.
* @Dd September 21, 2024
*/
#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
#define PLATFORM_WINDOWS
#ifdef _WIN64
#define PLATFORM_WIN64
#else
#define PLATFORM_WIN32
#endif
#elif __APPLE__
#define PLATFORM_DARWIN
#include <TargetConditionals.h>
#if TARGET_IPHONE_SIMULATOR
#define PLATFORM_IPHONE
#elif TARGET_OS_MACCATALYST
#define PLATFORM_CATALYST
#elif TARGET_OS_IPHONE
#define PLATFORM_IPHONE
#elif TARGET_OS_MAC
#define PLATFORM_MAC
#else
# error "Unknown Apple platform"
#endif
#elif __ANDROID__
#define PLATFORM_ANDROID
#elif __linux__
#define PLATFORM_LINUX
#elif __unix__
#define PLATFORM_UNIX
#elif defined(_POSIX_VERSION)
#define PLATFORM_POSIX
#else
# error "Unknown compiler"
#endif
#endif /* CYTOPLASM_PLATFORM_H */

26
include/Cytoplasm/Sha.h
View file

@ -41,16 +41,6 @@
* due to the lack of a 64-bit integer type, so that hash
* function has been omitted.
*/
#include <stddef.h>
/**
* This is an enum to be used to identify the type of SHA used.
*/
typedef enum HashType
{
HASH_SHA1,
HASH_SHA256
} HashType;
/**
* This function takes a pointer to a NULL-terminated C string, and
@ -74,20 +64,6 @@ extern unsigned char * Sha256(char *);
*/
extern unsigned char * Sha1(char *);
/**
* This function behaves just like
* .Fn Sha256 ,
* except that it allows for a generic byte array, instead of a string.
*/
extern unsigned char * Sha256Raw(unsigned char *, size_t);
/**
* This function behaves just like
* .Fn Sha1 ,
* except that it allows for a generic byte array, instead of a string.
*/
extern unsigned char * Sha1Raw(unsigned char *, size_t);
/**
* Convert a SHA byte buffer into a hex string. These hex strings
* are typically what is transmitted, stored, and compared, however
@ -95,6 +71,6 @@ extern unsigned char * Sha1Raw(unsigned char *, size_t);
* bytes directly, which is why the conversion to a hex string is
* a separate step.
*/
extern char * ShaToHex(unsigned char *, HashType);
extern char * ShaToHex(unsigned char *);
#endif /* CYTOPLASM_SHA_H */

977
src/Db.c Normal file
View file

@ -0,0 +1,977 @@
/*
* Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation files
* (the "Software"), to deal in the Software without restriction,
* including without limitation the rights to use, copy, modify, merge,
* publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <Db.h>
#include <Memory.h>
#include <Json.h>
#include <Util.h>
#include <Str.h>
#include <Stream.h>
#include <Log.h>
#include <sys/types.h>
#include <dirent.h>
#include <pthread.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
struct Db
{
char *dir;
pthread_mutex_t lock;
size_t cacheSize;
size_t maxCache;
HashMap *cache;
/*
* The cache uses a double linked list (see DbRef
* below) to know which objects are most and least
* recently used. The following diagram helps me
* know what way all the pointers go, because it
* can get very confusing sometimes. For example,
* there's nothing stopping "next" from pointing to
* least recent, and "prev" from pointing to most
* recent, so hopefully this clarifies the pointer
* terminology used when dealing with the linked
* list:
*
* mostRecent leastRecent
* | prev prev | prev
* +---+ ---> +---+ ---> +---+ ---> NULL
* |ref| |ref| |ref|
* NULL <--- +---+ <--- +---+ <--- +---+
* next next next
*/
DbRef *mostRecent;
DbRef *leastRecent;
};
struct DbRef
{
HashMap *json;
uint64_t ts;
size_t size;
Array *name;
DbRef *prev;
DbRef *next;
int fd;
Stream *stream;
};
static void
StringArrayFree(Array * arr)
{
size_t i;
for (i = 0; i < ArraySize(arr); i++)
{
Free(ArrayGet(arr, i));
}
ArrayFree(arr);
}
static ssize_t DbComputeSize(HashMap *);
static ssize_t
DbComputeSizeOfValue(JsonValue * val)
{
MemoryInfo *a;
ssize_t total = 0;
size_t i;
union
{
char *str;
Array *arr;
} u;
if (!val)
{
return -1;
}
a = MemoryInfoGet(val);
if (a)
{
total += MemoryInfoGetSize(a);
}
switch (JsonValueType(val))
{
case JSON_OBJECT:
total += DbComputeSize(JsonValueAsObject(val));
break;
case JSON_ARRAY:
u.arr = JsonValueAsArray(val);
a = MemoryInfoGet(u.arr);
if (a)
{
total += MemoryInfoGetSize(a);
}
for (i = 0; i < ArraySize(u.arr); i++)
{
total += DbComputeSizeOfValue(ArrayGet(u.arr, i));
}
break;
case JSON_STRING:
u.str = JsonValueAsString(val);
a = MemoryInfoGet(u.str);
if (a)
{
total += MemoryInfoGetSize(a);
}
break;
case JSON_NULL:
case JSON_INTEGER:
case JSON_FLOAT:
case JSON_BOOLEAN:
default:
/* These don't use any extra heap space */
break;
}
return total;
}
static ssize_t
DbComputeSize(HashMap * json)
{
char *key;
JsonValue *val;
MemoryInfo *a;
size_t total;
if (!json)
{
return -1;
}
total = 0;
a = MemoryInfoGet(json);
if (a)
{
total += MemoryInfoGetSize(a);
}
while (HashMapIterate(json, &key, (void **) &val))
{
a = MemoryInfoGet(key);
if (a)
{
total += MemoryInfoGetSize(a);
}
total += DbComputeSizeOfValue(val);
}
return total;
}
static char *
DbHashKey(Array * args)
{
size_t i;
char *str = NULL;
for (i = 0; i < ArraySize(args); i++)
{
char *tmp = StrConcat(2, str, ArrayGet(args, i));
Free(str);
str = tmp;
}
return str;
}
static char
DbSanitiseChar(char input)
{
switch (input)
{
case '/':
return '_';
case '.':
return '-';
}
return input;
}
static char *
DbDirName(Db * db, Array * args, size_t strip)
{
size_t i, j;
char *str = StrConcat(2, db->dir, "/");
for (i = 0; i < ArraySize(args) - strip; i++)
{
char *tmp;
char *sanitise = StrDuplicate(ArrayGet(args, i));
for (j = 0; j < strlen(sanitise); j++)
{
sanitise[j] = DbSanitiseChar(sanitise[j]);
}
tmp = StrConcat(3, str, sanitise, "/");
Free(str);
Free(sanitise);
str = tmp;
}
return str;
}
static char *
DbFileName(Db * db, Array * args)
{
size_t i;
char *str = StrConcat(2, db->dir, "/");
for (i = 0; i < ArraySize(args); i++)
{
char *tmp;
char *arg = StrDuplicate(ArrayGet(args, i));
size_t j = 0;
/* Sanitize name to prevent directory traversal attacks */
while (arg[j])
{
arg[j] = DbSanitiseChar(arg[j]);
j++;
}
tmp = StrConcat(3, str, arg,
(i < ArraySize(args) - 1) ? "/" : ".json");
Free(arg);
Free(str);
str = tmp;
}
return str;
}
static void
DbCacheEvict(Db * db)
{
DbRef *ref = db->leastRecent;
DbRef *tmp;
while (ref && db->cacheSize > db->maxCache)
{
char *hash;
JsonFree(ref->json);
hash = DbHashKey(ref->name);
HashMapDelete(db->cache, hash);
Free(hash);
StringArrayFree(ref->name);
db->cacheSize -= ref->size;
if (ref->next)
{
ref->next->prev = ref->prev;
}
else
{
db->mostRecent = ref->prev;
}
if (ref->prev)
{
ref->prev->next = ref->next;
}
else
{
db->leastRecent = ref->next;
}
tmp = ref->next;
Free(ref);
ref = tmp;
}
}
Db *
DbOpen(char *dir, size_t cache)
{
Db *db;
pthread_mutexattr_t attr;
if (!dir)
{
return NULL;
}
db = Malloc(sizeof(Db));
if (!db)
{
return NULL;
}
db->dir = dir;
db->maxCache = cache;
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&db->lock, &attr);
pthread_mutexattr_destroy(&attr);
db->mostRecent = NULL;
db->leastRecent = NULL;
db->cacheSize = 0;
if (db->maxCache)
{
db->cache = HashMapCreate();
if (!db->cache)
{
return NULL;
}
}
else
{
db->cache = NULL;
}
return db;
}
void
DbMaxCacheSet(Db * db, size_t cache)
{
if (!db)
{
return;
}
pthread_mutex_lock(&db->lock);
db->maxCache = cache;
if (db->maxCache && !db->cache)
{
db->cache = HashMapCreate();
db->cacheSize = 0;
}
DbCacheEvict(db);
pthread_mutex_unlock(&db->lock);
}
void
DbClose(Db * db)
{
if (!db)
{
return;
}
pthread_mutex_lock(&db->lock);
DbMaxCacheSet(db, 0);
DbCacheEvict(db);
HashMapFree(db->cache);
pthread_mutex_unlock(&db->lock);
pthread_mutex_destroy(&db->lock);
Free(db);
}
static DbRef *
DbLockFromArr(Db * db, Array * args)
{
char *file;
char *hash;
DbRef *ref;
struct flock lock;
int fd;
Stream *stream;
if (!db || !args)
{
return NULL;
}
ref = NULL;
hash = NULL;
pthread_mutex_lock(&db->lock);
/* Check if the item is in the cache */
hash = DbHashKey(args);
ref = HashMapGet(db->cache, hash);
file = DbFileName(db, args);
fd = open(file, O_RDWR);
if (fd == -1)
{
if (ref)
{
HashMapDelete(db->cache, hash);
JsonFree(ref->json);
StringArrayFree(ref->name);
db->cacheSize -= ref->size;
if (ref->next)
{
ref->next->prev = ref->prev;
}
else
{
db->mostRecent = ref->prev;
}
if (ref->prev)
{
ref->prev->next = ref->next;
}
else
{
db->leastRecent = ref->next;
}
if (!db->leastRecent)
{
db->leastRecent = db->mostRecent;
}
Free(ref);
}
ref = NULL;
goto finish;
}
stream = StreamFd(fd);
lock.l_start = 0;
lock.l_len = 0;
lock.l_type = F_WRLCK;
lock.l_whence = SEEK_SET;
/* Lock the file on the disk */
if (fcntl(fd, F_SETLK, &lock) < 0)
{
StreamClose(stream);
ref = NULL;
goto finish;
}
if (ref) /* In cache */
{
uint64_t diskTs = UtilLastModified(file);
ref->fd = fd;
ref->stream = stream;
if (diskTs > ref->ts)
{
/* File was modified on disk since it was cached */
HashMap *json = JsonDecode(ref->stream);
if (!json)
{
StreamClose(ref->stream);
ref = NULL;
goto finish;
}
JsonFree(ref->json);
ref->json = json;
ref->ts = diskTs;
ref->size = DbComputeSize(ref->json);
}
/* Float this ref to mostRecent */
if (ref->next)
{
ref->next->prev = ref->prev;
if (!ref->prev)
{
db->leastRecent = ref->next;
}
else
{
ref->prev->next = ref->next;
}
ref->prev = db->mostRecent;
ref->next = NULL;
if (db->mostRecent)
{
db->mostRecent->next = ref;
}
db->mostRecent = ref;
}
/* If there is no least recent, this is the only thing in the
* cache, so it is also least recent. */
if (!db->leastRecent)
{
db->leastRecent = ref;
}
/* The file on disk may be larger than what we have in memory,
* which may require items in cache to be evicted. */
DbCacheEvict(db);
}
else
{
Array *name;
size_t i;
/* Not in cache; load from disk */
ref = Malloc(sizeof(DbRef));
if (!ref)
{
StreamClose(stream);
goto finish;
}
ref->json = JsonDecode(stream);
if (!ref->json)
{
Free(ref);
StreamClose(stream);
ref = NULL;
goto finish;
}
ref->fd = fd;
ref->stream = stream;
name = ArrayCreate();
for (i = 0; i < ArraySize(args); i++)
{
ArrayAdd(name, StrDuplicate(ArrayGet(args, i)));
}
ref->name = name;
if (db->cache)
{
ref->ts = UtilTsMillis();
ref->size = DbComputeSize(ref->json);
HashMapSet(db->cache, hash, ref);
db->cacheSize += ref->size;
ref->next = NULL;
ref->prev = db->mostRecent;
if (db->mostRecent)
{
db->mostRecent->next = ref;
}
db->mostRecent = ref;
if (!db->leastRecent)
{
db->leastRecent = ref;
}
/* Adding this item to the cache may case it to grow too
* large, requiring some items to be evicted */
DbCacheEvict(db);
}
}
finish:
if (!ref)
{
pthread_mutex_unlock(&db->lock);
}
Free(file);
Free(hash);
return ref;
}
DbRef *
DbCreate(Db * db, size_t nArgs,...)
{
Stream *fp;
char *file;
char *dir;
va_list ap;
Array *args;
DbRef *ret;
if (!db)
{
return NULL;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return NULL;
}
pthread_mutex_lock(&db->lock);
file = DbFileName(db, args);
if (UtilLastModified(file))
{
Free(file);
ArrayFree(args);
pthread_mutex_unlock(&db->lock);
return NULL;
}
dir = DbDirName(db, args, 1);
if (UtilMkdir(dir, 0750) < 0)
{
Free(file);
ArrayFree(args);
Free(dir);
pthread_mutex_unlock(&db->lock);
return NULL;
}
Free(dir);
fp = StreamOpen(file, "w");
Free(file);
if (!fp)
{
ArrayFree(args);
pthread_mutex_unlock(&db->lock);
return NULL;
}
StreamPuts(fp, "{}");
StreamClose(fp);
/* DbLockFromArr() will lock again for us */
pthread_mutex_unlock(&db->lock);
ret = DbLockFromArr(db, args);
ArrayFree(args);
return ret;
}
bool
DbDelete(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
char *file;
char *hash;
bool ret = true;
DbRef *ref;
if (!db)
{
return false;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
pthread_mutex_lock(&db->lock);
hash = DbHashKey(args);
file = DbFileName(db, args);
ref = HashMapGet(db->cache, hash);
if (ref)
{
HashMapDelete(db->cache, hash);
JsonFree(ref->json);
StringArrayFree(ref->name);
db->cacheSize -= ref->size;
if (ref->next)
{
ref->next->prev = ref->prev;
}
else
{
db->mostRecent = ref->prev;
}
if (ref->prev)
{
ref->prev->next = ref->next;
}
else
{
db->leastRecent = ref->next;
}
if (!db->leastRecent)
{
db->leastRecent = db->mostRecent;
}
Free(ref);
}
Free(hash);
if (UtilLastModified(file))
{
ret = (remove(file) == 0);
}
pthread_mutex_unlock(&db->lock);
ArrayFree(args);
Free(file);
return ret;
}
DbRef *
DbLock(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
DbRef *ret;
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return NULL;
}
ret = DbLockFromArr(db, args);
ArrayFree(args);
return ret;
}
bool
DbUnlock(Db * db, DbRef * ref)
{
bool destroy;
if (!db || !ref)
{
return false;
}
lseek(ref->fd, 0L, SEEK_SET);
if (ftruncate(ref->fd, 0) < 0)
{
pthread_mutex_unlock(&db->lock);
Log(LOG_ERR, "Failed to truncate file on disk.");
Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno));
return false;
}
JsonEncode(ref->json, ref->stream, JSON_DEFAULT);
StreamClose(ref->stream);
if (db->cache)
{
char *key = DbHashKey(ref->name);
if (HashMapGet(db->cache, key))
{
db->cacheSize -= ref->size;
ref->size = DbComputeSize(ref->json);
db->cacheSize += ref->size;
/* If this ref has grown significantly since we last
* computed its size, it may have filled the cache and
* require some items to be evicted. */
DbCacheEvict(db);
destroy = false;
}
else
{
destroy = true;
}
Free(key);
}
else
{
destroy = true;
}
if (destroy)
{
JsonFree(ref->json);
StringArrayFree(ref->name);
Free(ref);
}
pthread_mutex_unlock(&db->lock);
return true;
}
bool
DbExists(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
char *file;
bool ret;
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return false;
}
pthread_mutex_lock(&db->lock);
file = DbFileName(db, args);
ret = (UtilLastModified(file) != 0);
pthread_mutex_unlock(&db->lock);
Free(file);
ArrayFree(args);
return ret;
}
Array *
DbList(Db * db, size_t nArgs,...)
{
Array *result;
Array *path;
DIR *files;
struct dirent *file;
char *dir;
va_list ap;
if (!db || !nArgs)
{
return NULL;
}
result = ArrayCreate();
if (!result)
{
return NULL;
}
va_start(ap, nArgs);
path = ArrayFromVarArgs(nArgs, ap);
dir = DbDirName(db, path, 0);
pthread_mutex_lock(&db->lock);
files = opendir(dir);
if (!files)
{
ArrayFree(path);
ArrayFree(result);
Free(dir);
pthread_mutex_unlock(&db->lock);
return NULL;
}
while ((file = readdir(files)))
{
size_t namlen = strlen(file->d_name);
if (namlen > 5)
{
int nameOffset = namlen - 5;
if (StrEquals(file->d_name + nameOffset, ".json"))
{
file->d_name[nameOffset] = '\0';
ArrayAdd(result, StrDuplicate(file->d_name));
}
}
}
closedir(files);
ArrayFree(path);
Free(dir);
pthread_mutex_unlock(&db->lock);
return result;
}
void
DbListFree(Array * arr)
{
StringArrayFree(arr);
}
HashMap *
DbJson(DbRef * ref)
{
return ref ? ref->json : NULL;
}
bool
DbJsonSet(DbRef * ref, HashMap * json)
{
if (!ref || !json)
{
return false;
}
JsonFree(ref->json);
ref->json = JsonDuplicate(json);
return true;
}

562
src/Db/Db.c
View file

@ -1,562 +0,0 @@
/*
* Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation files
* (the "Software"), to deal in the Software without restriction,
* including without limitation the rights to use, copy, modify, merge,
* publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <Db.h>
#include <Memory.h>
#include <Json.h>
#include <Util.h>
#include <Str.h>
#include <Stream.h>
#include <Log.h>
#include <sys/types.h>
#include <dirent.h>
#include <pthread.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include "Db/Internal.h"
void
StringArrayFree(Array * arr)
{
size_t i;
for (i = 0; i < ArraySize(arr); i++)
{
Free(ArrayGet(arr, i));
}
ArrayFree(arr);
}
static ssize_t DbComputeSize(HashMap *);
static ssize_t
DbComputeSizeOfValue(JsonValue * val)
{
MemoryInfo *a;
ssize_t total = 0;
size_t i;
union
{
char *str;
Array *arr;
} u;
if (!val)
{
return -1;
}
a = MemoryInfoGet(val);
if (a)
{
total += MemoryInfoGetSize(a);
}
switch (JsonValueType(val))
{
case JSON_OBJECT:
total += DbComputeSize(JsonValueAsObject(val));
break;
case JSON_ARRAY:
u.arr = JsonValueAsArray(val);
a = MemoryInfoGet(u.arr);
if (a)
{
total += MemoryInfoGetSize(a);
}
for (i = 0; i < ArraySize(u.arr); i++)
{
total += DbComputeSizeOfValue(ArrayGet(u.arr, i));
}
break;
case JSON_STRING:
u.str = JsonValueAsString(val);
a = MemoryInfoGet(u.str);
if (a)
{
total += MemoryInfoGetSize(a);
}
break;
case JSON_NULL:
case JSON_INTEGER:
case JSON_FLOAT:
case JSON_BOOLEAN:
default:
/* These don't use any extra heap space */
break;
}
return total;
}
static ssize_t
DbComputeSize(HashMap * json)
{
char *key;
JsonValue *val;
MemoryInfo *a;
size_t total;
if (!json)
{
return -1;
}
total = 0;
a = MemoryInfoGet(json);
if (a)
{
total += MemoryInfoGetSize(a);
}
while (HashMapIterate(json, &key, (void **) &val))
{
a = MemoryInfoGet(key);
if (a)
{
total += MemoryInfoGetSize(a);
}
total += DbComputeSizeOfValue(val);
}
return total;
}
static char *
DbHashKey(Array * args)
{
size_t i;
char *str = NULL;
for (i = 0; i < ArraySize(args); i++)
{
char *tmp = StrConcat(2, str, ArrayGet(args, i));
Free(str);
str = tmp;
}
return str;
}
static void
DbCacheEvict(Db * db)
{
DbRef *ref = db->leastRecent;
DbRef *tmp;
while (ref && db->cacheSize > db->maxCache)
{
char *hash;
JsonFree(ref->json);
hash = DbHashKey(ref->name);
HashMapDelete(db->cache, hash);
Free(hash);
StringArrayFree(ref->name);
db->cacheSize -= ref->size;
if (ref->next)
{
ref->next->prev = ref->prev;
}
else
{
db->mostRecent = ref->prev;
}
if (ref->prev)
{
ref->prev->next = ref->next;
}
else
{
db->leastRecent = ref->next;
}
tmp = ref->next;
Free(ref);
ref = tmp;
}
}
void
DbMaxCacheSet(Db * db, size_t cache)
{
if (!db)
{
return;
}
pthread_mutex_lock(&db->lock);
db->maxCache = cache;
if (db->maxCache && !db->cache)
{
db->cache = HashMapCreate();
db->cacheSize = 0;
}
DbCacheEvict(db);
pthread_mutex_unlock(&db->lock);
}
void
DbClose(Db * db)
{
if (!db)
{
return;
}
pthread_mutex_lock(&db->lock);
if (db->close)
{
db->close(db);
}
DbMaxCacheSet(db, 0);
DbCacheEvict(db);
HashMapFree(db->cache);
pthread_mutex_unlock(&db->lock);
pthread_mutex_destroy(&db->lock);
Free(db);
}
DbRef *
DbCreateArgs(Db * db, Array *args)
{
if (!args || !db || !db->create)
{
return NULL;
}
return db->create(db, args);
}
DbRef *
DbCreate(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
DbRef *ret;
if (!db || !db->create)
{
return NULL;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return NULL;
}
ret = DbCreateArgs(db, args);
ArrayFree(args);
return ret;
}
bool
DbDeleteArgs(Db * db, Array *args)
{
if (!args || !db || !db->delete)
{
return false;
}
return db->delete(db, args);
}
bool
DbDelete(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
bool ret = true;
if (!db)
{
return false;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
ret = DbDeleteArgs(db, args);
ArrayFree(args);
return ret;
}
DbRef *
DbLockArgs(Db * db, Array *args)
{
if (!args || !db || !db->lockFunc)
{
return NULL;
}
return db->lockFunc(db, DB_HINT_WRITE, args);
}
DbRef *
DbLock(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
DbRef *ret;
if (!db)
{
return NULL;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args || !db->lockFunc)
{
return NULL;
}
ret = DbLockArgs(db, args);
ArrayFree(args);
return ret;
}
DbRef *
DbLockIntentArgs(Db * db, DbHint hint, Array *args)
{
if (!db || !args || !db->lockFunc)
{
return NULL;
}
return db->lockFunc(db, hint, args);
}
DbRef *
DbLockIntent(Db * db, DbHint hint, size_t nArgs,...)
{
va_list ap;
Array *args;
DbRef *ret;
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args || !db->lockFunc)
{
return NULL;
}
ret = DbLockIntentArgs(db, hint, args);
ArrayFree(args);
return ret;
}
bool
DbUnlock(Db * db, DbRef * ref)
{
if (!db || !ref || !db->unlock)
{
return false;
}
return db->unlock(db, ref);
}
bool
DbExistsArgs(Db * db, Array *args)
{
if (!args || !db || !db->exists)
{
return false;
}
return db->exists(db, args);
}
bool
DbExists(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
bool ret;
if (!db || !nArgs || !db->exists)
{
return false;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return false;
}
ret = DbExistsArgs(db, args);
ArrayFree(args);
return ret;
}
Array *
DbListArgs(Db *db, Array *args)
{
if (!db || !args || !db->list)
{
return NULL;
}
return db->list(db, args);
}
Array *
DbList(Db * db, size_t nArgs,...)
{
Array *result;
Array *path;
va_list ap;
if (!db || !nArgs || !db->list)
{
return NULL;
}
va_start(ap, nArgs);
path = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
result = DbListArgs(db, path);
ArrayFree(path);
return result;
}
void
DbListFree(Array * arr)
{
StringArrayFree(arr);
}
HashMap *
DbJson(DbRef * ref)
{
return ref ? ref->json : NULL;
}
bool
DbJsonSet(DbRef * ref, HashMap * json)
{
if (!ref || !json)
{
return false;
}
JsonFree(ref->json);
ref->json = JsonDuplicate(json);
return true;
}
void
DbInit(Db *db)
{
pthread_mutexattr_t attr;
if (!db)
{
return;
}
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&db->lock, &attr);
pthread_mutexattr_destroy(&attr);
db->mostRecent = NULL;
db->leastRecent = NULL;
db->cacheSize = 0;
db->maxCache = 0;
if (db->maxCache)
{
db->cache = HashMapCreate();
}
else
{
db->cache = NULL;
}
}
void
DbRefInit(Db *db, DbRef *ref)
{
if (!db || !ref)
{
return;
}
ref->prev = db->mostRecent;
ref->next = NULL;
ref->json = NULL;
ref->name = NULL;
/* As default values to be overwritten by impls */
ref->ts = UtilTsMillis();
ref->size = 0;
/* TODO: Append the ref to the cache list.
* I removed it because it broke everything and crashed all the time.
* My bad! */
}
void
StringArrayAppend(Array *arr, char *str)
{
if (!arr || !str)
{
return;
}
ArrayAdd(arr, StrDuplicate(str));
}

375
src/Db/Flat.c
View file

@ -1,375 +0,0 @@
#include <Db.h>
#include <Memory.h>
#include <Json.h>
#include <Util.h>
#include <Str.h>
#include <Stream.h>
#include <Log.h>
#include <sys/types.h>
#include <dirent.h>
#include <pthread.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include "Db/Internal.h"
typedef struct FlatDb {
Db base;
char *dir;
/* Theres not much to do here. */
} FlatDb;
typedef struct FlatDbRef {
DbRef base;
Stream *stream;
int fd;
} FlatDbRef;
static char
DbSanitiseChar(char input)
{
switch (input)
{
case '/':
return '_';
case '.':
return '-';
}
return input;
}
static char *
DbDirName(FlatDb * db, Array * args, size_t strip)
{
size_t i, j;
char *str = StrConcat(2, db->dir, "/");
for (i = 0; i < ArraySize(args) - strip; i++)
{
char *tmp;
char *sanitise = StrDuplicate(ArrayGet(args, i));
size_t len = strlen(sanitise);
for (j = 0; j < len; j++)
{
sanitise[j] = DbSanitiseChar(sanitise[j]);
}
tmp = StrConcat(3, str, sanitise, "/");
Free(str);
Free(sanitise);
str = tmp;
}
return str;
}
static char *
DbFileName(FlatDb * db, Array * args)
{
size_t i;
char *str = StrConcat(2, db->dir, "/");
for (i = 0; i < ArraySize(args); i++)
{
char *tmp;
char *arg = StrDuplicate(ArrayGet(args, i));
size_t j = 0;
/* Sanitize name to prevent directory traversal attacks */
while (arg[j])
{
arg[j] = DbSanitiseChar(arg[j]);
j++;
}
tmp = StrConcat(3, str, arg,
(i < ArraySize(args) - 1) ? "/" : ".json");
Free(arg);
Free(str);
str = tmp;
}
return str;
}
static DbRef *
FlatLock(Db *d, DbHint hint, Array *dir)
{
FlatDb *db = (FlatDb *) d;
FlatDbRef *ref = NULL;
size_t i;
char *path = NULL;
if (!d || !dir)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
path = DbFileName(db, dir);
/* TODO: Caching */
{
int fd = open(path, O_RDWR);
Stream *stream;
struct flock lock;
if (fd == -1)
{
ref = NULL;
goto end;
}
stream = StreamFd(fd);
if (!stream)
{
ref = NULL;
goto end;
}
lock.l_start = 0;
lock.l_len = 0;
lock.l_type = F_WRLCK;
lock.l_whence = SEEK_SET;
if (fcntl(fd, F_SETLK, &lock) < 0)
{
StreamClose(stream);
ref = NULL;
goto end;
}
ref = Malloc(sizeof(*ref));
DbRefInit(d, (DbRef *) ref);
/* TODO: Hints */
ref->base.hint = hint;
ref->base.ts = UtilLastModified(path);
ref->base.json = JsonDecode(stream);
ref->stream = stream;
ref->fd = fd;
if (!ref->base.json)
{
Free(ref);
StreamClose(stream);
ref = NULL;
goto end;
}
ref->base.name = ArrayCreate();
for (i = 0; i < ArraySize(dir); i++)
{
StringArrayAppend(ref->base.name, ArrayGet(dir, i));
}
}
end:
Free(path);
if (!ref)
{
pthread_mutex_unlock(&d->lock);
}
return (DbRef *) ref;
}
static bool
FlatUnlock(Db *d, DbRef *r)
{
FlatDbRef *ref = (FlatDbRef *) r;
if (!d || !r)
{
return false;
}
lseek(ref->fd, 0L, SEEK_SET);
if (ftruncate(ref->fd, 0) < 0)
{
pthread_mutex_unlock(&d->lock);
Log(LOG_ERR, "Failed to truncate file on disk.");
Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno));
return false;
}
JsonEncode(ref->base.json, ref->stream, JSON_DEFAULT);
StreamClose(ref->stream);
JsonFree(ref->base.json);
StringArrayFree(ref->base.name);
Free(ref);
pthread_mutex_unlock(&d->lock);
return true;
}
static DbRef *
FlatCreate(Db *d, Array *dir)
{
FlatDb *db = (FlatDb *) d;
char *path, *dirPath;
Stream *fp;
DbRef *ret;
if (!d || !dir)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
path = DbFileName(db, dir);
if (UtilLastModified(path))
{
Free(path);
pthread_mutex_unlock(&d->lock);
return NULL;
}
dirPath = DbDirName(db, dir, 1);
if (UtilMkdir(dirPath, 0750) < 0)
{
Free(path);
Free(dirPath);
pthread_mutex_unlock(&d->lock);
return NULL;
}
Free(dirPath);
fp = StreamOpen(path, "w");
Free(path);
if (!fp)
{
pthread_mutex_unlock(&d->lock);
return NULL;
}
StreamPuts(fp, "{}");
StreamClose(fp);
/* FlatLock() will lock again for us */
pthread_mutex_unlock(&d->lock);
ret = FlatLock(d, DB_HINT_WRITE, dir);
return ret;
}
static bool
FlatDelete(Db *d, Array *dir)
{
bool ret = false;
char *file;
FlatDb *db = (FlatDb *) d;
if (!d || !dir)
{
return false;
}
pthread_mutex_lock(&d->lock);
file = DbFileName(db, dir);
/* TODO: Unlink the entry from the linkedlist */
if (UtilLastModified(file))
{
ret = remove(file) == 0;
}
Free(file);
pthread_mutex_unlock(&d->lock);
return ret;
}
static Array *
FlatList(Db *d, Array *dir)
{
FlatDb *db = (FlatDb *) d;
struct dirent *file;
Array *ret;
DIR *files;
char *path;
if (!d || !dir)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
path = DbDirName(db, dir, 0);
files = opendir(path);
if (!files)
{
Free(path);
pthread_mutex_unlock(&d->lock);
return NULL;
}
ret = ArrayCreate();
while ((file = readdir(files)))
{
size_t namlen = strlen(file->d_name);
if (namlen > 5)
{
int nameOffset = namlen - 5;
if (StrEquals(file->d_name + nameOffset, ".json"))
{
file->d_name[nameOffset] = '\0';
StringArrayAppend(ret, file->d_name);
}
}
}
closedir(files);
Free(path);
pthread_mutex_unlock(&d->lock);
return ret;
}
static bool
FlatExists(Db *d, Array *dir)
{
FlatDb *db = (FlatDb *) d;
char *path;
bool ret;
if (!d || !dir)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
path = DbFileName(db, dir);
ret = UtilLastModified(path) != 0;
Free(path);
pthread_mutex_unlock(&d->lock);
return ret;
}
Db *
DbOpen(char *dir, size_t cache)
{
FlatDb *db;
if (!dir)
{
return NULL;
}
db = Malloc(sizeof(*db));
DbInit((Db *) db);
db->dir = dir;
db->base.cacheSize = cache;
db->base.lockFunc = FlatLock;
db->base.unlock = FlatUnlock;
db->base.create = FlatCreate;
db->base.delete = FlatDelete;
db->base.exists = FlatExists;
db->base.list = FlatList;
db->base.close = NULL;
return (Db *) db;
}

101
src/Db/Internal.h
View file

@ -1,101 +0,0 @@
/*
* Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation files
* (the "Software"), to deal in the Software without restriction,
* including without limitation the rights to use, copy, modify, merge,
* publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef CYTOPLASM_DB_INTERNAL_H
#define CYTOPLASM_DB_INTERNAL_H
#include <HashMap.h>
#include <Db.h>
#include <pthread.h>
/* TODO: Define the base structures to define a database internally.
* All "implementations" shall have them as a first element, so that
* basic, general functions can work on them properly. */
/* The base structure of a database */
struct Db
{
pthread_mutex_t lock;
size_t cacheSize;
size_t maxCache;
HashMap *cache;
/*
* The cache uses a double linked list (see DbRef
* below) to know which objects are most and least
* recently used. The following diagram helps me
* know what way all the pointers go, because it
* can get very confusing sometimes. For example,
* there's nothing stopping "next" from pointing to
* least recent, and "prev" from pointing to most
* recent, so hopefully this clarifies the pointer
* terminology used when dealing with the linked
* list:
*
* mostRecent leastRecent
* | prev prev | prev
* +---+ ---> +---+ ---> +---+ ---> NULL
* |ref| |ref| |ref|
* NULL <--- +---+ <--- +---+ <--- +---+
* next next next
*/
DbRef *mostRecent;
DbRef *leastRecent;
/* Functions for implementation-specific operations
* (opening a ref, closing a db, removing an entry, ...) */
DbRef * (*lockFunc)(Db *, DbHint, Array *);
DbRef * (*create)(Db *, Array *);
Array * (*list)(Db *, Array *);
bool (*unlock)(Db *, DbRef *);
bool (*delete)(Db *, Array *);
bool (*exists)(Db *, Array *);
void (*close)(Db *);
/* Implementation-specific constructs */
};
struct DbRef
{
HashMap *json;
uint64_t ts;
size_t size;
Array *name;
DbRef *prev;
DbRef *next;
DbHint hint;
/* Implementation-specific constructs */
};
extern void DbInit(Db *);
extern void DbRefInit(Db *, DbRef *);
extern void StringArrayFree(Array *);
extern void StringArrayAppend(Array *, char *);
#endif

593
src/Db/LMDB.c
View file

@ -1,593 +0,0 @@
#include <Memory.h>
#include <Json.h>
#include <Log.h>
#include <Db.h>
#include "Db/Internal.h"
#include <stdint.h>
#include <string.h>
#include <stdlib.h>
#ifdef EDB_LMDB
#include <lmdb.h>
typedef struct LMDB {
Db base; /* The base implementation required to pass */
MDB_env *environ;
MDB_dbi dbi;
} LMDB;
typedef struct LMDBRef {
DbRef base;
/* TODO: LMDB-dependent stuff */
MDB_txn *transaction;
MDB_dbi dbi;
} LMDBRef;
/* Helper functions */
static MDB_val
LMDBTranslateKey(Array *key)
{
MDB_val ret = { 0 };
char *data = NULL;
size_t length = 0;
size_t i;
if (!key || ArraySize(key) > 255)
{
return ret;
}
data = Realloc(data, ++length);
data[0] = ArraySize(key);
/* Now, let's push every item */
for (i = 0; i < ArraySize(key); i++)
{
char *entry = ArrayGet(key, i);
size_t offset = length;
data = Realloc(data, (length += strlen(entry) + 1));
memcpy(data + offset, entry, strlen(entry));
data[length - 1] = '\0';
}
/* We now have every key */
ret.mv_size = length;
ret.mv_data = data;
return ret;
}
static void
LMDBKillKey(MDB_val key)
{
if (!key.mv_data || !key.mv_size)
{
return;
}
Free(key.mv_data);
}
static HashMap *
LMDBDecode(MDB_val val)
{
FILE *fakefile;
Stream *fakestream;
HashMap *ret;
if (!val.mv_data || !val.mv_size)
{
return NULL;
}
fakefile = fmemopen(val.mv_data, val.mv_size, "r");
fakestream = StreamFile(fakefile);
ret = JsonDecode(fakestream);
StreamClose(fakestream);
return ret;
}
static bool
LMDBKeyStartsWith(MDB_val key, MDB_val starts)
{
size_t i;
if (!key.mv_size || !starts.mv_size)
{
return false;
}
if (key.mv_size < starts.mv_size)
{
return false;
}
for (i = 0; i < starts.mv_size; i++)
{
char keyC = ((char *) key.mv_data)[i];
char startC = ((char *) starts.mv_data)[i];
if (keyC != startC)
{
return false;
}
}
return true;
}
static char *
LMDBKeyHead(MDB_val key)
{
char *end;
if (!key.mv_size || !key.mv_data)
{
return NULL;
}
/* -2 because we have a NULL byte in there */
end = ((char *) key.mv_data) + key.mv_size - 1;
if ((void *) end < key.mv_data)
{
/* Uh oh. */
return NULL;
}
/* Doing >= will lead to cases where it is sent straight to the
* start. Don't do that. */
while ((void *) (end - 1) > key.mv_data && *(end - 1))
{
end--;
}
return end;
}
static DbRef *
LMDBLock(Db *d, DbHint hint, Array *k)
{
LMDB *db = (LMDB *) d;
MDB_txn *transaction;
LMDBRef *ret = NULL;
MDB_val key, json_val;
int code, flags;
if (!d || !k)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
key = LMDBTranslateKey(k);
/* Create a transaction, honoring hints. */
/* TODO: Do we want to create a "main" transaction that everyone inherits
* from? */
flags = hint == DB_HINT_READONLY ? MDB_RDONLY : 0;
if ((code = mdb_txn_begin(db->environ, NULL, flags, &transaction)) != 0)
{
/* Very bad! */
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
goto end;
}
json_val.mv_size = 0;
json_val.mv_data = NULL;
code = mdb_get(transaction, db->dbi, &key, &json_val);
if (code == MDB_NOTFOUND)
{
mdb_txn_abort(transaction);
goto end;
}
else if (code != 0)
{
Log(LOG_ERR,
"%s: mdb_get failure: %s",
__func__, mdb_strerror(code)
);
mdb_txn_abort(transaction);
goto end;
}
ret = Malloc(sizeof(*ret));
DbRefInit(d, (DbRef *) ret);
/* TODO: Timestamp */
{
size_t i;
ret->base.name = ArrayCreate();
for (i = 0; i < ArraySize(k); i++)
{
char *ent = ArrayGet(k, i);
StringArrayAppend(ret->base.name, ent);
}
}
ret->base.json = LMDBDecode(json_val);
ret->base.hint = hint;
ret->transaction = NULL;
if (hint == DB_HINT_WRITE)
{
ret->transaction = transaction;
}
else
{
mdb_txn_abort(transaction);
}
end:
if (!ret || hint == DB_HINT_READONLY)
{
pthread_mutex_unlock(&d->lock);
}
LMDBKillKey(key);
return (DbRef *) ret;
}
static bool
LMDBExists(Db *d, Array *k)
{
MDB_val key, empty;
LMDB *db = (LMDB *) d;
MDB_txn *transaction;
int code;
bool ret = false;
if (!d || !k)
{
return false;
}
pthread_mutex_lock(&d->lock);
key = LMDBTranslateKey(k);
/* create a txn */
if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0)
{
/* Very bad! */
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
goto end;
}
ret = mdb_get(transaction, db->dbi, &key, &empty) == 0;
mdb_txn_abort(transaction);
end:
LMDBKillKey(key);
pthread_mutex_unlock(&d->lock);
return ret;
}
static bool
LMDBDelete(Db *d, Array *k)
{
MDB_val key, empty;
LMDB *db = (LMDB *) d;
MDB_txn *transaction;
int code;
bool ret = false;
if (!d || !k)
{
return false;
}
pthread_mutex_lock(&d->lock);
key = LMDBTranslateKey(k);
/* create a txn */
if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0)
{
/* Very bad! */
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
goto end;
}
ret = mdb_del(transaction, db->dbi, &key, &empty) == 0;
mdb_txn_commit(transaction);
end:
LMDBKillKey(key);
pthread_mutex_unlock(&d->lock);
return ret;
}
static bool
LMDBUnlock(Db *d, DbRef *r)
{
LMDBRef *ref = (LMDBRef *) r;
LMDB *db = (LMDB *) d;
FILE *fakestream;
Stream *stream;
MDB_val key, val;
bool ret = true;
DbHint hint = r ? r->hint : 0;
if (!d || !r)
{
return false;
}
val.mv_data = NULL;
val.mv_size = 0;
if (ref->transaction && r->hint == DB_HINT_WRITE)
{
key = LMDBTranslateKey(r->name);
fakestream = open_memstream((char **) &val.mv_data, &val.mv_size);
stream = StreamFile(fakestream);
JsonEncode(r->json, stream, JSON_DEFAULT);
StreamFlush(stream);
StreamClose(stream);
ret = mdb_put(ref->transaction, db->dbi, &key, &val, 0) == 0;
mdb_txn_commit(ref->transaction);
LMDBKillKey(key);
}
StringArrayFree(ref->base.name);
JsonFree(ref->base.json);
Free(ref);
if (val.mv_data)
{
free(val.mv_data);
}
if (ret && hint == DB_HINT_WRITE)
{
pthread_mutex_unlock(&d->lock);
}
return ret;
}
static DbRef *
LMDBCreate(Db *d, Array *k)
{
LMDB *db = (LMDB *) d;
MDB_txn *transaction;
LMDBRef *ret = NULL;
MDB_val key, empty_json;
int code;
if (!d || !k)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
key = LMDBTranslateKey(k);
/* create a txn */
if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0)
{
/* Very bad! */
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
goto end;
}
empty_json.mv_size = 2;
empty_json.mv_data = "{}";
/* put data in it */
code = mdb_put(transaction, db->dbi, &key, &empty_json, MDB_NOOVERWRITE);
if (code == MDB_KEYEXIST)
{
mdb_txn_abort(transaction);
goto end;
}
else if (code == MDB_MAP_FULL)
{
Log(LOG_ERR, "%s: db is full", __func__);
mdb_txn_abort(transaction);
goto end;
}
else if (code != 0)
{
Log(LOG_ERR, "%s: mdb_put failure: %s", __func__, mdb_strerror(code));
mdb_txn_abort(transaction);
goto end;
}
ret = Malloc(sizeof(*ret));
DbRefInit(d, (DbRef *) ret);
/* TODO: Timestamp */
{
size_t i;
ret->base.name = ArrayCreate();
for (i = 0; i < ArraySize(k); i++)
{
char *ent = ArrayGet(k, i);
StringArrayAppend(ret->base.name, ent);
}
}
ret->base.hint = DB_HINT_WRITE;
ret->base.json = HashMapCreate();
ret->transaction = transaction;
end:
if (!ret)
{
pthread_mutex_unlock(&d->lock);
}
LMDBKillKey(key);
return (DbRef *) ret;
}
static Array *
LMDBList(Db *d, Array *k)
{
LMDB *db = (LMDB *) d;
MDB_val key = { 0 };
MDB_val subKey;
MDB_val val;
Array *ret = NULL;
MDB_cursor *cursor;
MDB_cursor_op op = MDB_SET_RANGE;
MDB_txn *txn;
int code;
if (!d || !k)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
/* Marked as read-only, as we just don't need to write anything
* when listing */
if ((code = mdb_txn_begin(db->environ, NULL, MDB_RDONLY, &txn)) != 0)
{
/* Very bad! */
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
goto end;
}
if ((code = mdb_cursor_open(txn, db->dbi, &cursor)) != 0)
{
Log(LOG_ERR,
"%s: could not get cursor: %s",
__func__, mdb_strerror(code)
);
mdb_txn_abort(txn);
goto end;
}
key = LMDBTranslateKey(k);
/* Small hack to get it to list subitems */
((char *) key.mv_data)[0]++;
ret = ArrayCreate();
subKey = key;
while (mdb_cursor_get(cursor, &subKey, &val, op) == 0)
{
/* This searches by *increasing* order. The problem is that it may
* extend to unwanted points. Since the values are sorted, we can
* just exit if the subkey is incorrect. */
char *head = LMDBKeyHead(subKey);
if (!LMDBKeyStartsWith(subKey, key))
{
break;
}
StringArrayAppend(ret, head);
op = MDB_NEXT;
}
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
end:
LMDBKillKey(key);
pthread_mutex_unlock(&d->lock);
return ret;
}
/* Implementation functions */
static void
LMDBClose(Db *d)
{
LMDB *db = (LMDB *) d;
if (!d)
{
return;
}
mdb_dbi_close(db->environ, db->dbi);
mdb_env_close(db->environ);
}
Db *
DbOpenLMDB(char *dir, size_t size)
{
/* TODO */
MDB_env *env = NULL;
MDB_txn *txn;
MDB_dbi dbi;
int code;
LMDB *db;
if (!dir || !size)
{
return NULL;
}
/* Try initialising LMDB */
if ((code = mdb_env_create(&env)) != 0)
{
Log(LOG_ERR,
"%s: could not create LMDB env: %s",
__func__, mdb_strerror(code)
);
return NULL;
}
if ((code = mdb_env_set_mapsize(env, size) != 0))
{
Log(LOG_ERR,
"%s: could not set mapsize to %lu: %s",
__func__, (unsigned long) size,
mdb_strerror(code)
);
mdb_env_close(env);
return NULL;
}
mdb_env_set_maxdbs(env, 4);
if ((code = mdb_env_open(env, dir, MDB_NOTLS, 0644)) != 0)
{
Log(LOG_ERR,
"%s: could not open LMDB env: %s",
__func__, mdb_strerror(code)
);
mdb_env_close(env);
return NULL;
}
/* Initialise a DBI */
{
if ((code = mdb_txn_begin(env, NULL, 0, &txn)) != 0)
{
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
mdb_env_close(env);
return NULL;
}
if ((code = mdb_dbi_open(txn, "db", MDB_CREATE, &dbi)) != 0)
{
Log(LOG_ERR,
"%s: could not get transaction dbi: %s",
__func__, mdb_strerror(code)
);
mdb_txn_abort(txn);
mdb_env_close(env);
return NULL;
}
mdb_txn_commit(txn);
}
db = Malloc(sizeof(*db));
DbInit((Db *) db);
db->environ = env;
db->dbi = dbi;
db->base.lockFunc = LMDBLock;
db->base.create = LMDBCreate;
db->base.unlock = LMDBUnlock;
db->base.delete = LMDBDelete;
db->base.exists = LMDBExists;
db->base.close = LMDBClose;
db->base.list = LMDBList;
return (Db *) db;
}
#else
Db *
DbOpenLMDB(char *dir, size_t size)
{
/* Unimplemented function */
Log(LOG_ERR,
"LMDB support is not enabled. Please compile with --use-lmdb"
);
(void) size;
(void) dir;
return NULL;
}
#endif

19
src/Json.c
View file

@ -1348,25 +1348,6 @@ JsonDecode(Stream * stream)
return result;
}
JsonValue *
JsonValueDecode(Stream *stream)
{
JsonValue *result;
JsonParserState state;
state.stream = stream;
state.token = NULL;
JsonTokenSeek(&state);
result = JsonDecodeValue(&state);
if (state.token)
{
Free(state.token);
}
return result;
}
JsonValue *
JsonGet(HashMap * json, size_t nArgs,...)

260
src/Memory.c
View file

@ -42,52 +42,29 @@
#define MEMORY_FILE_SIZE 256
#define MEM_BOUND_TYPE uint64_t
#define MEM_BOUND 0xDEADBEEFBEEFDEAD
#define MEM_MAGIC 0xDEADBEEFDEADBEEF
struct MemoryInfo
{
uint64_t magic;
size_t size;
size_t unalignedSize;
char file[MEMORY_FILE_SIZE];
int line;
void *pointer;
MemoryInfo *prev;
MemoryInfo *next;
MEM_BOUND_TYPE leftBoundary;
};
#define MEM_SIZE_ACTUAL(x) (MemoryAlignBoundary((x) * sizeof(uint8_t)) + sizeof(MEM_BOUND_TYPE))
#define MEM_START_BOUNDARY(info) (info->leftBoundary)
#define MEM_END_BOUNDARY(info) (*(((MEM_BOUND_TYPE *) (((uint8_t *) info->pointer) + info->size)) - 1))
#define MEM_BOUND_TYPE uint32_t
#define MEM_BOUND 0xDEADBEEF
#define MEM_BOUND_LOWER(p) *((MEM_BOUND_TYPE *) p)
#define MEM_BOUND_UPPER(p, x) *(((MEM_BOUND_TYPE *) (((uint8_t *) p) + x)) + 1)
#define MEM_SIZE_ACTUAL(x) (((x) * sizeof(uint8_t)) + (2 * sizeof(MEM_BOUND_TYPE)))
static pthread_mutex_t lock;
static void (*hook) (MemoryAction, MemoryInfo *, void *) = MemoryDefaultHook;
static void *hookArgs = NULL;
static MemoryInfo **allocations = NULL;
static size_t allocationsSize = 0;
static size_t allocationsLen = 0;
static MemoryInfo *allocationTail = NULL;
/* Simple range of "plausible" boundaries for heap, serving as an extra
* check */
static void *heapStart, *heapEnd;
static size_t MemoryAlignBoundary(size_t size)
{
size_t boundSize = sizeof(MEM_BOUND_TYPE);
size_t remainder = size % boundSize;
size_t closest = size / boundSize + !!remainder;
return closest * boundSize;
}
int
MemoryRuntimeInit(void)
{
@ -102,8 +79,6 @@ MemoryRuntimeInit(void)
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
ret = pthread_mutex_init(&lock, &attr);
pthread_mutexattr_destroy(&attr);
heapStart = NULL;
heapEnd = NULL;
ret = (ret == 0);
@ -118,27 +93,70 @@ MemoryRuntimeDestroy(void)
return pthread_mutex_destroy(&lock) == 0;
}
static size_t
MemoryHash(void *p)
{
return (((size_t) p) >> 2 * 7) % allocationsSize;
}
static int
MemoryInsert(MemoryInfo * a)
{
if (allocationTail)
{
allocationTail->next = a;
}
a->next = NULL;
a->prev = allocationTail;
a->magic = MEM_MAGIC;
size_t hash;
if (!heapStart || heapStart > (void *) a)
if (!allocations)
{
heapStart = a;
}
if (!heapEnd || heapEnd < (void *) a)
{
heapEnd = a;
allocationsSize = MEMORY_TABLE_CHUNK;
allocations = calloc(allocationsSize, sizeof(void *));
if (!allocations)
{
return 0;
}
}
allocationTail = a;
/* If the next insertion would cause the table to be at least 3/4
* full, re-allocate and re-hash. */
if ((allocationsLen + 1) >= ((allocationsSize * 3) >> 2))
{
size_t i;
size_t tmpAllocationsSize = allocationsSize;
MemoryInfo **tmpAllocations;
allocationsSize += MEMORY_TABLE_CHUNK;
tmpAllocations = calloc(allocationsSize, sizeof(void *));
if (!tmpAllocations)
{
return 0;
}
for (i = 0; i < tmpAllocationsSize; i++)
{
if (allocations[i])
{
hash = MemoryHash(allocations[i]->pointer);
while (tmpAllocations[hash])
{
hash = (hash + 1) % allocationsSize;
}
tmpAllocations[hash] = allocations[i];
}
}
free(allocations);
allocations = tmpAllocations;
}
hash = MemoryHash(a->pointer);
while (allocations[hash])
{
hash = (hash + 1) % allocationsSize;
}
allocations[hash] = a;
allocationsLen++;
return 1;
@ -147,32 +165,30 @@ MemoryInsert(MemoryInfo * a)
static void
MemoryDelete(MemoryInfo * a)
{
MemoryInfo *aPrev = a->prev;
MemoryInfo *aNext = a->next;
size_t hash = MemoryHash(a->pointer);
size_t count = 0;
if (aPrev)
while (count <= allocationsSize)
{
aPrev->next = aNext;
if (allocations[hash] && allocations[hash] == a)
{
allocations[hash] = NULL;
allocationsLen--;
return;
}
else
{
hash = (hash + 1) % allocationsSize;
count++;
}
}
if (aNext)
{
aNext->prev = aPrev;
}
if (a == allocationTail)
{
allocationTail = aPrev;
}
a->magic = ~MEM_MAGIC;
}
static int
MemoryCheck(MemoryInfo * a)
{
if (MEM_START_BOUNDARY(a) != MEM_BOUND ||
a->magic != MEM_MAGIC ||
MEM_END_BOUNDARY(a) != MEM_BOUND)
if (MEM_BOUND_LOWER(a->pointer) != MEM_BOUND ||
MEM_BOUND_UPPER(a->pointer, a->size - (2 * sizeof(MEM_BOUND_TYPE))) != MEM_BOUND)
{
if (hook)
{
@ -189,32 +205,38 @@ MemoryAllocate(size_t size, const char *file, int line)
void *p;
MemoryInfo *a;
//MemoryIterate(NULL, NULL);
MemoryIterate(NULL, NULL);
pthread_mutex_lock(&lock);
a = malloc(sizeof(MemoryInfo) + MEM_SIZE_ACTUAL(size));
a = malloc(sizeof(MemoryInfo));
if (!a)
{
pthread_mutex_unlock(&lock);
return NULL;
}
p = a + 1;
p = malloc(MEM_SIZE_ACTUAL(size));
if (!p)
{
free(a);
pthread_mutex_unlock(&lock);
return NULL;
}
memset(p, 0, MEM_SIZE_ACTUAL(size));
MEM_BOUND_LOWER(p) = MEM_BOUND;
MEM_BOUND_UPPER(p, size) = MEM_BOUND;
a->size = MEM_SIZE_ACTUAL(size);
a->unalignedSize = size;
strncpy(a->file, file, MEMORY_FILE_SIZE - 1);
a->line = line;
a->pointer = p;
MEM_START_BOUNDARY(a) = MEM_BOUND;
MEM_END_BOUNDARY(a) = MEM_BOUND;
if (!MemoryInsert(a))
{
free(a);
free(p);
pthread_mutex_unlock(&lock);
return NULL;
}
@ -225,7 +247,7 @@ MemoryAllocate(size_t size, const char *file, int line)
}
pthread_mutex_unlock(&lock);
return p;
return ((MEM_BOUND_TYPE *) p) + 1;
}
void *
@ -234,7 +256,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line)
MemoryInfo *a;
void *new = NULL;
//MemoryIterate(NULL, NULL);
MemoryIterate(NULL, NULL);
if (!p)
{
@ -245,28 +267,25 @@ MemoryReallocate(void *p, size_t size, const char *file, int line)
if (a)
{
pthread_mutex_lock(&lock);
MemoryDelete(a);
new = realloc(a, sizeof(MemoryInfo) + MEM_SIZE_ACTUAL(size));
new = realloc(a->pointer, MEM_SIZE_ACTUAL(size));
if (new)
{
a = new;
a->unalignedSize = size;
MemoryDelete(a);
a->size = MEM_SIZE_ACTUAL(size);
strncpy(a->file, file, MEMORY_FILE_SIZE - 1);
a->line = line;
a->magic = MEM_MAGIC;
a->pointer = a + 1;
a->pointer = new;
MemoryInsert(a);
MEM_START_BOUNDARY(a) = MEM_BOUND;
MEM_END_BOUNDARY(a) = MEM_BOUND;
MEM_BOUND_LOWER(a->pointer) = MEM_BOUND;
MEM_BOUND_UPPER(a->pointer, size) = MEM_BOUND;
if (hook)
{
hook(MEMORY_REALLOCATE, a, hookArgs);
}
}
pthread_mutex_unlock(&lock);
}
@ -284,7 +303,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line)
}
}
return a->pointer;
return ((MEM_BOUND_TYPE *) new) + 1;
}
void
@ -292,7 +311,7 @@ MemoryFree(void *p, const char *file, int line)
{
MemoryInfo *a;
//MemoryIterate(NULL, NULL);
MemoryIterate(NULL, NULL);
if (!p)
{
@ -310,6 +329,7 @@ MemoryFree(void *p, const char *file, int line)
hook(MEMORY_FREE, a, hookArgs);
}
MemoryDelete(a);
free(a->pointer);
free(a);
pthread_mutex_unlock(&lock);
@ -332,15 +352,17 @@ MemoryFree(void *p, const char *file, int line)
size_t
MemoryAllocated(void)
{
size_t i;
size_t total = 0;
MemoryInfo *cur;
pthread_mutex_lock(&lock);
/* TODO */
for (cur = allocationTail; cur; cur = cur->prev)
for (i = 0; i < allocationsSize; i++)
{
total += MemoryInfoGetSize(cur);
if (allocations[i])
{
total += MemoryInfoGetSize(allocations[i]);
}
}
pthread_mutex_unlock(&lock);
@ -351,44 +373,55 @@ MemoryAllocated(void)
void
MemoryFreeAll(void)
{
MemoryInfo *cur;
MemoryInfo *prev;
size_t i;
pthread_mutex_lock(&lock);
/* TODO */
for (cur = allocationTail; cur; cur = prev)
for (i = 0; i < allocationsSize; i++)
{
prev = cur->prev;
free(cur);
if (allocations[i])
{
free(allocations[i]->pointer);
free(allocations[i]);
}
}
free(allocations);
allocations = NULL;
allocationsSize = 0;
allocationsLen = 0;
pthread_mutex_unlock(&lock);
}
MemoryInfo *
MemoryInfoGet(void *po)
MemoryInfoGet(void *p)
{
void *p = po;
size_t hash, count;
pthread_mutex_lock(&lock);
p = ((MemoryInfo *) p) - 1;
if (p < heapStart || p > heapEnd)
{
pthread_mutex_unlock(&lock);
return NULL;
}
p = ((MEM_BOUND_TYPE *) p) - 1;
hash = MemoryHash(p);
if (((MemoryInfo *)p)->magic != MEM_MAGIC)
count = 0;
while (count <= allocationsSize)
{
p = NULL;
if (!allocations[hash] || allocations[hash]->pointer != p)
{
hash = (hash + 1) % allocationsSize;
count++;
}
else
{
MemoryCheck(allocations[hash]);
pthread_mutex_unlock(&lock);
return allocations[hash];
}
}
pthread_mutex_unlock(&lock);
return p;
return NULL;
}
size_t
@ -399,7 +432,7 @@ MemoryInfoGetSize(MemoryInfo * a)
return 0;
}
return a->size ? a->unalignedSize : 0;
return a->size ? a->size - (2 * sizeof(MEM_BOUND_TYPE)) : 0;
}
const char *
@ -432,22 +465,25 @@ MemoryInfoGetPointer(MemoryInfo * a)
return NULL;
}
return a->pointer;
return ((MEM_BOUND_TYPE *) a->pointer) + 1;
}
void
MemoryIterate(void (*iterFunc) (MemoryInfo *, void *), void *args)
MemoryIterate(void (*iterFunc) (MemoryInfo *, void *), void *args)
{
MemoryInfo *cur;
size_t i;
pthread_mutex_lock(&lock);
for (cur = allocationTail; cur; cur = cur->prev)
for (i = 0; i < allocationsSize; i++)
{
MemoryCheck(cur);
if (iterFunc)
if (allocations[i])
{
iterFunc(cur, args);
MemoryCheck(allocations[i]);
if (iterFunc)
{
iterFunc(allocations[i], args);
}
}
}

33
src/Sha.c
View file

@ -28,44 +28,21 @@
#include <string.h>
char *
ShaToHex(unsigned char *bytes, HashType type)
ShaToHex(unsigned char *bytes)
{
size_t i = 0, size;
char *str;
size_t i = 0;
char *str = Malloc(((strlen((char *) bytes) * 2) + 1) * sizeof(char));
switch (type)
{
case HASH_SHA1:
size = 20;
break;
case HASH_SHA256:
size = 32;
break;
default:
return NULL;
}
str = Malloc(((size * 2) + 1) * sizeof(char));
if (!str)
{
return NULL;
}
for (i = 0; i < size; i++)
while (bytes[i] != '\0')
{
snprintf(str + (2 * i), 3, "%02x", bytes[i]);
i++;
}
return str;
}
unsigned char *
Sha256(char *str)
{
return Sha256Raw((unsigned char *) str, str ? strlen(str) : 0);
}
unsigned char *
Sha1(char *str)
{
return Sha1Raw((unsigned char *) str, str ? strlen(str) : 0);
}

26
src/Sha/Sha1.c
View file

@ -28,27 +28,6 @@
#include <limits.h>
/* TODO: Verify LibreSSL support later */
#if defined(TLS_IMPL) && (TLS_IMPL == TLS_OPENSSL)
#include <openssl/sha.h>
unsigned char *
Sha1Raw(unsigned char *str, size_t len)
{
unsigned char *digest;
if (!str)
{
return NULL;
}
digest = Malloc(20 + 1);
SHA1(str, len, digest);
digest[20] = '\0';
return digest;
}
#else
#define LOAD32H(x, y) \
{ \
x = ((uint32_t)((y)[0] & 255) << 24) | \
@ -261,7 +240,7 @@ Sha1Calculate(Sha1Context * ctx, unsigned char *out)
}
unsigned char *
Sha1Raw(unsigned char *str, size_t len)
Sha1(char *str)
{
Sha1Context ctx;
unsigned char *out;
@ -278,11 +257,10 @@ Sha1Raw(unsigned char *str, size_t len)
}
Sha1Init(&ctx);
Sha1Update(&ctx, str, len);
Sha1Update(&ctx, str, strlen(str));
Sha1Calculate(&ctx, out);
out[160 / 8] = '\0';
return out;
}
#endif

29
src/Sha/Sha256.c
View file

@ -21,36 +21,14 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <Memory.h>
#include <Sha.h>
#include <Memory.h>
#include <stdio.h>
#include <string.h>
#include <limits.h>
/* TODO: Verify LibreSSL support later */
#if defined(TLS_IMPL) && (TLS_IMPL == TLS_OPENSSL)
#include <openssl/sha.h>
unsigned char *
Sha256Raw(unsigned char *str, size_t len)
{
unsigned char *digest;
if (!str)
{
return NULL;
}
digest = Malloc(32 + 1);
SHA256(str, len, digest);
digest[32] = '\0';
return digest;
}
#else
#define GET_UINT32(x) \
(((uint32_t)(x)[0] << 24) | \
((uint32_t)(x)[1] << 16) | \
@ -192,7 +170,7 @@ Sha256Process(Sha256Context * context, unsigned char *data, size_t length)
}
unsigned char *
Sha256Raw(unsigned char *str, size_t len)
Sha256(char *str)
{
Sha256Context context;
size_t i;
@ -228,7 +206,7 @@ Sha256Raw(unsigned char *str, size_t len)
context.length = 0;
memset(context.buffer, 0, 64);
Sha256Process(&context, str, len);
Sha256Process(&context, (unsigned char *) str, strlen(str));
memset(fill, 0, 64);
fill[0] = 0x80;
@ -252,4 +230,3 @@ Sha256Raw(unsigned char *str, size_t len)
return out;
}
#endif

2
src/Tls/TlsLibreSSL.c
View file

@ -23,7 +23,7 @@
*/
#include <Tls.h>
#if defined(TLS_IMPL) && (TLS_IMPL == TLS_LIBRESSL)
#if TLS_IMPL == TLS_LIBRESSL
#include <Memory.h>
#include <Log.h>

2
src/Tls/TlsOpenSSL.c
View file

@ -23,7 +23,7 @@
*/
#include <Tls.h>
#if defined(TLS_IMPL) && (TLS_IMPL == TLS_OPENSSL)
#if TLS_IMPL == TLS_OPENSSL
#include <Memory.h>
#include <Log.h>

9
src/Util.c
View file

@ -24,7 +24,6 @@
#include <Util.h>
#include <Memory.h>
#include <Platform.h>
#include <stdio.h>
#include <stdlib.h>
@ -90,10 +89,6 @@ UtilTsMillis(void)
return ts;
}
#ifdef PLATFORM_DARWIN
#define st_mtim st_mtimespec
#endif
uint64_t
UtilLastModified(char *path)
{
@ -110,10 +105,6 @@ UtilLastModified(char *path)
return ts;
}
#ifdef PLATFORM_DARWIN
#undef st_mtim
#endif
int
UtilMkdir(const char *dir, const mode_t mode)
{

271
tools/j2s.c
View file

@ -38,7 +38,6 @@
#define MAX_DEPENDENCIES 32
#define IsDelimited(field, c1, c2) (*field == c1 && field[strlen(field) - 1] == c2)
static char *
Trim(char c, char *str)
{
@ -76,14 +75,10 @@ TypeToJsonType(char *type)
}
else
{
if (IsDelimited(type, '[', ']'))
if (*type == '[' && type[strlen(type) - 1] == ']')
{
return JSON_ARRAY;
}
else if (IsDelimited(type, '{', '}'))
{
return JSON_OBJECT;
}
else
{
return JSON_OBJECT;
@ -96,7 +91,7 @@ JsonTypeToStr(JsonType type)
{
switch (type)
{
case JSON_OBJECT:
case JSON_OBJECT:
return "JSON_OBJECT";
case JSON_ARRAY:
return "JSON_ARRAY";
@ -330,7 +325,6 @@ Main(Array * args)
{
char *fieldType;
bool isArrType = false;
bool isObjType = false;
JsonValue *requiredVal;
JsonValue *ignoreVal;
@ -349,18 +343,12 @@ Main(Array * args)
goto finish;
}
if (IsDelimited(fieldType, '[', ']'))
if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']')
{
fieldType++;
fieldType[strlen(fieldType) - 1] = '\0';
isArrType = true;
}
else if (IsDelimited(fieldType, '{', '}'))
{
fieldType++;
fieldType[strlen(fieldType) - 1] = '\0';
isObjType = true;
}
if (!StrEquals(fieldType, "object") &&
!StrEquals(fieldType, "array") &&
@ -386,10 +374,6 @@ Main(Array * args)
{
fieldType[strlen(fieldType)] = ']';
}
else if (isObjType)
{
fieldType[strlen(fieldType)] = '}';
}
requiredVal = HashMapGet(fieldObj, "required");
if (requiredVal && JsonValueType(requiredVal) != JSON_BOOLEAN)
@ -545,13 +529,11 @@ Main(Array * args)
{
cType = "double";
}
else if (StrEquals(fieldType, "object") ||
IsDelimited(fieldType, '{', '}'))
else if (StrEquals(fieldType, "object"))
{
cType = "HashMap *";
}
else if (StrEquals(fieldType, "array") ||
IsDelimited(fieldType, '[', ']'))
else if (StrEquals(fieldType, "array") || (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']'))
{
cType = "Array *";
}
@ -560,20 +542,7 @@ Main(Array * args)
cType = fieldType;
}
if (IsDelimited(fieldType, '{', '}') ||
IsDelimited(fieldType, '[', ']'))
{
char end = fieldType[strlen(fieldType) - 1];
fieldType[strlen(fieldType) - 1] = '\0';
StreamPrintf(headerFile, " %s /* of %s */ %s;\n", cType, fieldType + 1, field);
fieldType[strlen(fieldType)] = end;
}
else
{
StreamPrintf(headerFile, " %s %s;\n", cType, field);
}
StreamPrintf(headerFile, " %s %s;\n", cType, field);
}
StreamPrintf(headerFile, "} %s;\n\n", type);
@ -704,134 +673,7 @@ Main(Array * args)
StreamPrintf(implFile, " out->%s = JsonValueAsObject(val);\n", key);
StreamPrintf(implFile, " Free(val); /* Not JsonValueFree() because we want the inner value. */\n");
}
else if (IsDelimited(fieldType, '{', '}'))
{
fieldType++;
fieldType[strlen(fieldType) - 1] = '\0';
isEnum = StrEquals(JsonValueAsString(JsonGet(types, 2, fieldType, "type")), "enum");
jsonType = isEnum ? JSON_STRING : TypeToJsonType(fieldType);
StreamPrintf(implFile, " out->%s = HashMapCreate();\n", key);
StreamPrintf(implFile, " if (!out->%s)\n", key);
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " *errp = \"Failed to allocate memory for %s.%s.\";\n", type, key);
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " else\n");
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " HashMap *obj = JsonValueAsObject(val);\n");
StreamPrintf(implFile, " char *objKey;\n");
StreamPrintf(implFile, " JsonValue *v;\n");
StreamPrintf(implFile, "\n");
StreamPrintf(implFile, " while (HashMapIterate(obj, &objKey, (void **) &v))\n");
StreamPrintf(implFile, " {\n");
if (StrEquals(fieldType, "integer") ||
StrEquals(fieldType, "float") ||
StrEquals(fieldType, "boolean"))
{
char *cType;
if (StrEquals(fieldType, "integer"))
{
cType = "int64_t";
}
else if (StrEquals(fieldType, "float"))
{
cType = "double";
}
else if (StrEquals(fieldType, "boolean"))
{
cType = "bool";
}
else
{
/* Should never happen */
cType = NULL;
}
*fieldType = toupper(*fieldType);
StreamPrintf(implFile, " %s *ref;\n", cType);
StreamPrintf(implFile, " if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType));
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " *errp = \"%s.%s{} contains an invalid value.\";\n", type, key);
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " ref = Malloc(sizeof(%s));\n", cType);
StreamPrintf(implFile, " if (!ref)\n");
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " *errp = \"Unable to allocate memory for object value.\";\n");
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " *ref = JsonValueAs%s(v);\n", fieldType);
StreamPrintf(implFile, " HashMapSet(out->%s, objKey, ref);\n", key);
*fieldType = tolower(*fieldType);
}
else if (StrEquals(fieldType, "string"))
{
StreamPrintf(implFile, " if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType));
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " *errp = \"%s.%s[] contains an invalid value.\";\n", type, key);
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " HashMapSet(out->%s, objKey, StrDuplicate(JsonValueAsString(v)));\n", key);
}
else if (StrEquals(fieldType, "object"))
{
StreamPrintf(implFile, " if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType));
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " *errp = \"%s.%s[] contains an invalid value.\";\n", type, key);
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " HashMapSet(out->%s, objKey, JsonDuplicate(JsonValueAsObject(v)));\n", key);
}
else
{
if (isEnum)
{
StreamPrintf(implFile, " int parseResult;\n");
}
StreamPrintf(implFile, " %s *parsed;\n", fieldType);
StreamPrintf(implFile, " if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType));
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " *errp = \"%s.%s[] contains an invalid value.\";\n", type, key);
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " parsed = Malloc(sizeof(%s));\n", fieldType);
StreamPrintf(implFile, " if (!parsed)\n");
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " *errp = \"Unable to allocate memory for array value.\";\n");
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
if (isEnum)
{
StreamPrintf(implFile, " parseResult = %sFromStr(JsonValueAsString(v));\n", fieldType);
StreamPrintf(implFile, " *parsed = parseResult;\n");
StreamPrintf(implFile, " if (parseResult == -1)\n");
}
else
{
StreamPrintf(implFile, " if (!%sFromJson(JsonValueAsObject(v), parsed, errp))\n", fieldType);
}
StreamPrintf(implFile, " {\n");
if (!isEnum)
{
StreamPrintf(implFile, " %sFree(parsed);\n", fieldType);
}
StreamPrintf(implFile, " Free(parsed);\n");
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " HashMapSet(out->%s, objKey, parsed);\n", key);
}
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " }\n");
fieldType[strlen(fieldType)] = '}';
}
else if (IsDelimited(fieldType, '[', ']'))
else if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']')
{
fieldType++;
fieldType[strlen(fieldType) - 1] = '\0';
@ -1073,73 +915,7 @@ Main(Array * args)
{
StreamPrintf(implFile, " HashMapSet(json, \"%s\", JsonValueObject(JsonDuplicate(val->%s)));\n", Trim('_', key), key);
}
else if (IsDelimited(fieldType, '{', '}'))
{
int isPrimitive;
fieldType++;
fieldType[strlen(fieldType) - 1] = '\0';
isEnum = StrEquals(JsonValueAsString(JsonGet(types, 2, fieldType, "type")), "enum");
isPrimitive = StrEquals(fieldType, "integer") ||
StrEquals(fieldType, "boolean") ||
StrEquals(fieldType, "float");
StreamPrintf(implFile, " if (val->%s)\n", key);
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " char *objKey;\n");
StreamPrintf(implFile, " void *objVal;\n");
StreamPrintf(implFile, " HashMap *jsonObj = HashMapCreate();\n");
StreamPrintf(implFile, " if (!jsonObj)\n");
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " JsonFree(json);\n");
StreamPrintf(implFile, " return NULL;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " while (HashMapIterate(val->%s, &objKey, &objVal))\n", key);
StreamPrintf(implFile, " {\n");
if (StrEquals(fieldType, "string"))
{
StreamPrintf(implFile, " HashMapSet(jsonObj, objKey, JsonValueString(objVal));\n", key);
}
else if (!isPrimitive)
{
StreamPrintf(implFile, " HashMapSet(jsonObj, objKey, JsonValueObject(%sToJson(objVal)));\n", fieldType, key);
}
else
{
char *cType;
if (StrEquals(fieldType, "integer"))
{
cType = "int64_t";
}
else if (StrEquals(fieldType, "float"))
{
cType = "double";
}
else if (StrEquals(fieldType, "boolean"))
{
cType = "bool";
}
else
{
/* Should never happen */
cType = NULL;
}
*fieldType = toupper(*fieldType);
StreamPrintf(implFile, " HashMapSet(jsonObj, objKey, (JsonValue%s(*((%s *) objVal))));\n", fieldType, cType, key);
*fieldType = tolower(*fieldType);
}
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " HashMapSet(json, \"%s\", JsonValueObject(jsonObj));\n", Trim('_', key));
StreamPrintf(implFile, " }\n");
fieldType[strlen(fieldType)] = '}';
}
else if (IsDelimited(fieldType, '[', ']'))
else if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']')
{
int isPrimitive;
@ -1260,36 +1036,7 @@ Main(Array * args)
{
StreamPrintf(implFile, " Free(val->%s);\n", key);
}
else if (IsDelimited(fieldType, '{', '}'))
{
int isPrimitive;
fieldType++;
fieldType[strlen(fieldType) - 1] = '\0';
isEnum = StrEquals(JsonValueAsString(JsonGet(types, 2, fieldType, "type")), "enum");
isPrimitive = StrEquals(fieldType, "boolean") ||
StrEquals(fieldType, "float") ||
StrEquals(fieldType, "integer") ||
StrEquals(fieldType, "string");
StreamPrintf(implFile, " if (val->%s)\n", key);
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " char *objKey;\n");
StreamPrintf(implFile, " void *objVal;\n");
StreamPrintf(implFile, " while (HashMapIterate(val->%s, &objKey, &objVal))\n", key);
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " %sFree(objVal);\n", (!isEnum && !isPrimitive) ? fieldType : "", key);
if (!isEnum && !isPrimitive)
{
StreamPrintf(implFile, " Free(objVal);\n", key);
}
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " HashMapFree(val->%s);\n", key);
StreamPrintf(implFile, " }\n");
fieldType[strlen(fieldType)] = '}';
}
else if (IsDelimited(fieldType, '[', ']'))
else if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']')
{
int isPrimitive;