Telodendria/Cytoplasm
Telodendria/Cytoplasm
3
3
Fork 5
Code Issues 16 Pull requests 3 Projects Releases 2 Packages Activity Actions

Compare commits

base: Telodendria:master
Branches Tags
Telodendria:master
lda:db-args
lda:raw-types
lda:alignment
lda:add-mbed
lda:master
lda:fix-deadlock
lda:fix-configure
lda:sha-revamp
lda:opt-ssl-for-sha
lda:lmdbwerk
lda:hashmap-del-leak
lda:mem-moment
lda:fix-tls
lda:fix-duplicate-leak
lda:fix-stringify
lda:tls-file-leak
lda:direct-filenames
lda:fix-sanitation
lda:fix-router-issue
lda:meta-j2s-types
lda:encode-http
Telodendria:v0.4.1
Telodendria:v0.4.0
..
compare: lda:hashmap-del-leak
Branches Tags
lda:db-args
lda:raw-types
lda:alignment
lda:add-mbed
lda:master
lda:fix-deadlock
lda:fix-configure
lda:sha-revamp
lda:opt-ssl-for-sha
lda:lmdbwerk
lda:hashmap-del-leak
lda:mem-moment
lda:fix-tls
lda:fix-duplicate-leak
lda:fix-stringify
lda:tls-file-leak
lda:direct-filenames
lda:fix-sanitation
lda:fix-router-issue
lda:meta-j2s-types
lda:encode-http
Telodendria:master
Telodendria:v0.4.1
Telodendria:v0.4.0

No commits in common. "master" and "hashmap-del-leak" have entirely different histories.
master ... hashmap-de

21 changed files with 1185 additions and 2224 deletions
Show stats Expand all files Collapse all files

18
.forgejo/workflows/compile.yaml
View file

@ -1,18 +0,0 @@
name: Compile Cytoplasm
run-name: Compile Cytoplasm on ${{ forgejo.actor }}
on: [push, pull_request]
jobs:
"Compile Cytoplasm":
strategy:
matrix:
os: [alpine]
arch: [aarch64]
runs-on: ["${{ matrix.os }}", "${{ matrix.arch }}"]
steps:
- name: Check out repository
uses: actions/checkout@v4
- name: Configure Cytoplasm
run: ./configure
- name: Build Cytoplasm
run: make

25
.gitea/workflows/compile.yaml Normal file
View file

@ -0,0 +1,25 @@
name: Compile Cytoplasm
run-name: Compile Cytoplasm on ${{ gitea.actor }}
on: [push]
jobs:
"Compile Cytoplasm":
strategy:
matrix:
os: [debian-v12.4, alpine-v3.19, openbsd-v7.4, freebsd-v14.0, netbsd-v9.3]
arch: [x86, x86_64]
exclude:
# 32-bit OpenBSD does not behave well in QEMU. Even when using
# QEMU to emulate i386, it utilizes 100% of its CPU core and is
# still extremely sluggish. Thus, we don't have a working 32-bit
# OpenBSD runner, so exclude it from the matrix configuration.
- os: openbsd-v7.4
arch: x86
runs-on: ["${{ matrix.os }}", "${{ matrix.arch }}"]
steps:
- name: Check out repository
uses: actions/checkout@v3
- name: Configure Cytoplasm
run: ./configure
- name: Build Cytoplasm
run: make

29
configure vendored
View file

@ -15,7 +15,7 @@ TOOLS="tools"
# Default compiler flags. These must be supported by all POSIX C compilers.
# "Fancy" compilers that have additional options must be detected and set below.
CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE} -I${SRC}";
CFLAGS="-O1 -D_DEFAULT_SOURCE -I${INCLUDE}"
LIBS="-lm -lpthread"
# Default args for all platforms.
@ -24,10 +24,10 @@ SCRIPT_ARGS="--prefix=/usr/local --lib-name=Cytoplasm"
# Set SSL flags depending on the platform.
case "$(uname)" in
OpenBSD)
SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl --disable-lmdb"
SCRIPT_ARGS="${SCRIPT_ARGS} --with-libressl"
;;
*)
SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl --disable-lmdb"
SCRIPT_ARGS="${SCRIPT_ARGS} --with-openssl"
;;
esac
@ -37,7 +37,7 @@ case "$(uname)" in
# These systems typically use GCC.
SCRIPT_ARGS="${SCRIPT_ARGS} --cc=gcc"
;;
OpenBSD|FreeBSD|Darwin)
OpenBSD|FreeBSD)
# These systems typically use Clang.
SCRIPT_ARGS="${SCRIPT_ARGS} --cc=clang"
;;
@ -80,14 +80,6 @@ for arg in $SCRIPT_ARGS; do
TLS_IMPL=""
TLS_LIBS=""
;;
--with-lmdb)
EDB_IMPL="EDB_LMDB"
EDB_LIBS="-llmdb"
;;
--disable-lmdb)
EDB_IMPL=""
EDB_LIBS=""
;;
--prefix=*)
PREFIX=$(echo "$arg" | cut -d '=' -f 2-)
;;
@ -112,11 +104,6 @@ if [ -n "$TLS_IMPL" ]; then
LIBS="${LIBS} ${TLS_LIBS}"
fi
if [ -n "$EDB_IMPL" ]; then
CFLAGS="${CFLAGS} -D${EDB_IMPL}"
LIBS="${LIBS} ${EDB_LIBS}"
fi
CFLAGS="${CFLAGS} '-DLIB_NAME=\"${LIB_NAME}\"' ${DEBUG}"
LDFLAGS="${LIBS} ${LDFLAGS}"
@ -153,7 +140,7 @@ print_obj() {
get_deps() {
src="$1"
${CC} -I${SRC} -I${INCLUDE} -E "$src" \
${CC} -I${INCLUDE} -E "$src" \
| grep '^#' \
| awk '{print $3}' \
| cut -d '"' -f 2 \
@ -269,10 +256,8 @@ ${TAB}done
${LIB_NAME}: ${OUT}/lib/lib${LIB_NAME}.a ${OUT}/lib/lib${LIB_NAME}.so
install: ${LIB_NAME}
${TAB}mkdir -p \$(PREFIX)/${OUT}/lib
${TAB}mkdir -p \$(PREFIX)/lib
${TAB}cp ${OUT}/lib/lib${LIB_NAME}.a \$(PREFIX)/lib/lib${LIB_NAME}.a
${TAB}cp ${OUT}/lib/lib${LIB_NAME}.so \$(PREFIX)/lib/lib${LIB_NAME}.so
${TAB}install -D ${OUT}/lib/lib${LIB_NAME}.a \$(PREFIX)/lib/lib${LIB_NAME}.a
${TAB}install -D ${OUT}/lib/lib${LIB_NAME}.so \$(PREFIX)/lib/lib${LIB_NAME}.so
$(collect ${INCLUDE}/ '' '' \$\(PREFIX\)/include/${LIB_NAME}/ install_out)
$(collect ${INCLUDE}/ .h .3 \$\(PREFIX\)/man/man3/${LIB_NAME}- install_man)
$(collect ${TOOLS}/ '.c' '' \$\(PREFIX\)/bin/ install_tool)

29
include/Cytoplasm/Db.h
View file

@ -48,17 +48,6 @@
*/
typedef struct Db Db;
/**
* Some "hints" for the database backend for operations.
* Hints are a way for the program to declare what to except of it
* (and the program MUST adhere to these hints, but the backend
* MAY adhere).
*/
typedef enum DbHint {
DB_HINT_READONLY, /* The database reference is treated as read-only */
DB_HINT_WRITE /* The database reference is treated as read/write */
} DbHint;
/**
* When an object is locked, a reference is returned. This reference
* is owned by the current thread, and the database is inaccessible to
@ -79,15 +68,6 @@ typedef struct DbRef DbRef;
*/
extern Db * DbOpen(char *, size_t);
/**
* Open a LMDB data directory. This function is similar to
* .Fn DbOpen ,
* but works with a LMDB-based backend, with the second argument
* being the maximum filesize. This function fails with NULL if ever
* called without LMDB enabled at compiletime.
*/
extern Db * DbOpenLMDB(char *, size_t);
/**
* Close the database. This function will flush anything in the cache
* to the disk, and then close the data directory. It assumes that
@ -129,15 +109,6 @@ extern DbRef * DbCreate(Db *, size_t,...);
*/
extern DbRef * DbLock(Db *, size_t,...);
/**
* Behaves like
* .Fn DbLock ,
* but with hints on the reference itself, as
* .Fn DbLock
* itself is read/write.
*/
extern DbRef * DbLockIntent(Db *, DbHint, size_t,...);
/**
* Immediately and permanently remove an object from the database.
* This function assumes the object is not locked, otherwise undefined

6
include/Cytoplasm/Json.h
View file

@ -297,12 +297,6 @@ extern size_t JsonEncode(HashMap *, Stream *, int);
*/
extern HashMap * JsonDecode(Stream *);
/**
* Decodes a JSON value from thr current input strram and parse it
* into a JsonValue.
*/
extern JsonValue * JsonValueDecode(Stream *);
/**
* A convenience function that allows the caller to retrieve and
* arbitrarily deep keys within a JSON object. It takes a root JSON

69
include/Cytoplasm/Platform.h
View file

@ -1,69 +0,0 @@
/*
* Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation files
* (the "Software"), to deal in the Software without restriction,
* including without limitation the rights to use, copy, modify, merge,
* publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef CYTOPLASM_PLATFORM_H
#define CYTOPLASM_PLATFORM_H
/***
* @Nm Platform
* @Nd A simple macro header that determines what platform the application
* is being built for.
* @Dd September 21, 2024
*/
#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
#define PLATFORM_WINDOWS
#ifdef _WIN64
#define PLATFORM_WIN64
#else
#define PLATFORM_WIN32
#endif
#elif __APPLE__
#define PLATFORM_DARWIN
#include <TargetConditionals.h>
#if TARGET_IPHONE_SIMULATOR
#define PLATFORM_IPHONE
#elif TARGET_OS_MACCATALYST
#define PLATFORM_CATALYST
#elif TARGET_OS_IPHONE
#define PLATFORM_IPHONE
#elif TARGET_OS_MAC
#define PLATFORM_MAC
#else
# error "Unknown Apple platform"
#endif
#elif __ANDROID__
#define PLATFORM_ANDROID
#elif __linux__
#define PLATFORM_LINUX
#elif __unix__
#define PLATFORM_UNIX
#elif defined(_POSIX_VERSION)
#define PLATFORM_POSIX
#else
# error "Unknown compiler"
#endif
#endif /* CYTOPLASM_PLATFORM_H */

26
include/Cytoplasm/Sha.h
View file

@ -41,16 +41,6 @@
* due to the lack of a 64-bit integer type, so that hash
* function has been omitted.
*/
#include <stddef.h>
/**
* This is an enum to be used to identify the type of SHA used.
*/
typedef enum HashType
{
HASH_SHA1,
HASH_SHA256
} HashType;
/**
* This function takes a pointer to a NULL-terminated C string, and
@ -74,20 +64,6 @@ extern unsigned char * Sha256(char *);
*/
extern unsigned char * Sha1(char *);
/**
* This function behaves just like
* .Fn Sha256 ,
* except that it allows for a generic byte array, instead of a string.
*/
extern unsigned char * Sha256Raw(unsigned char *, size_t);
/**
* This function behaves just like
* .Fn Sha1 ,
* except that it allows for a generic byte array, instead of a string.
*/
extern unsigned char * Sha1Raw(unsigned char *, size_t);
/**
* Convert a SHA byte buffer into a hex string. These hex strings
* are typically what is transmitted, stored, and compared, however
@ -95,6 +71,6 @@ extern unsigned char * Sha1Raw(unsigned char *, size_t);
* bytes directly, which is why the conversion to a hex string is
* a separate step.
*/
extern char * ShaToHex(unsigned char *, HashType);
extern char * ShaToHex(unsigned char *);
#endif /* CYTOPLASM_SHA_H */

977
src/Db.c Normal file
View file

@ -0,0 +1,977 @@
/*
* Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation files
* (the "Software"), to deal in the Software without restriction,
* including without limitation the rights to use, copy, modify, merge,
* publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <Db.h>
#include <Memory.h>
#include <Json.h>
#include <Util.h>
#include <Str.h>
#include <Stream.h>
#include <Log.h>
#include <sys/types.h>
#include <dirent.h>
#include <pthread.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
struct Db
{
char *dir;
pthread_mutex_t lock;
size_t cacheSize;
size_t maxCache;
HashMap *cache;
/*
* The cache uses a double linked list (see DbRef
* below) to know which objects are most and least
* recently used. The following diagram helps me
* know what way all the pointers go, because it
* can get very confusing sometimes. For example,
* there's nothing stopping "next" from pointing to
* least recent, and "prev" from pointing to most
* recent, so hopefully this clarifies the pointer
* terminology used when dealing with the linked
* list:
*
* mostRecent leastRecent
* | prev prev | prev
* +---+ ---> +---+ ---> +---+ ---> NULL
* |ref| |ref| |ref|
* NULL <--- +---+ <--- +---+ <--- +---+
* next next next
*/
DbRef *mostRecent;
DbRef *leastRecent;
};
struct DbRef
{
HashMap *json;
uint64_t ts;
size_t size;
Array *name;
DbRef *prev;
DbRef *next;
int fd;
Stream *stream;
};
static void
StringArrayFree(Array * arr)
{
size_t i;
for (i = 0; i < ArraySize(arr); i++)
{
Free(ArrayGet(arr, i));
}
ArrayFree(arr);
}
static ssize_t DbComputeSize(HashMap *);
static ssize_t
DbComputeSizeOfValue(JsonValue * val)
{
MemoryInfo *a;
ssize_t total = 0;
size_t i;
union
{
char *str;
Array *arr;
} u;
if (!val)
{
return -1;
}
a = MemoryInfoGet(val);
if (a)
{
total += MemoryInfoGetSize(a);
}
switch (JsonValueType(val))
{
case JSON_OBJECT:
total += DbComputeSize(JsonValueAsObject(val));
break;
case JSON_ARRAY:
u.arr = JsonValueAsArray(val);
a = MemoryInfoGet(u.arr);
if (a)
{
total += MemoryInfoGetSize(a);
}
for (i = 0; i < ArraySize(u.arr); i++)
{
total += DbComputeSizeOfValue(ArrayGet(u.arr, i));
}
break;
case JSON_STRING:
u.str = JsonValueAsString(val);
a = MemoryInfoGet(u.str);
if (a)
{
total += MemoryInfoGetSize(a);
}
break;
case JSON_NULL:
case JSON_INTEGER:
case JSON_FLOAT:
case JSON_BOOLEAN:
default:
/* These don't use any extra heap space */
break;
}
return total;
}
static ssize_t
DbComputeSize(HashMap * json)
{
char *key;
JsonValue *val;
MemoryInfo *a;
size_t total;
if (!json)
{
return -1;
}
total = 0;
a = MemoryInfoGet(json);
if (a)
{
total += MemoryInfoGetSize(a);
}
while (HashMapIterate(json, &key, (void **) &val))
{
a = MemoryInfoGet(key);
if (a)
{
total += MemoryInfoGetSize(a);
}
total += DbComputeSizeOfValue(val);
}
return total;
}
static char *
DbHashKey(Array * args)
{
size_t i;
char *str = NULL;
for (i = 0; i < ArraySize(args); i++)
{
char *tmp = StrConcat(2, str, ArrayGet(args, i));
Free(str);
str = tmp;
}
return str;
}
static char
DbSanitiseChar(char input)
{
switch (input)
{
case '/':
return '_';
case '.':
return '-';
}
return input;
}
static char *
DbDirName(Db * db, Array * args, size_t strip)
{
size_t i, j;
char *str = StrConcat(2, db->dir, "/");
for (i = 0; i < ArraySize(args) - strip; i++)
{
char *tmp;
char *sanitise = StrDuplicate(ArrayGet(args, i));
for (j = 0; j < strlen(sanitise); j++)
{
sanitise[j] = DbSanitiseChar(sanitise[j]);
}
tmp = StrConcat(3, str, sanitise, "/");
Free(str);
Free(sanitise);
str = tmp;
}
return str;
}
static char *
DbFileName(Db * db, Array * args)
{
size_t i;
char *str = StrConcat(2, db->dir, "/");
for (i = 0; i < ArraySize(args); i++)
{
char *tmp;
char *arg = StrDuplicate(ArrayGet(args, i));
size_t j = 0;
/* Sanitize name to prevent directory traversal attacks */
while (arg[j])
{
arg[j] = DbSanitiseChar(arg[j]);
j++;
}
tmp = StrConcat(3, str, arg,
(i < ArraySize(args) - 1) ? "/" : ".json");
Free(arg);
Free(str);
str = tmp;
}
return str;
}
static void
DbCacheEvict(Db * db)
{
DbRef *ref = db->leastRecent;
DbRef *tmp;
while (ref && db->cacheSize > db->maxCache)
{
char *hash;
JsonFree(ref->json);
hash = DbHashKey(ref->name);
HashMapDelete(db->cache, hash);
Free(hash);
StringArrayFree(ref->name);
db->cacheSize -= ref->size;
if (ref->next)
{
ref->next->prev = ref->prev;
}
else
{
db->mostRecent = ref->prev;
}
if (ref->prev)
{
ref->prev->next = ref->next;
}
else
{
db->leastRecent = ref->next;
}
tmp = ref->next;
Free(ref);
ref = tmp;
}
}
Db *
DbOpen(char *dir, size_t cache)
{
Db *db;
pthread_mutexattr_t attr;
if (!dir)
{
return NULL;
}
db = Malloc(sizeof(Db));
if (!db)
{
return NULL;
}
db->dir = dir;
db->maxCache = cache;
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&db->lock, &attr);
pthread_mutexattr_destroy(&attr);
db->mostRecent = NULL;
db->leastRecent = NULL;
db->cacheSize = 0;
if (db->maxCache)
{
db->cache = HashMapCreate();
if (!db->cache)
{
return NULL;
}
}
else
{
db->cache = NULL;
}
return db;
}
void
DbMaxCacheSet(Db * db, size_t cache)
{
if (!db)
{
return;
}
pthread_mutex_lock(&db->lock);
db->maxCache = cache;
if (db->maxCache && !db->cache)
{
db->cache = HashMapCreate();
db->cacheSize = 0;
}
DbCacheEvict(db);
pthread_mutex_unlock(&db->lock);
}
void
DbClose(Db * db)
{
if (!db)
{
return;
}
pthread_mutex_lock(&db->lock);
DbMaxCacheSet(db, 0);
DbCacheEvict(db);
HashMapFree(db->cache);
pthread_mutex_unlock(&db->lock);
pthread_mutex_destroy(&db->lock);
Free(db);
}
static DbRef *
DbLockFromArr(Db * db, Array * args)
{
char *file;
char *hash;
DbRef *ref;
struct flock lock;
int fd;
Stream *stream;
if (!db || !args)
{
return NULL;
}
ref = NULL;
hash = NULL;
pthread_mutex_lock(&db->lock);
/* Check if the item is in the cache */
hash = DbHashKey(args);
ref = HashMapGet(db->cache, hash);
file = DbFileName(db, args);
fd = open(file, O_RDWR);
if (fd == -1)
{
if (ref)
{
HashMapDelete(db->cache, hash);
JsonFree(ref->json);
StringArrayFree(ref->name);
db->cacheSize -= ref->size;
if (ref->next)
{
ref->next->prev = ref->prev;
}
else
{
db->mostRecent = ref->prev;
}
if (ref->prev)
{
ref->prev->next = ref->next;
}
else
{
db->leastRecent = ref->next;
}
if (!db->leastRecent)
{
db->leastRecent = db->mostRecent;
}
Free(ref);
}
ref = NULL;
goto finish;
}
stream = StreamFd(fd);
lock.l_start = 0;
lock.l_len = 0;
lock.l_type = F_WRLCK;
lock.l_whence = SEEK_SET;
/* Lock the file on the disk */
if (fcntl(fd, F_SETLK, &lock) < 0)
{
StreamClose(stream);
ref = NULL;
goto finish;
}
if (ref) /* In cache */
{
uint64_t diskTs = UtilLastModified(file);
ref->fd = fd;
ref->stream = stream;
if (diskTs > ref->ts)
{
/* File was modified on disk since it was cached */
HashMap *json = JsonDecode(ref->stream);
if (!json)
{
StreamClose(ref->stream);
ref = NULL;
goto finish;
}
JsonFree(ref->json);
ref->json = json;
ref->ts = diskTs;
ref->size = DbComputeSize(ref->json);
}
/* Float this ref to mostRecent */
if (ref->next)
{
ref->next->prev = ref->prev;
if (!ref->prev)
{
db->leastRecent = ref->next;
}
else
{
ref->prev->next = ref->next;
}
ref->prev = db->mostRecent;
ref->next = NULL;
if (db->mostRecent)
{
db->mostRecent->next = ref;
}
db->mostRecent = ref;
}
/* If there is no least recent, this is the only thing in the
* cache, so it is also least recent. */
if (!db->leastRecent)
{
db->leastRecent = ref;
}
/* The file on disk may be larger than what we have in memory,
* which may require items in cache to be evicted. */
DbCacheEvict(db);
}
else
{
Array *name;
size_t i;
/* Not in cache; load from disk */
ref = Malloc(sizeof(DbRef));
if (!ref)
{
StreamClose(stream);
goto finish;
}
ref->json = JsonDecode(stream);
if (!ref->json)
{
Free(ref);
StreamClose(stream);
ref = NULL;
goto finish;
}
ref->fd = fd;
ref->stream = stream;
name = ArrayCreate();
for (i = 0; i < ArraySize(args); i++)
{
ArrayAdd(name, StrDuplicate(ArrayGet(args, i)));
}
ref->name = name;
if (db->cache)
{
ref->ts = UtilTsMillis();
ref->size = DbComputeSize(ref->json);
HashMapSet(db->cache, hash, ref);
db->cacheSize += ref->size;
ref->next = NULL;
ref->prev = db->mostRecent;
if (db->mostRecent)
{
db->mostRecent->next = ref;
}
db->mostRecent = ref;
if (!db->leastRecent)
{
db->leastRecent = ref;
}
/* Adding this item to the cache may case it to grow too
* large, requiring some items to be evicted */
DbCacheEvict(db);
}
}
finish:
if (!ref)
{
pthread_mutex_unlock(&db->lock);
}
Free(file);
Free(hash);
return ref;
}
DbRef *
DbCreate(Db * db, size_t nArgs,...)
{
Stream *fp;
char *file;
char *dir;
va_list ap;
Array *args;
DbRef *ret;
if (!db)
{
return NULL;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return NULL;
}
pthread_mutex_lock(&db->lock);
file = DbFileName(db, args);
if (UtilLastModified(file))
{
Free(file);
ArrayFree(args);
pthread_mutex_unlock(&db->lock);
return NULL;
}
dir = DbDirName(db, args, 1);
if (UtilMkdir(dir, 0750) < 0)
{
Free(file);
ArrayFree(args);
Free(dir);
pthread_mutex_unlock(&db->lock);
return NULL;
}
Free(dir);
fp = StreamOpen(file, "w");
Free(file);
if (!fp)
{
ArrayFree(args);
pthread_mutex_unlock(&db->lock);
return NULL;
}
StreamPuts(fp, "{}");
StreamClose(fp);
/* DbLockFromArr() will lock again for us */
pthread_mutex_unlock(&db->lock);
ret = DbLockFromArr(db, args);
ArrayFree(args);
return ret;
}
bool
DbDelete(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
char *file;
char *hash;
bool ret = true;
DbRef *ref;
if (!db)
{
return false;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
pthread_mutex_lock(&db->lock);
hash = DbHashKey(args);
file = DbFileName(db, args);
ref = HashMapGet(db->cache, hash);
if (ref)
{
HashMapDelete(db->cache, hash);
JsonFree(ref->json);
StringArrayFree(ref->name);
db->cacheSize -= ref->size;
if (ref->next)
{
ref->next->prev = ref->prev;
}
else
{
db->mostRecent = ref->prev;
}
if (ref->prev)
{
ref->prev->next = ref->next;
}
else
{
db->leastRecent = ref->next;
}
if (!db->leastRecent)
{
db->leastRecent = db->mostRecent;
}
Free(ref);
}
Free(hash);
if (UtilLastModified(file))
{
ret = (remove(file) == 0);
}
pthread_mutex_unlock(&db->lock);
ArrayFree(args);
Free(file);
return ret;
}
DbRef *
DbLock(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
DbRef *ret;
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return NULL;
}
ret = DbLockFromArr(db, args);
ArrayFree(args);
return ret;
}
bool
DbUnlock(Db * db, DbRef * ref)
{
bool destroy;
if (!db || !ref)
{
return false;
}
lseek(ref->fd, 0L, SEEK_SET);
if (ftruncate(ref->fd, 0) < 0)
{
pthread_mutex_unlock(&db->lock);
Log(LOG_ERR, "Failed to truncate file on disk.");
Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno));
return false;
}
JsonEncode(ref->json, ref->stream, JSON_DEFAULT);
StreamClose(ref->stream);
if (db->cache)
{
char *key = DbHashKey(ref->name);
if (HashMapGet(db->cache, key))
{
db->cacheSize -= ref->size;
ref->size = DbComputeSize(ref->json);
db->cacheSize += ref->size;
/* If this ref has grown significantly since we last
* computed its size, it may have filled the cache and
* require some items to be evicted. */
DbCacheEvict(db);
destroy = false;
}
else
{
destroy = true;
}
Free(key);
}
else
{
destroy = true;
}
if (destroy)
{
JsonFree(ref->json);
StringArrayFree(ref->name);
Free(ref);
}
pthread_mutex_unlock(&db->lock);
return true;
}
bool
DbExists(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
char *file;
bool ret;
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return false;
}
pthread_mutex_lock(&db->lock);
file = DbFileName(db, args);
ret = (UtilLastModified(file) != 0);
pthread_mutex_unlock(&db->lock);
Free(file);
ArrayFree(args);
return ret;
}
Array *
DbList(Db * db, size_t nArgs,...)
{
Array *result;
Array *path;
DIR *files;
struct dirent *file;
char *dir;
va_list ap;
if (!db || !nArgs)
{
return NULL;
}
result = ArrayCreate();
if (!result)
{
return NULL;
}
va_start(ap, nArgs);
path = ArrayFromVarArgs(nArgs, ap);
dir = DbDirName(db, path, 0);
pthread_mutex_lock(&db->lock);
files = opendir(dir);
if (!files)
{
ArrayFree(path);
ArrayFree(result);
Free(dir);
pthread_mutex_unlock(&db->lock);
return NULL;
}
while ((file = readdir(files)))
{
size_t namlen = strlen(file->d_name);
if (namlen > 5)
{
int nameOffset = namlen - 5;
if (StrEquals(file->d_name + nameOffset, ".json"))
{
file->d_name[nameOffset] = '\0';
ArrayAdd(result, StrDuplicate(file->d_name));
}
}
}
closedir(files);
ArrayFree(path);
Free(dir);
pthread_mutex_unlock(&db->lock);
return result;
}
void
DbListFree(Array * arr)
{
StringArrayFree(arr);
}
HashMap *
DbJson(DbRef * ref)
{
return ref ? ref->json : NULL;
}
bool
DbJsonSet(DbRef * ref, HashMap * json)
{
if (!ref || !json)
{
return false;
}
JsonFree(ref->json);
ref->json = JsonDuplicate(json);
return true;
}

499
src/Db/Db.c
View file

@ -1,499 +0,0 @@
/*
* Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation files
* (the "Software"), to deal in the Software without restriction,
* including without limitation the rights to use, copy, modify, merge,
* publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <Db.h>
#include <Memory.h>
#include <Json.h>
#include <Util.h>
#include <Str.h>
#include <Stream.h>
#include <Log.h>
#include <sys/types.h>
#include <dirent.h>
#include <pthread.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include "Db/Internal.h"
void
StringArrayFree(Array * arr)
{
size_t i;
for (i = 0; i < ArraySize(arr); i++)
{
Free(ArrayGet(arr, i));
}
ArrayFree(arr);
}
static ssize_t DbComputeSize(HashMap *);
static ssize_t
DbComputeSizeOfValue(JsonValue * val)
{
MemoryInfo *a;
ssize_t total = 0;
size_t i;
union
{
char *str;
Array *arr;
} u;
if (!val)
{
return -1;
}
a = MemoryInfoGet(val);
if (a)
{
total += MemoryInfoGetSize(a);
}
switch (JsonValueType(val))
{
case JSON_OBJECT:
total += DbComputeSize(JsonValueAsObject(val));
break;
case JSON_ARRAY:
u.arr = JsonValueAsArray(val);
a = MemoryInfoGet(u.arr);
if (a)
{
total += MemoryInfoGetSize(a);
}
for (i = 0; i < ArraySize(u.arr); i++)
{
total += DbComputeSizeOfValue(ArrayGet(u.arr, i));
}
break;
case JSON_STRING:
u.str = JsonValueAsString(val);
a = MemoryInfoGet(u.str);
if (a)
{
total += MemoryInfoGetSize(a);
}
break;
case JSON_NULL:
case JSON_INTEGER:
case JSON_FLOAT:
case JSON_BOOLEAN:
default:
/* These don't use any extra heap space */
break;
}
return total;
}
static ssize_t
DbComputeSize(HashMap * json)
{
char *key;
JsonValue *val;
MemoryInfo *a;
size_t total;
if (!json)
{
return -1;
}
total = 0;
a = MemoryInfoGet(json);
if (a)
{
total += MemoryInfoGetSize(a);
}
while (HashMapIterate(json, &key, (void **) &val))
{
a = MemoryInfoGet(key);
if (a)
{
total += MemoryInfoGetSize(a);
}
total += DbComputeSizeOfValue(val);
}
return total;
}
static char *
DbHashKey(Array * args)
{
size_t i;
char *str = NULL;
for (i = 0; i < ArraySize(args); i++)
{
char *tmp = StrConcat(2, str, ArrayGet(args, i));
Free(str);
str = tmp;
}
return str;
}
static void
DbCacheEvict(Db * db)
{
DbRef *ref = db->leastRecent;
DbRef *tmp;
while (ref && db->cacheSize > db->maxCache)
{
char *hash;
JsonFree(ref->json);
hash = DbHashKey(ref->name);
HashMapDelete(db->cache, hash);
Free(hash);
StringArrayFree(ref->name);
db->cacheSize -= ref->size;
if (ref->next)
{
ref->next->prev = ref->prev;
}
else
{
db->mostRecent = ref->prev;
}
if (ref->prev)
{
ref->prev->next = ref->next;
}
else
{
db->leastRecent = ref->next;
}
tmp = ref->next;
Free(ref);
ref = tmp;
}
}
void
DbMaxCacheSet(Db * db, size_t cache)
{
if (!db)
{
return;
}
pthread_mutex_lock(&db->lock);
db->maxCache = cache;
if (db->maxCache && !db->cache)
{
db->cache = HashMapCreate();
db->cacheSize = 0;
}
DbCacheEvict(db);
pthread_mutex_unlock(&db->lock);
}
void
DbClose(Db * db)
{
if (!db)
{
return;
}
pthread_mutex_lock(&db->lock);
if (db->close)
{
db->close(db);
}
DbMaxCacheSet(db, 0);
DbCacheEvict(db);
HashMapFree(db->cache);
pthread_mutex_unlock(&db->lock);
pthread_mutex_destroy(&db->lock);
Free(db);
}
DbRef *
DbCreate(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
DbRef *ret;
if (!db || !db->create)
{
return NULL;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return NULL;
}
ret = db->create(db, args);
ArrayFree(args);
return ret;
}
bool
DbDelete(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
bool ret = true;
if (!db)
{
return false;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
ret = db->delete(db, args);
ArrayFree(args);
return ret;
}
DbRef *
DbLock(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
DbRef *ret;
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args || !db->lockFunc)
{
return NULL;
}
ret = db->lockFunc(db, DB_HINT_WRITE, args);
ArrayFree(args);
return ret;
}
DbRef *
DbLockIntent(Db * db, DbHint hint, size_t nArgs,...)
{
va_list ap;
Array *args;
DbRef *ret;
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args || !db->lockFunc)
{
return NULL;
}
ret = db->lockFunc(db, hint, args);
ArrayFree(args);
return ret;
}
bool
DbUnlock(Db * db, DbRef * ref)
{
if (!db || !ref || !db->unlock)
{
return false;
}
return db->unlock(db, ref);
}
bool
DbExists(Db * db, size_t nArgs,...)
{
va_list ap;
Array *args;
bool ret;
if (!db || !nArgs || !db->exists)
{
return false;
}
va_start(ap, nArgs);
args = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
if (!args)
{
return false;
}
ret = db->exists(db, args);
ArrayFree(args);
return ret;
}
Array *
DbList(Db * db, size_t nArgs,...)
{
Array *result;
Array *path;
va_list ap;
if (!db || !nArgs || !db->list)
{
return NULL;
}
va_start(ap, nArgs);
path = ArrayFromVarArgs(nArgs, ap);
va_end(ap);
result = db->list(db, path);
ArrayFree(path);
return result;
}
void
DbListFree(Array * arr)
{
StringArrayFree(arr);
}
HashMap *
DbJson(DbRef * ref)
{
return ref ? ref->json : NULL;
}
bool
DbJsonSet(DbRef * ref, HashMap * json)
{
if (!ref || !json)
{
return false;
}
JsonFree(ref->json);
ref->json = JsonDuplicate(json);
return true;
}
void
DbInit(Db *db)
{
pthread_mutexattr_t attr;
if (!db)
{
return;
}
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&db->lock, &attr);
pthread_mutexattr_destroy(&attr);
db->mostRecent = NULL;
db->leastRecent = NULL;
db->cacheSize = 0;
db->maxCache = 0;
if (db->maxCache)
{
db->cache = HashMapCreate();
}
else
{
db->cache = NULL;
}
}
void
DbRefInit(Db *db, DbRef *ref)
{
if (!db || !ref)
{
return;
}
ref->prev = db->mostRecent;
ref->next = NULL;
ref->json = NULL;
ref->name = NULL;
/* As default values to be overwritten by impls */
ref->ts = UtilTsMillis();
ref->size = 0;
/* TODO: Append the ref to the cache list.
* I removed it because it broke everything and crashed all the time.
* My bad! */
}
void
StringArrayAppend(Array *arr, char *str)
{
if (!arr || !str)
{
return;
}
ArrayAdd(arr, StrDuplicate(str));
}

374
src/Db/Flat.c
View file

@ -1,374 +0,0 @@
#include <Db.h>
#include <Memory.h>
#include <Json.h>
#include <Util.h>
#include <Str.h>
#include <Stream.h>
#include <Log.h>
#include <sys/types.h>
#include <dirent.h>
#include <pthread.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include "Db/Internal.h"
typedef struct FlatDb {
Db base;
char *dir;
/* Theres not much to do here. */
} FlatDb;
typedef struct FlatDbRef {
DbRef base;
Stream *stream;
int fd;
} FlatDbRef;
static char
DbSanitiseChar(char input)
{
switch (input)
{
case '/':
return '_';
case '.':
return '-';
}
return input;
}
static char *
DbDirName(FlatDb * db, Array * args, size_t strip)
{
size_t i, j;
char *str = StrConcat(2, db->dir, "/");
for (i = 0; i < ArraySize(args) - strip; i++)
{
char *tmp;
char *sanitise = StrDuplicate(ArrayGet(args, i));
for (j = 0; j < strlen(sanitise); j++)
{
sanitise[j] = DbSanitiseChar(sanitise[j]);
}
tmp = StrConcat(3, str, sanitise, "/");
Free(str);
Free(sanitise);
str = tmp;
}
return str;
}
static char *
DbFileName(FlatDb * db, Array * args)
{
size_t i;
char *str = StrConcat(2, db->dir, "/");
for (i = 0; i < ArraySize(args); i++)
{
char *tmp;
char *arg = StrDuplicate(ArrayGet(args, i));
size_t j = 0;
/* Sanitize name to prevent directory traversal attacks */
while (arg[j])
{
arg[j] = DbSanitiseChar(arg[j]);
j++;
}
tmp = StrConcat(3, str, arg,
(i < ArraySize(args) - 1) ? "/" : ".json");
Free(arg);
Free(str);
str = tmp;
}
return str;
}
static DbRef *
FlatLock(Db *d, DbHint hint, Array *dir)
{
FlatDb *db = (FlatDb *) d;
FlatDbRef *ref = NULL;
size_t i;
char *path = NULL;
if (!d || !dir)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
path = DbFileName(db, dir);
/* TODO: Caching */
{
int fd = open(path, O_RDWR);
Stream *stream;
struct flock lock;
if (fd == -1)
{
ref = NULL;
goto end;
}
stream = StreamFd(fd);
if (!stream)
{
ref = NULL;
goto end;
}
lock.l_start = 0;
lock.l_len = 0;
lock.l_type = F_WRLCK;
lock.l_whence = SEEK_SET;
if (fcntl(fd, F_SETLK, &lock) < 0)
{
StreamClose(stream);
ref = NULL;
goto end;
}
ref = Malloc(sizeof(*ref));
DbRefInit(d, (DbRef *) ref);
/* TODO: Hints */
ref->base.hint = hint;
ref->base.ts = UtilLastModified(path);
ref->base.json = JsonDecode(stream);
ref->stream = stream;
ref->fd = fd;
if (!ref->base.json)
{
Free(ref);
StreamClose(stream);
ref = NULL;
goto end;
}
ref->base.name = ArrayCreate();
for (i = 0; i < ArraySize(dir); i++)
{
StringArrayAppend(ref->base.name, ArrayGet(dir, i));
}
}
end:
Free(path);
if (!ref)
{
pthread_mutex_unlock(&d->lock);
}
return (DbRef *) ref;
}
static bool
FlatUnlock(Db *d, DbRef *r)
{
FlatDbRef *ref = (FlatDbRef *) r;
if (!d || !r)
{
return false;
}
lseek(ref->fd, 0L, SEEK_SET);
if (ftruncate(ref->fd, 0) < 0)
{
pthread_mutex_unlock(&d->lock);
Log(LOG_ERR, "Failed to truncate file on disk.");
Log(LOG_ERR, "Error on fd %d: %s", ref->fd, strerror(errno));
return false;
}
JsonEncode(ref->base.json, ref->stream, JSON_DEFAULT);
StreamClose(ref->stream);
JsonFree(ref->base.json);
StringArrayFree(ref->base.name);
Free(ref);
pthread_mutex_unlock(&d->lock);
return true;
}
static DbRef *
FlatCreate(Db *d, Array *dir)
{
FlatDb *db = (FlatDb *) d;
char *path, *dirPath;
Stream *fp;
DbRef *ret;
if (!d || !dir)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
path = DbFileName(db, dir);
if (UtilLastModified(path))
{
Free(path);
pthread_mutex_unlock(&d->lock);
return NULL;
}
dirPath = DbDirName(db, dir, 1);
if (UtilMkdir(dirPath, 0750) < 0)
{
Free(path);
Free(dirPath);
pthread_mutex_unlock(&d->lock);
return NULL;
}
Free(dirPath);
fp = StreamOpen(path, "w");
Free(path);
if (!fp)
{
pthread_mutex_unlock(&d->lock);
return NULL;
}
StreamPuts(fp, "{}");
StreamClose(fp);
/* FlatLock() will lock again for us */
pthread_mutex_unlock(&d->lock);
ret = FlatLock(d, DB_HINT_WRITE, dir);
return ret;
}
static bool
FlatDelete(Db *d, Array *dir)
{
bool ret = false;
char *file;
FlatDb *db = (FlatDb *) d;
if (!d || !dir)
{
return false;
}
pthread_mutex_lock(&d->lock);
file = DbFileName(db, dir);
/* TODO: Unlink the entry from the linkedlist */
if (UtilLastModified(file))
{
ret = remove(file) == 0;
}
Free(file);
pthread_mutex_unlock(&d->lock);
return ret;
}
static Array *
FlatList(Db *d, Array *dir)
{
FlatDb *db = (FlatDb *) d;
struct dirent *file;
Array *ret;
DIR *files;
char *path;
if (!d || !dir)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
path = DbDirName(db, dir, 0);
files = opendir(path);
if (!files)
{
Free(path);
pthread_mutex_unlock(&d->lock);
return NULL;
}
ret = ArrayCreate();
while ((file = readdir(files)))
{
size_t namlen = strlen(file->d_name);
if (namlen > 5)
{
int nameOffset = namlen - 5;
if (StrEquals(file->d_name + nameOffset, ".json"))
{
file->d_name[nameOffset] = '\0';
StringArrayAppend(ret, file->d_name);
}
}
}
closedir(files);
Free(path);
pthread_mutex_unlock(&d->lock);
return ret;
}
static bool
FlatExists(Db *d, Array *dir)
{
FlatDb *db = (FlatDb *) d;
char *path;
bool ret;
if (!d || !dir)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
path = DbFileName(db, dir);
ret = UtilLastModified(path) != 0;
Free(path);
pthread_mutex_unlock(&d->lock);
return ret;
}
Db *
DbOpen(char *dir, size_t cache)
{
FlatDb *db;
if (!dir)
{
return NULL;
}
db = Malloc(sizeof(*db));
DbInit((Db *) db);
db->dir = dir;
db->base.cacheSize = cache;
db->base.lockFunc = FlatLock;
db->base.unlock = FlatUnlock;
db->base.create = FlatCreate;
db->base.delete = FlatDelete;
db->base.exists = FlatExists;
db->base.list = FlatList;
db->base.close = NULL;
return (Db *) db;
}

101
src/Db/Internal.h
View file

@ -1,101 +0,0 @@
/*
* Copyright (C) 2022-2024 Jordan Bancino <@jordan:bancino.net>
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation files
* (the "Software"), to deal in the Software without restriction,
* including without limitation the rights to use, copy, modify, merge,
* publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef CYTOPLASM_DB_INTERNAL_H
#define CYTOPLASM_DB_INTERNAL_H
#include <HashMap.h>
#include <Db.h>
#include <pthread.h>
/* TODO: Define the base structures to define a database internally.
* All "implementations" shall have them as a first element, so that
* basic, general functions can work on them properly. */
/* The base structure of a database */
struct Db
{
pthread_mutex_t lock;
size_t cacheSize;
size_t maxCache;
HashMap *cache;
/*
* The cache uses a double linked list (see DbRef
* below) to know which objects are most and least
* recently used. The following diagram helps me
* know what way all the pointers go, because it
* can get very confusing sometimes. For example,
* there's nothing stopping "next" from pointing to
* least recent, and "prev" from pointing to most
* recent, so hopefully this clarifies the pointer
* terminology used when dealing with the linked
* list:
*
* mostRecent leastRecent
* | prev prev | prev
* +---+ ---> +---+ ---> +---+ ---> NULL
* |ref| |ref| |ref|
* NULL <--- +---+ <--- +---+ <--- +---+
* next next next
*/
DbRef *mostRecent;
DbRef *leastRecent;
/* Functions for implementation-specific operations
* (opening a ref, closing a db, removing an entry, ...) */
DbRef * (*lockFunc)(Db *, DbHint, Array *);
DbRef * (*create)(Db *, Array *);
Array * (*list)(Db *, Array *);
bool (*unlock)(Db *, DbRef *);
bool (*delete)(Db *, Array *);
bool (*exists)(Db *, Array *);
void (*close)(Db *);
/* Implementation-specific constructs */
};
struct DbRef
{
HashMap *json;
uint64_t ts;
size_t size;
Array *name;
DbRef *prev;
DbRef *next;
DbHint hint;
/* Implementation-specific constructs */
};
extern void DbInit(Db *);
extern void DbRefInit(Db *, DbRef *);
extern void StringArrayFree(Array *);
extern void StringArrayAppend(Array *, char *);
#endif

591
src/Db/LMDB.c
View file

@ -1,591 +0,0 @@
#include <Memory.h>
#include <Json.h>
#include <Log.h>
#include <Db.h>
#include "Db/Internal.h"
#include <stdint.h>
#include <string.h>
#include <stdlib.h>
#ifdef EDB_LMDB
#include <lmdb.h>
typedef struct LMDB {
Db base; /* The base implementation required to pass */
MDB_env *environ;
MDB_dbi dbi;
} LMDB;
typedef struct LMDBRef {
DbRef base;
/* TODO: LMDB-dependent stuff */
MDB_txn *transaction;
MDB_dbi dbi;
} LMDBRef;
/* Helper functions */
static MDB_val
LMDBTranslateKey(Array *key)
{
MDB_val ret = { 0 };
char *data = NULL;
size_t length = 0;
size_t i;
if (!key || ArraySize(key) > 255)
{
return ret;
}
data = Realloc(data, ++length);
data[0] = ArraySize(key);
/* Now, let's push every item */
for (i = 0; i < ArraySize(key); i++)
{
char *entry = ArrayGet(key, i);
size_t offset = length;
data = Realloc(data, (length += strlen(entry) + 1));
memcpy(data + offset, entry, strlen(entry));
data[length - 1] = '\0';
}
/* We now have every key */
ret.mv_size = length;
ret.mv_data = data;
return ret;
}
static void
LMDBKillKey(MDB_val key)
{
if (!key.mv_data || !key.mv_size)
{
return;
}
Free(key.mv_data);
}
static HashMap *
LMDBDecode(MDB_val val)
{
FILE *fakefile;
Stream *fakestream;
HashMap *ret;
if (!val.mv_data || !val.mv_size)
{
return NULL;
}
fakefile = fmemopen(val.mv_data, val.mv_size, "r");
fakestream = StreamFile(fakefile);
ret = JsonDecode(fakestream);
StreamClose(fakestream);
return ret;
}
static bool
LMDBKeyStartsWith(MDB_val key, MDB_val starts)
{
size_t i;
if (!key.mv_size || !starts.mv_size)
{
return false;
}
if (key.mv_size < starts.mv_size)
{
return false;
}
for (i = 0; i < starts.mv_size; i++)
{
char keyC = ((char *) key.mv_data)[i];
char startC = ((char *) starts.mv_data)[i];
if (keyC != startC)
{
return false;
}
}
return true;
}
static char *
LMDBKeyHead(MDB_val key)
{
char *end;
if (!key.mv_size || !key.mv_data)
{
return NULL;
}
/* -2 because we have a NULL byte in there */
end = ((char *) key.mv_data) + key.mv_size - 1;
if ((void *) end < key.mv_data)
{
/* Uh oh. */
return NULL;
}
while ((void *) (end - 1) >= key.mv_data && *(end - 1))
{
end--;
}
return end;
}
static DbRef *
LMDBLock(Db *d, DbHint hint, Array *k)
{
LMDB *db = (LMDB *) d;
MDB_txn *transaction;
LMDBRef *ret = NULL;
MDB_val key, json_val;
int code, flags;
if (!d || !k)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
key = LMDBTranslateKey(k);
/* Create a transaction, honoring hints. */
/* TODO: Do we want to create a "main" transaction that everyone inherits
* from? */
flags = hint == DB_HINT_READONLY ? MDB_RDONLY : 0;
if ((code = mdb_txn_begin(db->environ, NULL, flags, &transaction)) != 0)
{
/* Very bad! */
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
goto end;
}
json_val.mv_size = 0;
json_val.mv_data = NULL;
code = mdb_get(transaction, db->dbi, &key, &json_val);
if (code == MDB_NOTFOUND)
{
mdb_txn_abort(transaction);
goto end;
}
else if (code != 0)
{
Log(LOG_ERR,
"%s: mdb_get failure: %s",
__func__, mdb_strerror(code)
);
mdb_txn_abort(transaction);
goto end;
}
ret = Malloc(sizeof(*ret));
DbRefInit(d, (DbRef *) ret);
/* TODO: Timestamp */
{
size_t i;
ret->base.name = ArrayCreate();
for (i = 0; i < ArraySize(k); i++)
{
char *ent = ArrayGet(k, i);
StringArrayAppend(ret->base.name, ent);
}
}
ret->base.json = LMDBDecode(json_val);
ret->base.hint = hint;
ret->transaction = NULL;
if (hint == DB_HINT_WRITE)
{
ret->transaction = transaction;
}
else
{
mdb_txn_abort(transaction);
}
end:
if (!ret || hint == DB_HINT_READONLY)
{
pthread_mutex_unlock(&d->lock);
}
LMDBKillKey(key);
return (DbRef *) ret;
}
static bool
LMDBExists(Db *d, Array *k)
{
MDB_val key, empty;
LMDB *db = (LMDB *) d;
MDB_txn *transaction;
int code;
bool ret = false;
if (!d || !k)
{
return false;
}
pthread_mutex_lock(&d->lock);
key = LMDBTranslateKey(k);
/* create a txn */
if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0)
{
/* Very bad! */
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
goto end;
}
ret = mdb_get(transaction, db->dbi, &key, &empty) == 0;
mdb_txn_abort(transaction);
end:
LMDBKillKey(key);
pthread_mutex_unlock(&d->lock);
return ret;
}
static bool
LMDBDelete(Db *d, Array *k)
{
MDB_val key, empty;
LMDB *db = (LMDB *) d;
MDB_txn *transaction;
int code;
bool ret = false;
if (!d || !k)
{
return false;
}
pthread_mutex_lock(&d->lock);
key = LMDBTranslateKey(k);
/* create a txn */
if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0)
{
/* Very bad! */
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
goto end;
}
ret = mdb_del(transaction, db->dbi, &key, &empty) == 0;
mdb_txn_commit(transaction);
end:
LMDBKillKey(key);
pthread_mutex_unlock(&d->lock);
return ret;
}
static bool
LMDBUnlock(Db *d, DbRef *r)
{
LMDBRef *ref = (LMDBRef *) r;
LMDB *db = (LMDB *) d;
FILE *fakestream;
Stream *stream;
MDB_val key, val;
bool ret = true;
DbHint hint = r ? r->hint : 0;
if (!d || !r)
{
return false;
}
val.mv_data = NULL;
val.mv_size = 0;
if (ref->transaction && r->hint == DB_HINT_WRITE)
{
key = LMDBTranslateKey(r->name);
fakestream = open_memstream((char **) &val.mv_data, &val.mv_size);
stream = StreamFile(fakestream);
JsonEncode(r->json, stream, JSON_DEFAULT);
StreamFlush(stream);
StreamClose(stream);
ret = mdb_put(ref->transaction, db->dbi, &key, &val, 0) == 0;
mdb_txn_commit(ref->transaction);
LMDBKillKey(key);
}
StringArrayFree(ref->base.name);
JsonFree(ref->base.json);
Free(ref);
if (val.mv_data)
{
free(val.mv_data);
}
if (ret && hint == DB_HINT_WRITE)
{
pthread_mutex_unlock(&d->lock);
}
return ret;
}
static DbRef *
LMDBCreate(Db *d, Array *k)
{
LMDB *db = (LMDB *) d;
MDB_txn *transaction;
LMDBRef *ret = NULL;
MDB_val key, empty_json;
int code;
if (!d || !k)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
key = LMDBTranslateKey(k);
/* create a txn */
if ((code = mdb_txn_begin(db->environ, NULL, 0, &transaction)) != 0)
{
/* Very bad! */
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
goto end;
}
empty_json.mv_size = 2;
empty_json.mv_data = "{}";
/* put data in it */
code = mdb_put(transaction, db->dbi, &key, &empty_json, MDB_NOOVERWRITE);
if (code == MDB_KEYEXIST)
{
mdb_txn_abort(transaction);
goto end;
}
else if (code == MDB_MAP_FULL)
{
Log(LOG_ERR, "%s: db is full", __func__);
mdb_txn_abort(transaction);
goto end;
}
else if (code != 0)
{
Log(LOG_ERR, "%s: mdb_put failure: %s", __func__, mdb_strerror(code));
mdb_txn_abort(transaction);
goto end;
}
ret = Malloc(sizeof(*ret));
DbRefInit(d, (DbRef *) ret);
/* TODO: Timestamp */
{
size_t i;
ret->base.name = ArrayCreate();
for (i = 0; i < ArraySize(k); i++)
{
char *ent = ArrayGet(k, i);
StringArrayAppend(ret->base.name, ent);
}
}
ret->base.hint = DB_HINT_WRITE;
ret->base.json = HashMapCreate();
ret->transaction = transaction;
end:
if (!ret)
{
pthread_mutex_unlock(&d->lock);
}
LMDBKillKey(key);
return (DbRef *) ret;
}
static Array *
LMDBList(Db *d, Array *k)
{
LMDB *db = (LMDB *) d;
MDB_val key = { 0 };
MDB_val subKey;
MDB_val val;
Array *ret = NULL;
MDB_cursor *cursor;
MDB_cursor_op op = MDB_SET_RANGE;
MDB_txn *txn;
int code;
if (!d || !k)
{
return NULL;
}
pthread_mutex_lock(&d->lock);
/* Marked as read-only, as we just don't need to write anything
* when listing */
if ((code = mdb_txn_begin(db->environ, NULL, MDB_RDONLY, &txn)) != 0)
{
/* Very bad! */
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
goto end;
}
if ((code = mdb_cursor_open(txn, db->dbi, &cursor)) != 0)
{
Log(LOG_ERR,
"%s: could not get cursor: %s",
__func__, mdb_strerror(code)
);
mdb_txn_abort(txn);
goto end;
}
key = LMDBTranslateKey(k);
/* Small hack to get it to list subitems */
((char *) key.mv_data)[0]++;
ret = ArrayCreate();
subKey = key;
while (mdb_cursor_get(cursor, &subKey, &val, op) == 0)
{
/* This searches by *increasing* order. The problem is that it may
* extend to unwanted points. Since the values are sorted, we can
* just exit if the subkey is incorrect. */
char *head = LMDBKeyHead(subKey);
if (!LMDBKeyStartsWith(subKey, key))
{
break;
}
StringArrayAppend(ret, head);
op = MDB_NEXT;
}
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
end:
LMDBKillKey(key);
pthread_mutex_unlock(&d->lock);
return ret;
}
/* Implementation functions */
static void
LMDBClose(Db *d)
{
LMDB *db = (LMDB *) d;
if (!d)
{
return;
}
mdb_dbi_close(db->environ, db->dbi);
mdb_env_close(db->environ);
}
Db *
DbOpenLMDB(char *dir, size_t size)
{
/* TODO */
MDB_env *env = NULL;
MDB_txn *txn;
MDB_dbi dbi;
int code;
LMDB *db;
if (!dir || !size)
{
return NULL;
}
/* Try initialising LMDB */
if ((code = mdb_env_create(&env)) != 0)
{
Log(LOG_ERR,
"%s: could not create LMDB env: %s",
__func__, mdb_strerror(code)
);
return NULL;
}
if ((code = mdb_env_set_mapsize(env, size) != 0))
{
Log(LOG_ERR,
"%s: could not set mapsize to %lu: %s",
__func__, (unsigned long) size,
mdb_strerror(code)
);
mdb_env_close(env);
return NULL;
}
mdb_env_set_maxdbs(env, 4);
if ((code = mdb_env_open(env, dir, MDB_NOTLS, 0644)) != 0)
{
Log(LOG_ERR,
"%s: could not open LMDB env: %s",
__func__, mdb_strerror(code)
);
mdb_env_close(env);
return NULL;
}
/* Initialise a DBI */
{
if ((code = mdb_txn_begin(env, NULL, 0, &txn)) != 0)
{
Log(LOG_ERR,
"%s: could not begin transaction: %s",
__func__, mdb_strerror(code)
);
mdb_env_close(env);
return NULL;
}
if ((code = mdb_dbi_open(txn, "db", MDB_CREATE, &dbi)) != 0)
{
Log(LOG_ERR,
"%s: could not get transaction dbi: %s",
__func__, mdb_strerror(code)
);
mdb_txn_abort(txn);
mdb_env_close(env);
return NULL;
}
mdb_txn_commit(txn);
}
db = Malloc(sizeof(*db));
DbInit((Db *) db);
db->environ = env;
db->dbi = dbi;
db->base.lockFunc = LMDBLock;
db->base.create = LMDBCreate;
db->base.unlock = LMDBUnlock;
db->base.delete = LMDBDelete;
db->base.exists = LMDBExists;
db->base.close = LMDBClose;
db->base.list = LMDBList;
return (Db *) db;
}
#else
Db *
DbOpenLMDB(char *dir, size_t size)
{
/* Unimplemented function */
Log(LOG_ERR,
"LMDB support is not enabled. Please compile with --use-lmdb"
);
(void) size;
(void) dir;
return NULL;
}
#endif

19
src/Json.c
View file

@ -1348,25 +1348,6 @@ JsonDecode(Stream * stream)
return result;
}
JsonValue *
JsonValueDecode(Stream *stream)
{
JsonValue *result;
JsonParserState state;
state.stream = stream;
state.token = NULL;
JsonTokenSeek(&state);
result = JsonDecodeValue(&state);
if (state.token)
{
Free(state.token);
}
return result;
}
JsonValue *
JsonGet(HashMap * json, size_t nArgs,...)

274
src/Memory.c
View file

@ -40,54 +40,29 @@
#define MEMORY_HEXDUMP_WIDTH 16
#endif
#define MEMORY_FILE_SIZE 256
#define MEM_BOUND_TYPE uint64_t
#define MEM_BOUND 0xDEADBEEFBEEFDEAD
#define MEM_MAGIC 0xDEADBEEFDEADBEEF
struct MemoryInfo
{
uint64_t magic;
size_t size;
size_t unalignedSize;
char file[MEMORY_FILE_SIZE];
const char *file;
int line;
void *pointer;
MemoryInfo *prev;
MemoryInfo *next;
MEM_BOUND_TYPE leftBoundary;
};
#define MEM_SIZE_ACTUAL(x) (MemoryAlignBoundary((x) * sizeof(uint8_t)) + sizeof(MEM_BOUND_TYPE))
#define MEM_START_BOUNDARY(info) (info->leftBoundary)
#define MEM_END_BOUNDARY(info) (*(((MEM_BOUND_TYPE *) (((uint8_t *) info->pointer) + info->size)) - 1))
#define MEM_BOUND_TYPE uint32_t
#define MEM_BOUND 0xDEADBEEF
#define MEM_BOUND_LOWER(p) *((MEM_BOUND_TYPE *) p)
#define MEM_BOUND_UPPER(p, x) *(((MEM_BOUND_TYPE *) (((uint8_t *) p) + x)) + 1)
#define MEM_SIZE_ACTUAL(x) (((x) * sizeof(uint8_t)) + (2 * sizeof(MEM_BOUND_TYPE)))
static pthread_mutex_t lock;
static void (*hook) (MemoryAction, MemoryInfo *, void *) = MemoryDefaultHook;
static void *hookArgs = NULL;
static MemoryInfo **allocations = NULL;
static size_t allocationsSize = 0;
static size_t allocationsLen = 0;
static MemoryInfo *allocationTail = NULL;
/* Simple range of "plausible" boundaries for heap, serving as an extra
* check */
static void *heapStart, *heapEnd;
static size_t MemoryAlignBoundary(size_t size)
{
size_t boundSize = sizeof(MEM_BOUND_TYPE);
size_t remainder = size % boundSize;
size_t closest = size / boundSize + !!remainder;
return closest * boundSize;
}
int
MemoryRuntimeInit(void)
{
@ -102,8 +77,6 @@ MemoryRuntimeInit(void)
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
ret = pthread_mutex_init(&lock, &attr);
pthread_mutexattr_destroy(&attr);
heapStart = NULL;
heapEnd = NULL;
ret = (ret == 0);
@ -118,27 +91,70 @@ MemoryRuntimeDestroy(void)
return pthread_mutex_destroy(&lock) == 0;
}
static size_t
MemoryHash(void *p)
{
return (((size_t) p) >> 2 * 7) % allocationsSize;
}
static int
MemoryInsert(MemoryInfo * a)
{
if (allocationTail)
{
allocationTail->next = a;
}
a->next = NULL;
a->prev = allocationTail;
a->magic = MEM_MAGIC;
size_t hash;
if (!heapStart || heapStart > (void *) a)
if (!allocations)
{
heapStart = a;
}
if (!heapEnd || heapEnd < (void *) a)
{
heapEnd = a;
allocationsSize = MEMORY_TABLE_CHUNK;
allocations = calloc(allocationsSize, sizeof(void *));
if (!allocations)
{
return 0;
}
}
allocationTail = a;
/* If the next insertion would cause the table to be at least 3/4
* full, re-allocate and re-hash. */
if ((allocationsLen + 1) >= ((allocationsSize * 3) >> 2))
{
size_t i;
size_t tmpAllocationsSize = allocationsSize;
MemoryInfo **tmpAllocations;
allocationsSize += MEMORY_TABLE_CHUNK;
tmpAllocations = calloc(allocationsSize, sizeof(void *));
if (!tmpAllocations)
{
return 0;
}
for (i = 0; i < tmpAllocationsSize; i++)
{
if (allocations[i])
{
hash = MemoryHash(allocations[i]->pointer);
while (tmpAllocations[hash])
{
hash = (hash + 1) % allocationsSize;
}
tmpAllocations[hash] = allocations[i];
}
}
free(allocations);
allocations = tmpAllocations;
}
hash = MemoryHash(a->pointer);
while (allocations[hash])
{
hash = (hash + 1) % allocationsSize;
}
allocations[hash] = a;
allocationsLen++;
return 1;
@ -147,32 +163,30 @@ MemoryInsert(MemoryInfo * a)
static void
MemoryDelete(MemoryInfo * a)
{
MemoryInfo *aPrev = a->prev;
MemoryInfo *aNext = a->next;
size_t hash = MemoryHash(a->pointer);
size_t count = 0;
if (aPrev)
while (count <= allocationsSize)
{
aPrev->next = aNext;
if (allocations[hash] && allocations[hash] == a)
{
allocations[hash] = NULL;
allocationsLen--;
return;
}
else
{
hash = (hash + 1) % allocationsSize;
count++;
}
}
if (aNext)
{
aNext->prev = aPrev;
}
if (a == allocationTail)
{
allocationTail = aPrev;
}
a->magic = ~MEM_MAGIC;
}
static int
MemoryCheck(MemoryInfo * a)
{
if (MEM_START_BOUNDARY(a) != MEM_BOUND ||
a->magic != MEM_MAGIC ||
MEM_END_BOUNDARY(a) != MEM_BOUND)
if (MEM_BOUND_LOWER(a->pointer) != MEM_BOUND ||
MEM_BOUND_UPPER(a->pointer, a->size - (2 * sizeof(MEM_BOUND_TYPE))) != MEM_BOUND)
{
if (hook)
{
@ -189,32 +203,38 @@ MemoryAllocate(size_t size, const char *file, int line)
void *p;
MemoryInfo *a;
//MemoryIterate(NULL, NULL);
MemoryIterate(NULL, NULL);
pthread_mutex_lock(&lock);
a = malloc(sizeof(MemoryInfo) + MEM_SIZE_ACTUAL(size));
a = malloc(sizeof(MemoryInfo));
if (!a)
{
pthread_mutex_unlock(&lock);
return NULL;
}
p = a + 1;
p = malloc(MEM_SIZE_ACTUAL(size));
if (!p)
{
free(a);
pthread_mutex_unlock(&lock);
return NULL;
}
memset(p, 0, MEM_SIZE_ACTUAL(size));
MEM_BOUND_LOWER(p) = MEM_BOUND;
MEM_BOUND_UPPER(p, size) = MEM_BOUND;
a->size = MEM_SIZE_ACTUAL(size);
a->unalignedSize = size;
strncpy(a->file, file, MEMORY_FILE_SIZE - 1);
a->file = file;
a->line = line;
a->pointer = p;
MEM_START_BOUNDARY(a) = MEM_BOUND;
MEM_END_BOUNDARY(a) = MEM_BOUND;
if (!MemoryInsert(a))
{
free(a);
free(p);
pthread_mutex_unlock(&lock);
return NULL;
}
@ -225,7 +245,7 @@ MemoryAllocate(size_t size, const char *file, int line)
}
pthread_mutex_unlock(&lock);
return p;
return ((MEM_BOUND_TYPE *) p) + 1;
}
void *
@ -234,7 +254,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line)
MemoryInfo *a;
void *new = NULL;
//MemoryIterate(NULL, NULL);
MemoryIterate(NULL, NULL);
if (!p)
{
@ -245,28 +265,25 @@ MemoryReallocate(void *p, size_t size, const char *file, int line)
if (a)
{
pthread_mutex_lock(&lock);
MemoryDelete(a);
new = realloc(a, sizeof(MemoryInfo) + MEM_SIZE_ACTUAL(size));
new = realloc(a->pointer, MEM_SIZE_ACTUAL(size));
if (new)
{
a = new;
a->unalignedSize = size;
MemoryDelete(a);
a->size = MEM_SIZE_ACTUAL(size);
strncpy(a->file, file, MEMORY_FILE_SIZE - 1);
a->file = file;
a->line = line;
a->magic = MEM_MAGIC;
a->pointer = a + 1;
a->pointer = new;
MemoryInsert(a);
MEM_START_BOUNDARY(a) = MEM_BOUND;
MEM_END_BOUNDARY(a) = MEM_BOUND;
MEM_BOUND_LOWER(a->pointer) = MEM_BOUND;
MEM_BOUND_UPPER(a->pointer, size) = MEM_BOUND;
if (hook)
{
hook(MEMORY_REALLOCATE, a, hookArgs);
}
}
pthread_mutex_unlock(&lock);
}
@ -276,7 +293,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line)
if (a)
{
a->size = 0;
strncpy(a->file, file, MEMORY_FILE_SIZE - 1);
a->file = file;
a->line = line;
a->pointer = p;
hook(MEMORY_BAD_POINTER, a, hookArgs);
@ -284,7 +301,7 @@ MemoryReallocate(void *p, size_t size, const char *file, int line)
}
}
return a->pointer;
return ((MEM_BOUND_TYPE *) new) + 1;
}
void
@ -292,7 +309,7 @@ MemoryFree(void *p, const char *file, int line)
{
MemoryInfo *a;
//MemoryIterate(NULL, NULL);
MemoryIterate(NULL, NULL);
if (!p)
{
@ -305,11 +322,12 @@ MemoryFree(void *p, const char *file, int line)
pthread_mutex_lock(&lock);
if (hook)
{
strncpy(a->file, file, MEMORY_FILE_SIZE - 1);
a->file = file;
a->line = line;
hook(MEMORY_FREE, a, hookArgs);
}
MemoryDelete(a);
free(a->pointer);
free(a);
pthread_mutex_unlock(&lock);
@ -319,7 +337,7 @@ MemoryFree(void *p, const char *file, int line)
a = malloc(sizeof(MemoryInfo));
if (a)
{
strncpy(a->file, file, MEMORY_FILE_SIZE - 1);
a->file = file;
a->line = line;
a->size = 0;
a->pointer = p;
@ -332,15 +350,17 @@ MemoryFree(void *p, const char *file, int line)
size_t
MemoryAllocated(void)
{
size_t i;
size_t total = 0;
MemoryInfo *cur;
pthread_mutex_lock(&lock);
/* TODO */
for (cur = allocationTail; cur; cur = cur->prev)
for (i = 0; i < allocationsSize; i++)
{
total += MemoryInfoGetSize(cur);
if (allocations[i])
{
total += MemoryInfoGetSize(allocations[i]);
}
}
pthread_mutex_unlock(&lock);
@ -351,44 +371,55 @@ MemoryAllocated(void)
void
MemoryFreeAll(void)
{
MemoryInfo *cur;
MemoryInfo *prev;
size_t i;
pthread_mutex_lock(&lock);
/* TODO */
for (cur = allocationTail; cur; cur = prev)
for (i = 0; i < allocationsSize; i++)
{
prev = cur->prev;
free(cur);
if (allocations[i])
{
free(allocations[i]->pointer);
free(allocations[i]);
}
}
free(allocations);
allocations = NULL;
allocationsSize = 0;
allocationsLen = 0;
pthread_mutex_unlock(&lock);
}
MemoryInfo *
MemoryInfoGet(void *po)
MemoryInfoGet(void *p)
{
void *p = po;
size_t hash, count;
pthread_mutex_lock(&lock);
p = ((MemoryInfo *) p) - 1;
if (p < heapStart || p > heapEnd)
{
pthread_mutex_unlock(&lock);
return NULL;
}
p = ((MEM_BOUND_TYPE *) p) - 1;
hash = MemoryHash(p);
if (((MemoryInfo *)p)->magic != MEM_MAGIC)
count = 0;
while (count <= allocationsSize)
{
p = NULL;
if (!allocations[hash] || allocations[hash]->pointer != p)
{
hash = (hash + 1) % allocationsSize;
count++;
}
else
{
MemoryCheck(allocations[hash]);
pthread_mutex_unlock(&lock);
return allocations[hash];
}
}
pthread_mutex_unlock(&lock);
return p;
return NULL;
}
size_t
@ -399,7 +430,7 @@ MemoryInfoGetSize(MemoryInfo * a)
return 0;
}
return a->size ? a->unalignedSize : 0;
return a->size ? a->size - (2 * sizeof(MEM_BOUND_TYPE)) : 0;
}
const char *
@ -432,22 +463,25 @@ MemoryInfoGetPointer(MemoryInfo * a)
return NULL;
}
return a->pointer;
return ((MEM_BOUND_TYPE *) a->pointer) + 1;
}
void
MemoryIterate(void (*iterFunc) (MemoryInfo *, void *), void *args)
MemoryIterate(void (*iterFunc) (MemoryInfo *, void *), void *args)
{
MemoryInfo *cur;
size_t i;
pthread_mutex_lock(&lock);
for (cur = allocationTail; cur; cur = cur->prev)
for (i = 0; i < allocationsSize; i++)
{
MemoryCheck(cur);
if (iterFunc)
if (allocations[i])
{
iterFunc(cur, args);
MemoryCheck(allocations[i]);
if (iterFunc)
{
iterFunc(allocations[i], args);
}
}
}

33
src/Sha.c
View file

@ -28,44 +28,21 @@
#include <string.h>
char *
ShaToHex(unsigned char *bytes, HashType type)
ShaToHex(unsigned char *bytes)
{
size_t i = 0, size;
char *str;
size_t i = 0;
char *str = Malloc(((strlen((char *) bytes) * 2) + 1) * sizeof(char));
switch (type)
{
case HASH_SHA1:
size = 20;
break;
case HASH_SHA256:
size = 32;
break;
default:
return NULL;
}
str = Malloc(((size * 2) + 1) * sizeof(char));
if (!str)
{
return NULL;
}
for (i = 0; i < size; i++)
while (bytes[i] != '\0')
{
snprintf(str + (2 * i), 3, "%02x", bytes[i]);
i++;
}
return str;
}
unsigned char *
Sha256(char *str)
{
return Sha256Raw((unsigned char *) str, str ? strlen(str) : 0);
}
unsigned char *
Sha1(char *str)
{
return Sha1Raw((unsigned char *) str, str ? strlen(str) : 0);
}

26
src/Sha/Sha1.c
View file

@ -28,27 +28,6 @@
#include <limits.h>
/* TODO: Verify LibreSSL support later */
#if defined(TLS_IMPL) && (TLS_IMPL == TLS_OPENSSL)
#include <openssl/sha.h>
unsigned char *
Sha1Raw(unsigned char *str, size_t len)
{
unsigned char *digest;
if (!str)
{
return NULL;
}
digest = Malloc(20 + 1);
SHA1(str, len, digest);
digest[20] = '\0';
return digest;
}
#else
#define LOAD32H(x, y) \
{ \
x = ((uint32_t)((y)[0] & 255) << 24) | \
@ -261,7 +240,7 @@ Sha1Calculate(Sha1Context * ctx, unsigned char *out)
}
unsigned char *
Sha1Raw(unsigned char *str, size_t len)
Sha1(char *str)
{
Sha1Context ctx;
unsigned char *out;
@ -278,11 +257,10 @@ Sha1Raw(unsigned char *str, size_t len)
}
Sha1Init(&ctx);
Sha1Update(&ctx, str, len);
Sha1Update(&ctx, str, strlen(str));
Sha1Calculate(&ctx, out);
out[160 / 8] = '\0';
return out;
}
#endif

29
src/Sha/Sha256.c
View file

@ -21,36 +21,14 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <Memory.h>
#include <Sha.h>
#include <Memory.h>
#include <stdio.h>
#include <string.h>
#include <limits.h>
/* TODO: Verify LibreSSL support later */
#if defined(TLS_IMPL) && (TLS_IMPL == TLS_OPENSSL)
#include <openssl/sha.h>
unsigned char *
Sha256Raw(unsigned char *str, size_t len)
{
unsigned char *digest;
if (!str)
{
return NULL;
}
digest = Malloc(32 + 1);
SHA256(str, len, digest);
digest[32] = '\0';
return digest;
}
#else
#define GET_UINT32(x) \
(((uint32_t)(x)[0] << 24) | \
((uint32_t)(x)[1] << 16) | \
@ -192,7 +170,7 @@ Sha256Process(Sha256Context * context, unsigned char *data, size_t length)
}
unsigned char *
Sha256Raw(unsigned char *str, size_t len)
Sha256(char *str)
{
Sha256Context context;
size_t i;
@ -228,7 +206,7 @@ Sha256Raw(unsigned char *str, size_t len)
context.length = 0;
memset(context.buffer, 0, 64);
Sha256Process(&context, str, len);
Sha256Process(&context, (unsigned char *) str, strlen(str));
memset(fill, 0, 64);
fill[0] = 0x80;
@ -252,4 +230,3 @@ Sha256Raw(unsigned char *str, size_t len)
return out;
}
#endif

2
src/Tls/TlsLibreSSL.c
View file

@ -23,7 +23,7 @@
*/
#include <Tls.h>
#if defined(TLS_IMPL) && (TLS_IMPL == TLS_LIBRESSL)
#if TLS_IMPL == TLS_LIBRESSL
#include <Memory.h>
#include <Log.h>

2
src/Tls/TlsOpenSSL.c
View file

@ -23,7 +23,7 @@
*/
#include <Tls.h>
#if defined(TLS_IMPL) && (TLS_IMPL == TLS_OPENSSL)
#if TLS_IMPL == TLS_OPENSSL
#include <Memory.h>
#include <Log.h>

9
src/Util.c
View file

@ -24,7 +24,6 @@
#include <Util.h>
#include <Memory.h>
#include <Platform.h>
#include <stdio.h>
#include <stdlib.h>
@ -90,10 +89,6 @@ UtilTsMillis(void)
return ts;
}
#ifdef PLATFORM_DARWIN
#define st_mtim st_mtimespec
#endif
uint64_t
UtilLastModified(char *path)
{
@ -110,10 +105,6 @@ UtilLastModified(char *path)
return ts;
}
#ifdef PLATFORM_DARWIN
#undef st_mtim
#endif
int
UtilMkdir(const char *dir, const mode_t mode)
{

271
tools/j2s.c
View file

@ -38,7 +38,6 @@
#define MAX_DEPENDENCIES 32
#define IsDelimited(field, c1, c2) (*field == c1 && field[strlen(field) - 1] == c2)
static char *
Trim(char c, char *str)
{
@ -76,14 +75,10 @@ TypeToJsonType(char *type)
}
else
{
if (IsDelimited(type, '[', ']'))
if (*type == '[' && type[strlen(type) - 1] == ']')
{
return JSON_ARRAY;
}
else if (IsDelimited(type, '{', '}'))
{
return JSON_OBJECT;
}
else
{
return JSON_OBJECT;
@ -96,7 +91,7 @@ JsonTypeToStr(JsonType type)
{
switch (type)
{
case JSON_OBJECT:
case JSON_OBJECT:
return "JSON_OBJECT";
case JSON_ARRAY:
return "JSON_ARRAY";
@ -330,7 +325,6 @@ Main(Array * args)
{
char *fieldType;
bool isArrType = false;
bool isObjType = false;
JsonValue *requiredVal;
JsonValue *ignoreVal;
@ -349,18 +343,12 @@ Main(Array * args)
goto finish;
}
if (IsDelimited(fieldType, '[', ']'))
if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']')
{
fieldType++;
fieldType[strlen(fieldType) - 1] = '\0';
isArrType = true;
}
else if (IsDelimited(fieldType, '{', '}'))
{
fieldType++;
fieldType[strlen(fieldType) - 1] = '\0';
isObjType = true;
}
if (!StrEquals(fieldType, "object") &&
!StrEquals(fieldType, "array") &&
@ -386,10 +374,6 @@ Main(Array * args)
{
fieldType[strlen(fieldType)] = ']';
}
else if (isObjType)
{
fieldType[strlen(fieldType)] = '}';
}
requiredVal = HashMapGet(fieldObj, "required");
if (requiredVal && JsonValueType(requiredVal) != JSON_BOOLEAN)
@ -545,13 +529,11 @@ Main(Array * args)
{
cType = "double";
}
else if (StrEquals(fieldType, "object") ||
IsDelimited(fieldType, '{', '}'))
else if (StrEquals(fieldType, "object"))
{
cType = "HashMap *";
}
else if (StrEquals(fieldType, "array") ||
IsDelimited(fieldType, '[', ']'))
else if (StrEquals(fieldType, "array") || (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']'))
{
cType = "Array *";
}
@ -560,20 +542,7 @@ Main(Array * args)
cType = fieldType;
}
if (IsDelimited(fieldType, '{', '}') ||
IsDelimited(fieldType, '[', ']'))
{
char end = fieldType[strlen(fieldType) - 1];
fieldType[strlen(fieldType) - 1] = '\0';
StreamPrintf(headerFile, " %s /* of %s */ %s;\n", cType, fieldType + 1, field);
fieldType[strlen(fieldType)] = end;
}
else
{
StreamPrintf(headerFile, " %s %s;\n", cType, field);
}
StreamPrintf(headerFile, " %s %s;\n", cType, field);
}
StreamPrintf(headerFile, "} %s;\n\n", type);
@ -704,134 +673,7 @@ Main(Array * args)
StreamPrintf(implFile, " out->%s = JsonValueAsObject(val);\n", key);
StreamPrintf(implFile, " Free(val); /* Not JsonValueFree() because we want the inner value. */\n");
}
else if (IsDelimited(fieldType, '{', '}'))
{
fieldType++;
fieldType[strlen(fieldType) - 1] = '\0';
isEnum = StrEquals(JsonValueAsString(JsonGet(types, 2, fieldType, "type")), "enum");
jsonType = isEnum ? JSON_STRING : TypeToJsonType(fieldType);
StreamPrintf(implFile, " out->%s = HashMapCreate();\n", key);
StreamPrintf(implFile, " if (!out->%s)\n", key);
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " *errp = \"Failed to allocate memory for %s.%s.\";\n", type, key);
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " else\n");
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " HashMap *obj = JsonValueAsObject(val);\n");
StreamPrintf(implFile, " char *objKey;\n");
StreamPrintf(implFile, " JsonValue *v;\n");
StreamPrintf(implFile, "\n");
StreamPrintf(implFile, " while (HashMapIterate(obj, &objKey, (void **) &v))\n");
StreamPrintf(implFile, " {\n");
if (StrEquals(fieldType, "integer") ||
StrEquals(fieldType, "float") ||
StrEquals(fieldType, "boolean"))
{
char *cType;
if (StrEquals(fieldType, "integer"))
{
cType = "int64_t";
}
else if (StrEquals(fieldType, "float"))
{
cType = "double";
}
else if (StrEquals(fieldType, "boolean"))
{
cType = "bool";
}
else
{
/* Should never happen */
cType = NULL;
}
*fieldType = toupper(*fieldType);
StreamPrintf(implFile, " %s *ref;\n", cType);
StreamPrintf(implFile, " if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType));
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " *errp = \"%s.%s{} contains an invalid value.\";\n", type, key);
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " ref = Malloc(sizeof(%s));\n", cType);
StreamPrintf(implFile, " if (!ref)\n");
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " *errp = \"Unable to allocate memory for object value.\";\n");
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " *ref = JsonValueAs%s(v);\n", fieldType);
StreamPrintf(implFile, " HashMapSet(out->%s, objKey, ref);\n", key);
*fieldType = tolower(*fieldType);
}
else if (StrEquals(fieldType, "string"))
{
StreamPrintf(implFile, " if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType));
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " *errp = \"%s.%s[] contains an invalid value.\";\n", type, key);
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " HashMapSet(out->%s, objKey, StrDuplicate(JsonValueAsString(v)));\n", key);
}
else if (StrEquals(fieldType, "object"))
{
StreamPrintf(implFile, " if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType));
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " *errp = \"%s.%s[] contains an invalid value.\";\n", type, key);
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " HashMapSet(out->%s, objKey, JsonDuplicate(JsonValueAsObject(v)));\n", key);
}
else
{
if (isEnum)
{
StreamPrintf(implFile, " int parseResult;\n");
}
StreamPrintf(implFile, " %s *parsed;\n", fieldType);
StreamPrintf(implFile, " if (JsonValueType(v) != %s)\n", JsonTypeToStr(jsonType));
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " *errp = \"%s.%s[] contains an invalid value.\";\n", type, key);
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " parsed = Malloc(sizeof(%s));\n", fieldType);
StreamPrintf(implFile, " if (!parsed)\n");
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " *errp = \"Unable to allocate memory for array value.\";\n");
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
if (isEnum)
{
StreamPrintf(implFile, " parseResult = %sFromStr(JsonValueAsString(v));\n", fieldType);
StreamPrintf(implFile, " *parsed = parseResult;\n");
StreamPrintf(implFile, " if (parseResult == -1)\n");
}
else
{
StreamPrintf(implFile, " if (!%sFromJson(JsonValueAsObject(v), parsed, errp))\n", fieldType);
}
StreamPrintf(implFile, " {\n");
if (!isEnum)
{
StreamPrintf(implFile, " %sFree(parsed);\n", fieldType);
}
StreamPrintf(implFile, " Free(parsed);\n");
StreamPrintf(implFile, " return false;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " HashMapSet(out->%s, objKey, parsed);\n", key);
}
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " }\n");
fieldType[strlen(fieldType)] = '}';
}
else if (IsDelimited(fieldType, '[', ']'))
else if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']')
{
fieldType++;
fieldType[strlen(fieldType) - 1] = '\0';
@ -1073,73 +915,7 @@ Main(Array * args)
{
StreamPrintf(implFile, " HashMapSet(json, \"%s\", JsonValueObject(JsonDuplicate(val->%s)));\n", Trim('_', key), key);
}
else if (IsDelimited(fieldType, '{', '}'))
{
int isPrimitive;
fieldType++;
fieldType[strlen(fieldType) - 1] = '\0';
isEnum = StrEquals(JsonValueAsString(JsonGet(types, 2, fieldType, "type")), "enum");
isPrimitive = StrEquals(fieldType, "integer") ||
StrEquals(fieldType, "boolean") ||
StrEquals(fieldType, "float");
StreamPrintf(implFile, " if (val->%s)\n", key);
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " char *objKey;\n");
StreamPrintf(implFile, " void *objVal;\n");
StreamPrintf(implFile, " HashMap *jsonObj = HashMapCreate();\n");
StreamPrintf(implFile, " if (!jsonObj)\n");
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " JsonFree(json);\n");
StreamPrintf(implFile, " return NULL;\n");
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " while (HashMapIterate(val->%s, &objKey, &objVal))\n", key);
StreamPrintf(implFile, " {\n");
if (StrEquals(fieldType, "string"))
{
StreamPrintf(implFile, " HashMapSet(jsonObj, objKey, JsonValueString(objVal));\n", key);
}
else if (!isPrimitive)
{
StreamPrintf(implFile, " HashMapSet(jsonObj, objKey, JsonValueObject(%sToJson(objVal)));\n", fieldType, key);
}
else
{
char *cType;
if (StrEquals(fieldType, "integer"))
{
cType = "int64_t";
}
else if (StrEquals(fieldType, "float"))
{
cType = "double";
}
else if (StrEquals(fieldType, "boolean"))
{
cType = "bool";
}
else
{
/* Should never happen */
cType = NULL;
}
*fieldType = toupper(*fieldType);
StreamPrintf(implFile, " HashMapSet(jsonObj, objKey, (JsonValue%s(*((%s *) objVal))));\n", fieldType, cType, key);
*fieldType = tolower(*fieldType);
}
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " HashMapSet(json, \"%s\", JsonValueObject(jsonObj));\n", Trim('_', key));
StreamPrintf(implFile, " }\n");
fieldType[strlen(fieldType)] = '}';
}
else if (IsDelimited(fieldType, '[', ']'))
else if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']')
{
int isPrimitive;
@ -1260,36 +1036,7 @@ Main(Array * args)
{
StreamPrintf(implFile, " Free(val->%s);\n", key);
}
else if (IsDelimited(fieldType, '{', '}'))
{
int isPrimitive;
fieldType++;
fieldType[strlen(fieldType) - 1] = '\0';
isEnum = StrEquals(JsonValueAsString(JsonGet(types, 2, fieldType, "type")), "enum");
isPrimitive = StrEquals(fieldType, "boolean") ||
StrEquals(fieldType, "float") ||
StrEquals(fieldType, "integer") ||
StrEquals(fieldType, "string");
StreamPrintf(implFile, " if (val->%s)\n", key);
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " char *objKey;\n");
StreamPrintf(implFile, " void *objVal;\n");
StreamPrintf(implFile, " while (HashMapIterate(val->%s, &objKey, &objVal))\n", key);
StreamPrintf(implFile, " {\n");
StreamPrintf(implFile, " %sFree(objVal);\n", (!isEnum && !isPrimitive) ? fieldType : "", key);
if (!isEnum && !isPrimitive)
{
StreamPrintf(implFile, " Free(objVal);\n", key);
}
StreamPrintf(implFile, " }\n");
StreamPrintf(implFile, " HashMapFree(val->%s);\n", key);
StreamPrintf(implFile, " }\n");
fieldType[strlen(fieldType)] = '}';
}
else if (IsDelimited(fieldType, '[', ']'))
else if (*fieldType == '[' && fieldType[strlen(fieldType) - 1] == ']')
{
int isPrimitive;