mirror of
https://github.com/shadowsocks/shadowsocks-libev.git
synced 2026-10-24 17:33:46 +00:00
Rewrite the cache on a fixed slab without per-entry allocation
cache.cpp keeps its C API but replaces uthash and the two mallocs per entry with one slab of nodes holding their keys inline, linked in last-use order, and an open-addressing index of node ids that hashes through the slab with a per-cache random seed. Both blocks are reserved by cache_create() and touched only as entries are used, so a large, mostly empty cache costs no resident memory. Semantics are unchanged: an insert evicts the least recently used entry once the cache holds capacity entries (the AEAD-2022 session table's 4095/4096 boundary), lookups refresh recency and age, cache_clear() drops stale entries, and free_cb may re-enter the cache. Re-inserting a key now replaces its data instead of shadowing it. cache_create_keyed() sizes keys per cache; the AEAD-2022 salt pool and session table use it. test_cache_cpp covers the boundary, ordering, replacement, ageing, re-entrant callbacks and limits; alloc_steady_cache requires zero allocations across 100,000 inserts on a full cache. AEAD-2022 ss-server drops from 84 to 80 allocations per relayed connection.
This commit is contained in:
+306
-220
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* cache.c - Manage the connection cache for UDPRELAY
|
||||
* cache.cpp - Fixed-capacity LRU cache for the UDP relay and AEAD-2022
|
||||
*
|
||||
* Copyright (C) 2013 - 2019, Max Lv <max.c.lv@gmail.com>
|
||||
*
|
||||
@@ -20,293 +20,379 @@
|
||||
* <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Original Author: Oliver Lorenz (ol), olli@olorenz.org, https://olorenz.org
|
||||
* License: This is licensed under the same terms as uthash itself
|
||||
*/
|
||||
// Layout: one slab of `capacity` nodes, each holding its key inline, linked
|
||||
// in least-recently-used order, plus an open-addressing index of node ids
|
||||
// (linear probing, backward-shift deletion, at most half full). Keys are
|
||||
// stored once, in the slab; the index hashes through it. Both blocks are
|
||||
// reserved by cache_create() and only touched as entries are used: nodes
|
||||
// are handed out in order before recycled ones, and the index stores
|
||||
// node id + 1 so that zero-filled memory reads as empty.
|
||||
|
||||
#ifdef HAVE_CONFIG_H
|
||||
#include "config.h"
|
||||
#endif
|
||||
|
||||
#include <errno.h>
|
||||
#include <stdlib.h>
|
||||
#include <cerrno>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <new>
|
||||
|
||||
#include <sodium.h>
|
||||
|
||||
#include "cache.h"
|
||||
#include "core/fixed_map.hpp"
|
||||
#include "core/region.hpp"
|
||||
#include "platform.h"
|
||||
#include "utils.h"
|
||||
|
||||
/** Creates a new cache object
|
||||
*
|
||||
* @param dst
|
||||
* Where the newly allocated cache object will be stored in
|
||||
*
|
||||
* @param capacity
|
||||
* The maximum number of elements this cache object can hold
|
||||
*
|
||||
* @return EINVAL if dst is NULL, ENOMEM if malloc fails, 0 otherwise
|
||||
*/
|
||||
namespace {
|
||||
|
||||
constexpr std::uint32_t kNone = UINT32_MAX;
|
||||
|
||||
struct Node {
|
||||
std::uint32_t prev = kNone;
|
||||
std::uint32_t next = kNone;
|
||||
double ts = 0;
|
||||
void *data = nullptr;
|
||||
std::uint16_t key_len = 0;
|
||||
bool used = false;
|
||||
// key bytes (max_key_len + 1, NUL-terminated for free_cb) follow
|
||||
};
|
||||
|
||||
std::size_t align_up(std::size_t n, std::size_t a)
|
||||
{
|
||||
return (n + a - 1) & ~(a - 1);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
struct cache {
|
||||
std::size_t capacity = 0; // as requested; eviction threshold
|
||||
std::size_t max_key_len = 0;
|
||||
std::size_t stride = 0; // bytes per node, key included
|
||||
void (*free_cb)(void *key, void *element) = nullptr;
|
||||
ss::Region nodes;
|
||||
ss::Region index; // node id + 1 per slot, 0 when empty
|
||||
std::uint32_t node_count = 0; // slab size (capacity, at least 1)
|
||||
std::uint32_t fresh = 0; // nodes [fresh, node_count) were never used
|
||||
std::uint32_t mask = 0; // index slots - 1
|
||||
std::uint32_t size = 0;
|
||||
std::uint32_t free_head = kNone;
|
||||
std::uint32_t oldest = kNone;
|
||||
std::uint32_t newest = kNone;
|
||||
std::uint64_t seed = 0;
|
||||
|
||||
Node *node(std::uint32_t id) const
|
||||
{
|
||||
return reinterpret_cast<Node *>(nodes.data() + static_cast<std::size_t>(id) * stride);
|
||||
}
|
||||
char *key(std::uint32_t id) const { return reinterpret_cast<char *>(node(id)) + sizeof(Node); }
|
||||
std::uint32_t *slots() const { return reinterpret_cast<std::uint32_t *>(index.data()); }
|
||||
bool empty_slot(std::uint32_t i) const { return slots()[i] == 0; }
|
||||
std::uint32_t at(std::uint32_t i) const { return slots()[i] - 1; }
|
||||
void put(std::uint32_t i, std::uint32_t id) const { slots()[i] = id + 1; }
|
||||
|
||||
std::uint32_t home(const char *k, std::size_t len) const
|
||||
{
|
||||
return static_cast<std::uint32_t>(ss::hash_bytes(k, len, seed)) & mask;
|
||||
}
|
||||
|
||||
bool matches(std::uint32_t id, const char *k, std::size_t len) const
|
||||
{
|
||||
const Node *n = node(id);
|
||||
return n->key_len == len && std::memcmp(key(id), k, len) == 0;
|
||||
}
|
||||
|
||||
// Index slot holding the key, or the empty slot that ends its probe run.
|
||||
std::uint32_t probe(const char *k, std::size_t len, bool &found) const
|
||||
{
|
||||
std::uint32_t i = home(k, len);
|
||||
while (!empty_slot(i)) {
|
||||
if (matches(at(i), k, len)) {
|
||||
found = true;
|
||||
return i;
|
||||
}
|
||||
i = (i + 1) & mask;
|
||||
}
|
||||
found = false;
|
||||
return i;
|
||||
}
|
||||
|
||||
void unindex(std::uint32_t hole)
|
||||
{
|
||||
slots()[hole] = 0;
|
||||
for (std::uint32_t j = (hole + 1) & mask; !empty_slot(j); j = (j + 1) & mask) {
|
||||
std::uint32_t id = at(j);
|
||||
std::uint32_t k = home(key(id), node(id)->key_len);
|
||||
bool movable = (hole <= j) ? (k <= hole || k > j) : (k <= hole && k > j);
|
||||
if (movable) {
|
||||
put(hole, id);
|
||||
slots()[j] = 0;
|
||||
hole = j;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void unlink(std::uint32_t id)
|
||||
{
|
||||
Node *n = node(id);
|
||||
if (n->prev != kNone) {
|
||||
node(n->prev)->next = n->next;
|
||||
} else {
|
||||
oldest = n->next;
|
||||
}
|
||||
if (n->next != kNone) {
|
||||
node(n->next)->prev = n->prev;
|
||||
} else {
|
||||
newest = n->prev;
|
||||
}
|
||||
n->prev = n->next = kNone;
|
||||
}
|
||||
|
||||
void link_newest(std::uint32_t id)
|
||||
{
|
||||
Node *n = node(id);
|
||||
n->prev = newest;
|
||||
n->next = kNone;
|
||||
if (newest != kNone) {
|
||||
node(newest)->next = id;
|
||||
} else {
|
||||
oldest = id;
|
||||
}
|
||||
newest = id;
|
||||
}
|
||||
|
||||
void touch(std::uint32_t id)
|
||||
{
|
||||
node(id)->ts = ss_monotonic_time();
|
||||
if (id != newest) {
|
||||
unlink(id);
|
||||
link_newest(id);
|
||||
}
|
||||
}
|
||||
|
||||
// Unlink the entry in index slot `slot` and release its data. The node
|
||||
// is back on the free list before free_cb runs, so free_cb may use the
|
||||
// cache.
|
||||
void drop(std::uint32_t slot, bool release_data)
|
||||
{
|
||||
std::uint32_t id = at(slot);
|
||||
Node *n = node(id);
|
||||
char key_copy[256];
|
||||
std::size_t len = n->key_len;
|
||||
std::memcpy(key_copy, key(id), len);
|
||||
key_copy[len] = '\0';
|
||||
void *data = n->data;
|
||||
|
||||
unindex(slot);
|
||||
unlink(id);
|
||||
n->used = false;
|
||||
n->data = nullptr;
|
||||
n->next = free_head;
|
||||
free_head = id;
|
||||
--size;
|
||||
|
||||
if (release_data && data != nullptr) {
|
||||
if (free_cb != nullptr) {
|
||||
free_cb(key_copy, data);
|
||||
} else {
|
||||
std::free(data); // heap-ok: caller-owned data, per the C API contract
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void drop_id(std::uint32_t id, bool release_data)
|
||||
{
|
||||
bool found = false;
|
||||
std::uint32_t slot = probe(key(id), node(id)->key_len, found);
|
||||
if (!found) {
|
||||
std::abort(); // index and slab disagree
|
||||
}
|
||||
drop(slot, release_data);
|
||||
}
|
||||
};
|
||||
|
||||
extern "C" {
|
||||
|
||||
int
|
||||
cache_create_keyed(struct cache **dst, size_t capacity, size_t max_key_len,
|
||||
void (*free_cb)(void *key, void *element))
|
||||
{
|
||||
if (dst == nullptr || max_key_len == 0 || max_key_len > 255 || capacity > (UINT32_MAX >> 2)) {
|
||||
return EINVAL;
|
||||
}
|
||||
auto *c = static_cast<struct cache *>(std::malloc(sizeof(struct cache))); // heap-ok: created at startup
|
||||
if (c == nullptr) {
|
||||
return ENOMEM;
|
||||
}
|
||||
new (c) struct cache();
|
||||
c->capacity = capacity;
|
||||
c->max_key_len = max_key_len;
|
||||
c->free_cb = free_cb;
|
||||
c->node_count = capacity == 0 ? 1 : static_cast<std::uint32_t>(capacity);
|
||||
c->stride = align_up(sizeof(Node) + max_key_len + 1, alignof(Node));
|
||||
std::uint32_t slots = 2;
|
||||
while (slots < c->node_count * 2U) {
|
||||
slots <<= 1;
|
||||
}
|
||||
c->mask = slots - 1;
|
||||
c->seed = (static_cast<std::uint64_t>(randombytes_random()) << 32) | randombytes_random();
|
||||
|
||||
auto nodes = ss::Region::reserve(static_cast<std::size_t>(c->node_count) * c->stride, alignof(Node));
|
||||
auto index = ss::Region::reserve(static_cast<std::size_t>(slots) * sizeof(std::uint32_t), alignof(std::uint32_t));
|
||||
if (!nodes || !index) {
|
||||
c->~cache();
|
||||
std::free(c); // heap-ok: failed startup reservation
|
||||
return ENOMEM;
|
||||
}
|
||||
c->nodes = std::move(*nodes);
|
||||
c->index = std::move(*index);
|
||||
*dst = c;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
cache_create(struct cache **dst, const size_t capacity,
|
||||
void (*free_cb)(void *key, void *element))
|
||||
{
|
||||
struct cache *created = NULL;
|
||||
|
||||
if (!dst) {
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
if ((created = (struct cache *)malloc(sizeof(*created))) == NULL) {
|
||||
return ENOMEM;
|
||||
}
|
||||
|
||||
created->max_entries = capacity;
|
||||
created->entries = NULL;
|
||||
created->free_cb = free_cb;
|
||||
*dst = created;
|
||||
return 0;
|
||||
return cache_create_keyed(dst, capacity, CACHE_DEFAULT_MAX_KEY, free_cb);
|
||||
}
|
||||
|
||||
/** Frees an allocated cache object
|
||||
*
|
||||
* @param cache
|
||||
* The cache object to free
|
||||
*
|
||||
* @param keep_data
|
||||
* Whether to free contained data or just delete references to it
|
||||
*
|
||||
* @return EINVAL if cache is NULL, 0 otherwise
|
||||
*/
|
||||
int
|
||||
cache_delete(struct cache *cache, int keep_data)
|
||||
{
|
||||
struct cache_entry *entry, *tmp;
|
||||
|
||||
if (!cache) {
|
||||
if (cache == nullptr) {
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
if (keep_data) {
|
||||
HASH_CLEAR(hh, cache->entries);
|
||||
} else {
|
||||
HASH_ITER(hh, cache->entries, entry, tmp){
|
||||
// NOLINTNEXTLINE(clang-analyzer-unix.Malloc): uthash unlinks here; entry is freed afterwards
|
||||
HASH_DEL(cache->entries, entry);
|
||||
if (entry->data != NULL) {
|
||||
if (cache->free_cb) {
|
||||
cache->free_cb(entry->key, entry->data);
|
||||
} else {
|
||||
ss_free(entry->data);
|
||||
}
|
||||
}
|
||||
ss_free(entry->key);
|
||||
ss_free(entry);
|
||||
}
|
||||
while (cache->oldest != kNone) {
|
||||
cache->drop_id(cache->oldest, !keep_data);
|
||||
}
|
||||
|
||||
ss_free(cache);
|
||||
cache->~cache();
|
||||
std::free(cache); // heap-ok: created at startup
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Clear old cache object
|
||||
*
|
||||
* @param cache
|
||||
* The cache object to clear
|
||||
*
|
||||
* @param age
|
||||
* Clear only objects older than the age (sec)
|
||||
*
|
||||
* @return EINVAL if cache is NULL, 0 otherwise
|
||||
*/
|
||||
int
|
||||
cache_clear(struct cache *cache, double age)
|
||||
{
|
||||
struct cache_entry *entry, *tmp;
|
||||
|
||||
if (!cache) {
|
||||
if (cache == nullptr) {
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
double now = ss_monotonic_time();
|
||||
|
||||
HASH_ITER(hh, cache->entries, entry, tmp){
|
||||
if (now - entry->ts > age) {
|
||||
// NOLINTNEXTLINE(clang-analyzer-unix.Malloc): uthash unlinks here; entry is freed afterwards
|
||||
HASH_DEL(cache->entries, entry);
|
||||
if (entry->data != NULL) {
|
||||
if (cache->free_cb) {
|
||||
cache->free_cb(entry->key, entry->data);
|
||||
} else {
|
||||
ss_free(entry->data);
|
||||
}
|
||||
}
|
||||
ss_free(entry->key);
|
||||
ss_free(entry);
|
||||
}
|
||||
// Entries are in last-use order, so the stale ones are at the front.
|
||||
while (cache->oldest != kNone && now - cache->node(cache->oldest)->ts > age) {
|
||||
cache->drop_id(cache->oldest, true);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Removes a cache entry
|
||||
*
|
||||
* @param cache
|
||||
* The cache object
|
||||
*
|
||||
* @param key
|
||||
* The key of the entry to remove
|
||||
*
|
||||
* @param key_len
|
||||
* The length of key
|
||||
*
|
||||
* @return EINVAL if cache is NULL, 0 otherwise
|
||||
*/
|
||||
int
|
||||
cache_remove(struct cache *cache, char *key, size_t key_len)
|
||||
{
|
||||
struct cache_entry *tmp;
|
||||
|
||||
if (!cache || !key) {
|
||||
if (cache == nullptr || key == nullptr) {
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
HASH_FIND(hh, cache->entries, key, key_len, tmp);
|
||||
|
||||
if (tmp) {
|
||||
HASH_DEL(cache->entries, tmp);
|
||||
if (tmp->data != NULL) {
|
||||
if (cache->free_cb) {
|
||||
cache->free_cb(tmp->key, tmp->data);
|
||||
} else {
|
||||
ss_free(tmp->data);
|
||||
}
|
||||
}
|
||||
ss_free(tmp->key);
|
||||
ss_free(tmp);
|
||||
bool found = false;
|
||||
std::uint32_t slot = cache->probe(key, key_len, found);
|
||||
if (found) {
|
||||
cache->drop(slot, true);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Checks if a given key is in the cache
|
||||
*
|
||||
* @param cache
|
||||
* The cache object
|
||||
*
|
||||
* @param key
|
||||
* The key to look-up
|
||||
*
|
||||
* @param key_len
|
||||
* The length of key
|
||||
*
|
||||
* @param result
|
||||
* Where to store the result if key is found.
|
||||
*
|
||||
* A warning: Even though result is just a pointer,
|
||||
* you have to call this function with a **ptr,
|
||||
* otherwise this will blow up in your face.
|
||||
*
|
||||
* @return EINVAL if cache is NULL, 0 otherwise
|
||||
*/
|
||||
int
|
||||
cache_lookup(struct cache *cache, char *key, size_t key_len, void *result)
|
||||
{
|
||||
struct cache_entry *tmp = NULL;
|
||||
void **out = (void **)result;
|
||||
|
||||
if (!cache || !key || !result) {
|
||||
if (cache == nullptr || key == nullptr || result == nullptr) {
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
HASH_FIND(hh, cache->entries, key, key_len, tmp);
|
||||
if (tmp) {
|
||||
HASH_DELETE(hh, cache->entries, tmp);
|
||||
tmp->ts = ss_monotonic_time();
|
||||
// NOLINTNEXTLINE(clang-analyzer-core.DivideZero): uthash bucket count is never zero
|
||||
HASH_ADD_KEYPTR(hh, cache->entries, tmp->key, key_len, tmp);
|
||||
*out = tmp->data;
|
||||
} else {
|
||||
*out = NULL;
|
||||
void **out = static_cast<void **>(result);
|
||||
bool found = false;
|
||||
std::uint32_t slot = cache->probe(key, key_len, found);
|
||||
if (!found) {
|
||||
*out = nullptr;
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::uint32_t id = cache->at(slot);
|
||||
cache->touch(id);
|
||||
*out = cache->node(id)->data;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
cache_key_exist(struct cache *cache, char *key, size_t key_len)
|
||||
{
|
||||
struct cache_entry *tmp = NULL;
|
||||
|
||||
if (!cache || !key) {
|
||||
if (cache == nullptr || key == nullptr) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
HASH_FIND(hh, cache->entries, key, key_len, tmp);
|
||||
if (tmp) {
|
||||
HASH_DELETE(hh, cache->entries, tmp);
|
||||
tmp->ts = ss_monotonic_time();
|
||||
// NOLINTNEXTLINE(clang-analyzer-core.DivideZero): uthash bucket count is never zero
|
||||
HASH_ADD_KEYPTR(hh, cache->entries, tmp->key, key_len, tmp);
|
||||
return 1;
|
||||
bool found = false;
|
||||
std::uint32_t slot = cache->probe(key, key_len, found);
|
||||
if (!found) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 0;
|
||||
cache->touch(cache->at(slot));
|
||||
return 1;
|
||||
}
|
||||
|
||||
/** Inserts a given <key, value> pair into the cache
|
||||
*
|
||||
* @param cache
|
||||
* The cache object
|
||||
*
|
||||
* @param key
|
||||
* The key that identifies <value>
|
||||
*
|
||||
* @param key_len
|
||||
* The length of key
|
||||
*
|
||||
* @param data
|
||||
* Data associated with <key>
|
||||
*
|
||||
* @return EINVAL if cache is NULL, ENOMEM if malloc fails, 0 otherwise
|
||||
*/
|
||||
int
|
||||
cache_insert(struct cache *cache, char *key, size_t key_len, void *data)
|
||||
{
|
||||
struct cache_entry *entry = NULL;
|
||||
struct cache_entry *tmp_entry = NULL;
|
||||
|
||||
if (!cache) {
|
||||
if (cache == nullptr || key == nullptr || key_len > cache->max_key_len) {
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
if ((entry = (struct cache_entry *)malloc(sizeof(*entry))) == NULL) {
|
||||
return ENOMEM;
|
||||
}
|
||||
|
||||
entry->key = (char *)ss_malloc(key_len + 1);
|
||||
memcpy(entry->key, key, key_len);
|
||||
entry->key[key_len] = 0;
|
||||
|
||||
entry->data = data;
|
||||
entry->ts = ss_monotonic_time();
|
||||
// NOLINTNEXTLINE(clang-analyzer-core.DivideZero): uthash bucket count is never zero
|
||||
HASH_ADD_KEYPTR(hh, cache->entries, entry->key, key_len, entry);
|
||||
|
||||
if (HASH_COUNT(cache->entries) >= cache->max_entries) {
|
||||
HASH_ITER(hh, cache->entries, entry, tmp_entry){
|
||||
HASH_DELETE(hh, cache->entries, entry);
|
||||
if (entry->data != NULL) {
|
||||
if (cache->free_cb) {
|
||||
cache->free_cb(entry->key, entry->data);
|
||||
} else {
|
||||
ss_free(entry->data);
|
||||
}
|
||||
bool found = false;
|
||||
std::uint32_t slot = cache->probe(key, key_len, found);
|
||||
if (found) {
|
||||
// Re-inserting a key replaces its data; the old data is released
|
||||
// as if the entry had been evicted.
|
||||
std::uint32_t id = cache->at(slot);
|
||||
Node *n = cache->node(id);
|
||||
void *old = n->data;
|
||||
n->data = data;
|
||||
cache->touch(id);
|
||||
if (old != nullptr && old != data) {
|
||||
char key_copy[256];
|
||||
std::memcpy(key_copy, key, key_len);
|
||||
key_copy[key_len] = '\0';
|
||||
if (cache->free_cb != nullptr) {
|
||||
cache->free_cb(key_copy, old);
|
||||
} else {
|
||||
std::free(old); // heap-ok: caller-owned data, per the C API contract
|
||||
}
|
||||
ss_free(entry->key);
|
||||
ss_free(entry);
|
||||
break;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
std::uint32_t id;
|
||||
if (cache->free_head != kNone) {
|
||||
id = cache->free_head;
|
||||
cache->free_head = cache->node(id)->next;
|
||||
} else if (cache->fresh < cache->node_count) {
|
||||
id = cache->fresh++;
|
||||
new (cache->node(id)) Node();
|
||||
} else {
|
||||
std::abort(); // eviction below keeps at least one node free
|
||||
}
|
||||
Node *n = cache->node(id);
|
||||
n->used = true;
|
||||
n->data = data;
|
||||
n->ts = ss_monotonic_time();
|
||||
n->key_len = static_cast<std::uint16_t>(key_len);
|
||||
std::memcpy(cache->key(id), key, key_len);
|
||||
cache->key(id)[key_len] = '\0';
|
||||
cache->put(slot, id);
|
||||
cache->link_newest(id);
|
||||
++cache->size;
|
||||
|
||||
if (cache->size >= cache->capacity) {
|
||||
cache->drop_id(cache->oldest, true);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t
|
||||
cache_size(const struct cache *cache)
|
||||
{
|
||||
return cache == nullptr ? 0 : cache->size;
|
||||
}
|
||||
|
||||
} // extern "C"
|
||||
|
||||
Reference in New Issue
Block a user