Commit 59cbecfc authored by Marek Vavrusa's avatar Marek Vavrusa

Merge remote-tracking branch 'origin/mem-estim'

Conflicts:
	doc/running.texi
parents 3832fb96 59418443
......@@ -281,3 +281,5 @@ src/zscanner/test/processing.h
src/zscanner/test/tests.c
src/zscanner/test/tests.h
src/zscanner/test/zscanner-tool.c
src/knot/zone/estimator.c
src/knot/zone/estimator.h
......@@ -63,6 +63,7 @@ Actions:
zonestatus Show status of configured zones.
checkconf Check current server configuration.
checkzone [zone] Check zone (all if not specified).
memstats [zone] Estimate memory consumption for zone (all if not specified).
@end example
Also, the server needs to create several files in order to run properly.
......@@ -104,8 +105,9 @@ $ knotd -c slave.conf
To start it as a daemon, just add a @code{-d} parameter. Unlike the foreground mode,
PID file will be created in @code{rundir} directory.
@example
$ knotd -d -c slave.conf start # start the daemon
$ knotd -d -c slave.conf # start the daemon
$ knotc -c slave.conf stop # stop the daemon
@end example
......@@ -120,6 +122,16 @@ you can use @code{knotc checkzone} action.
$ knotc -c master.conf checkzone example.com
@end example
For an approximate estimate of server's memory consumption, you can use the @code{knotc memstats} action.
This action prints count of resource records, percentage of signed records and finally estimation
of memory consumption for each zone, unless specified otherwise. Please note that estimated values might
differ from the actual consumption. Also, for slave servers with incoming transfers enabled,
be aware that the actual memory consumption might be double or more during transfers.
@example
$ knotc -c master.conf memstats example.com
@end example
Starting and stopping the daemon is the same as with the slave server in the previous section.
@node Controlling running daemon
......@@ -143,7 +155,7 @@ If you encounter a bug using this feature, please send it to Knot developers (@p
You can also choose to tear-down the server fully and restart with the @code{knotc restart} action.
Note that some actions like start, stop and restart cannot be done remotely.
@example
$ knotc -c master.conf running # check if running
$ knotc -c master.conf status # check if running
$ knotc -c master.conf restart # fully restart
@end example
......
......@@ -68,6 +68,9 @@ Check server configuration.
.TP
checkzone
Check zones before compiling (accepts specific zones, f.e. 'knotc checkzone example1.com example2.com').
.TP
memstats
Estimate memory consumption for zone files. Useful mainly for big zones.
.SS "EXAMPLES"
.TP
.B Setup a keyfile for remote control
......
......@@ -285,6 +285,8 @@ libknotd_la_SOURCES = \
knot/zone/zone-dump.c \
knot/zone/zone-load.h \
knot/zone/zone-load.c \
knot/zone/estimator.h \
knot/zone/estimator.c \
knot/server/server.h
zscanner_tool_SOURCES = \
......
......@@ -57,7 +57,7 @@ size_t ahtable_size (const ahtable_t*); // Number of stored keys.
/** Find the given key in the table, inserting it if it does not exist, and
* returning a pointer to it's key.
* returning a pointer to its key.
*
* This pointer is not guaranteed to be valid after additional calls to
* ahtable_get, ahtable_del, ahtable_clear, or other functions that modifies the
......
......@@ -382,11 +382,30 @@ static void node_apply(node_ptr node, void (*f)(value_t*,void*), void* d)
}
}
static void node_apply_ahtable(node_ptr node, void (*f)(void*,void*), void* d)
{
if (*node.flag & NODE_TYPE_TRIE) {
size_t i;
for (i = 0; i < NODE_CHILDS; ++i) {
if (i > 0 && node.t->xs[i].t == node.t->xs[i - 1].t) continue;
if (node.t->xs[i].t) node_apply_ahtable(node.t->xs[i], f, d);
}
}
else {
f(node.b, d);
}
}
void hattrie_apply_rev(hattrie_t* T, void (*f)(value_t*,void*), void* d)
{
node_apply(T->root, f, d);
}
void hattrie_apply_rev_ahtable(hattrie_t* T, void (*f)(void*,void*), void* d)
{
node_apply_ahtable(T->root, f, d);
}
int hattrie_split_mid(node_ptr node, unsigned *left_m, unsigned *right_m)
{
/* count the number of occourances of every leading character */
......
......@@ -54,6 +54,7 @@ hattrie_t* hattrie_dup (const hattrie_t*, value_t (*nval)(value_t));
void hattrie_build_index (hattrie_t*);
void hattrie_apply_rev (hattrie_t*, void (*f)(value_t*,void*), void* d);
void hattrie_apply_rev_ahtable(hattrie_t* T, void (*f)(void*,void*), void* d);
/** Find the given key in the trie, inserting it if it does not exist, and
* returning a pointer to it's key.
......
......@@ -44,6 +44,7 @@
#include "libknot/packet/query.h"
#include "libknot/packet/response.h"
#include "knot/zone/zone-load.h"
#include "knot/zone/estimator.h"
/*! \brief Controller flags. */
enum knotc_flag_t {
......@@ -86,6 +87,7 @@ static int cmd_status(int argc, char *argv[], unsigned flags);
static int cmd_zonestatus(int argc, char *argv[], unsigned flags);
static int cmd_checkconf(int argc, char *argv[], unsigned flags);
static int cmd_checkzone(int argc, char *argv[], unsigned flags);
static int cmd_memstats(int argc, char *argv[], unsigned flags);
/*! \brief Table of remote commands. */
knot_cmd_t knot_cmd_tbl[] = {
......@@ -98,6 +100,7 @@ knot_cmd_t knot_cmd_tbl[] = {
{&cmd_zonestatus, 0, "zonestatus", "", "\tShow status of configured zones."},
{&cmd_checkconf, 1, "checkconf", "", "\tCheck current server configuration."},
{&cmd_checkzone, 1, "checkzone", "[zone]", "Check zone (all if not specified)."},
{&cmd_memstats, 1, "memstats", "[zone]", "Estimate memory use for zone (all if not specified)."},
{NULL, 0, NULL, NULL, NULL}
};
......@@ -712,9 +715,123 @@ static int cmd_checkzone(int argc, char *argv[], unsigned flags)
knot_zone_deep_free(&z);
knot_zload_close(l);
log_zone_info("Zone %s OK.\n", zone->name);
}
return rc;
}
static int cmd_memstats(int argc, char *argv[], unsigned flags)
{
UNUSED(flags);
/* Zone checking */
int rc = 0;
node *n = 0;
size_t total_size = 0;
/* Generate databases for all zones */
WALK_LIST(n, conf()->zones) {
/* Fetch zone */
conf_zone_t *zone = (conf_zone_t *) n;
int zone_match = 0;
for (unsigned i = 0; i < (unsigned)argc; ++i) {
size_t len = strlen(zone->name);
/* All (except root) without final dot */
if (len > 1) {
len -= 1;
}
if (strncmp(zone->name, argv[i], len) == 0) {
zone_match = 1;
break;
}
}
if (!zone_match && argc > 0) {
/* WALK_LIST is a for-cycle. */
continue;
}
/* Init malloc wrapper for trie size estimation. */
size_t malloc_size = 0;
mm_ctx_t mem_ctx = { .ctx = &malloc_size,
.alloc = estimator_malloc,
.free = estimator_free };
/* Init memory estimation context. */
zone_estim_t est = {.node_table = hattrie_create_n(TRIE_BUCKET_SIZE, &mem_ctx),
.dname_table = hattrie_create_n(TRIE_BUCKET_SIZE, &mem_ctx),
.dname_size = 0, .rrset_size = 0,
.node_size = 0, .ahtable_size = 0,
.rdata_size = 0, .record_count = 0 };
if (est.node_table == NULL) {
if (est.dname_table) {
hattrie_free(est.dname_table);
}
log_server_error("Not enough memory.\n");
continue;
}
if (est.dname_table == NULL) {
if (est.node_table) {
hattrie_free(est.node_table);
}
log_server_error("Not enough memory.\n");
continue;
}
/* Create file loader. */
file_loader_t *loader = file_loader_create(zone->file, zone->name,
KNOT_CLASS_IN, 3600,
estimator_rrset_memsize_wrap,
process_error,
&est);
if (loader == NULL) {
log_zone_error("Could not load zone.\n");
hattrie_apply_rev(est.node_table, estimator_free_trie_node, NULL);
hattrie_apply_rev(est.dname_table, estimator_free_trie_node, NULL);
hattrie_free(est.node_table);
hattrie_free(est.dname_table);
return KNOT_ERROR;
}
/* Do a parser run, but do not actually create the zone. */
int ret = file_loader_process(loader);
if (ret != KNOT_EOK) {
log_zone_error("Failed to parse zone.\n");
hattrie_apply_rev(est.node_table, estimator_free_trie_node, NULL);
hattrie_apply_rev(est.dname_table, estimator_free_trie_node, NULL);
hattrie_free(est.node_table);
hattrie_free(est.dname_table);
return KNOT_ERROR;
}
/* Only size of ahtables inside trie's nodes is missing. */
assert(est.ahtable_size == 0);
est.ahtable_size = estimator_trie_ahtable_memsize(est.node_table);
/* Cleanup */
hattrie_apply_rev(est.node_table, estimator_free_trie_node, NULL);
hattrie_apply_rev(est.dname_table, estimator_free_trie_node, NULL);
hattrie_free(est.node_table);
hattrie_free(est.dname_table);
size_t zone_size = (size_t)(((double)(est.rdata_size +
est.node_size +
est.rrset_size +
est.dname_size +
est.ahtable_size +
malloc_size) * ESTIMATE_MAGIC) / (1024.0 * 1024.0));
log_zone_info("Zone %s: %zu RRs, used memory estimation is %zuMB.\n",
zone->name, est.record_count, zone_size);
file_loader_free(loader);
total_size += zone_size;
}
if (argc == 0) { // for all zones
log_zone_info("Estimated memory consumption for all zones is %zuMB.\n", total_size);
}
return rc;
}
/* Copyright (C) 2011 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <config.h>
#include <assert.h>
#include "estimator.h"
#include "dname.h"
#include "common/lists.h"
#include "libknot/zone/node.h"
#include "common/hattrie/ahtable.h"
#include "zscanner/scanner.h"
#include "common/descriptor.h"
// Constants used for tweaking, mostly malloc overhead
enum estim_consts {
MALLOC_OVER = sizeof(size_t), // according to malloc.c, this is minimum
DNAME_MULT = 1,
DNAME_ADD = MALLOC_OVER * 3, // dname itself, labels, name
RDATA_MULT = 1,
RDATA_ADD = MALLOC_OVER, // just raw rdata, but allocation is there
RRSET_MULT = 1,
RRSET_ADD = MALLOC_OVER * 3, // rrset itself, rdata array, indices
NODE_MULT = 1,
NODE_ADD = MALLOC_OVER * 2, // node itself, rrset array
AHTABLE_ADD = MALLOC_OVER * 3, // table, slots, slot sizes
MALLOC_MIN = 3 * MALLOC_OVER // minimun size of malloc'd chunk, -overhead
};
typedef struct type_list_item {
node n;
uint16_t type;
} type_list_item_t;
typedef struct dummy_node {
// For now only contains list of RR types
list node_list;
} dummy_node_t;
// return: 0 not present, 1 - present
static int find_in_list(list *node_list, uint16_t type)
{
node *n = NULL;
WALK_LIST(n, *node_list) {
type_list_item_t *l_entr = (type_list_item_t *)n;
assert(l_entr);
if (l_entr->type == type) {
return 1;
}
}
type_list_item_t *new_entry = xmalloc(sizeof(type_list_item_t));
new_entry->type = type;
add_head(node_list, (node *)new_entry);
return 0;
}
// return: 0 not present (added), 1 - present
static int dummy_node_add_type(dummy_node_t *n, uint16_t t)
{
return find_in_list(&n->node_list, t);
}
static size_t dname_memsize(const knot_dname_t *d)
{
size_t d_size = d->size;
size_t l_size = d->label_count;
if (d->size < MALLOC_MIN) {
d_size = MALLOC_MIN;
}
if (d->label_count < MALLOC_MIN) {
l_size = MALLOC_MIN;
}
return (sizeof(knot_dname_t) + d_size + l_size)
* DNAME_MULT + DNAME_ADD;
}
// return: 0 - unique, 1 - duplicate
static int insert_dname_into_table(hattrie_t *table, knot_dname_t *d,
dummy_node_t **n)
{
value_t *val = hattrie_tryget(table, (char *)d->name, d->size);
if (val == NULL) {
// Create new dummy node to use for this dname
*n = xmalloc(sizeof(dummy_node_t));
init_list(&(*n)->node_list);
*hattrie_get(table, (char *)d->name, d->size) = *n;
return 0;
} else {
// Return previously found dummy node
*n = (dummy_node_t *)(*val);
return 1;
}
}
// return: RDATA memsize, minus size of dnames inside
static size_t rdata_memsize(zone_estim_t *est, const scanner_t *scanner)
{
const rdata_descriptor_t *desc = get_rdata_descriptor(scanner->r_type);
size_t size = 0;
for (int i = 0; desc->block_types[i] != KNOT_RDATA_WF_END; ++i) {
// DNAME - pointer in memory
int item = desc->block_types[i];
if (descriptor_item_is_dname(item)) {
size += sizeof(knot_dname_t *);
knot_dname_t *dname =
knot_dname_new_from_wire(scanner->r_data +
scanner->r_data_blocks[i],
scanner->r_data_blocks[i + 1] -
scanner->r_data_blocks[i],
NULL);
if (dname == NULL) {
return KNOT_ERROR;
}
knot_dname_to_lower(dname);
dummy_node_t *n = NULL;
if (insert_dname_into_table(est->dname_table,
dname, &n) == 0) {
// First time we see this dname, add size
est->dname_size += dname_memsize(dname);
}
knot_dname_free(&dname);
} else if (descriptor_item_is_fixed(item)) {
// Fixed length
size += item;
} else {
// Variable length
size += scanner->r_data_blocks[i + 1] -
scanner->r_data_blocks[i];
}
}
return size * RDATA_MULT + RDATA_ADD;
}
static void rrset_memsize(zone_estim_t *est, const scanner_t *scanner)
{
const rdata_descriptor_t *desc = get_rdata_descriptor(scanner->r_type);
// Handle RRSet's owner
knot_dname_t *owner = knot_dname_new_from_wire(scanner->r_owner,
scanner->r_owner_length,
NULL);
dummy_node_t *n;
if (insert_dname_into_table(est->node_table, owner, &n) == 0) {
// First time we see this name == new node
est->node_size += sizeof(knot_node_t) * NODE_MULT + NODE_ADD;
// Also, RRSet's owner will now contain full dname
est->dname_size += dname_memsize(owner);
// Trie's nodes handled at the end of computation
}
knot_dname_free(&owner);
assert(n);
// We will always add RDATA
size_t rdlen = rdata_memsize(est, scanner);
if (rdlen < MALLOC_MIN) {
rdlen = MALLOC_MIN;
}
// DNAME's size not included (handled inside rdata_memsize())
est->rdata_size += rdlen;
est->record_count++;
/*
* RDATA size done, now add static part of RRSet to size.
* Do not add for RRs that would be merged.
* All possible duplicates will be added to total size.
*/
if (dummy_node_add_type(n, scanner->r_type) == 0) {
/*
* New RR type, add actual RRSet struct's size:
* MALLOC_MIN is added because of index array - usually not many RRs
* are in the RRSet and values in the array are type uint32, so
* 3 would be needed on 32bit system and 6 on 32bit system in order to
* be larger than MALLOC_MIN, so we use it instead. Of course, if there
* are more than 3/6 records in RRSet, measurement will not be precise.
*/
est->rrset_size += (sizeof(knot_rrset_t) + MALLOC_MIN)
* RRSET_MULT + RRSET_ADD;
// Add pointer in node's array
est->node_size += sizeof(knot_rrset_t *);
} else {
// Merge would happen, so just RDATA index is added
//est->rrset_size += sizeof(uint32_t);
}
}
void *estimator_malloc(void *ctx, size_t len)
{
size_t *count = (size_t *)ctx;
*count += len + MALLOC_OVER;
return xmalloc(len);
}
void estimator_free(void *p)
{
free(p);
}
static void get_ahtable_size(void *t, void *d)
{
ahtable_t *table = (ahtable_t *)t;
size_t *size = (size_t *)d;
// info about allocated chunks starts at table->n-th index
for (size_t i = table->n; i < table->n * 2; ++i) {
// add actual slot size (= allocated for slot)
*size += table->slot_sizes[i];
// each non-empty slot means allocation overhead
*size += table->slot_sizes[i] ? MALLOC_OVER : 0;
}
*size += sizeof(ahtable_t);
// slot sizes + allocated sizes
*size += (table->n * 2) * sizeof(uint32_t);
// slots
*size += table->n * sizeof(void *);
*size += AHTABLE_ADD;
}
size_t estimator_trie_ahtable_memsize(hattrie_t *table)
{
/*
* Iterate through trie's node, and get stats from each ahtable.
* Space taken up by the trie itself is measured using malloc wrapper.
* (Even for large zones, space taken by trie itself is very small)
*/
size_t size = 0;
hattrie_apply_rev_ahtable(table, get_ahtable_size, &size);
return size;
}
void estimator_rrset_memsize_wrap(const scanner_t *scanner)
{
rrset_memsize(scanner->data, scanner);
}
void estimator_free_trie_node(value_t *val, void *data)
{
UNUSED(data);
dummy_node_t *trie_n = (dummy_node_t *)(*val);
WALK_LIST_FREE(trie_n->node_list);
free(trie_n);
}
/* Copyright (C) 2013 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/*!
* \file estimator.h
*
* \author Jan Kadlec <jan.kadlec@nic.cz>
*
* \brief Memory estimation for zone files.
*
* \addtogroup config
* @{
*/
#ifndef _KNOT_ESTIMATOR_H_
#define _KNOT_ESTIMATOR_H_
#include "common/hattrie/hat-trie.h"
#include "zscanner/scanner.h"
// Mutiplicative constant, needed because of malloc's fragmentation
static const double ESTIMATE_MAGIC = 1.2;
/*!
* \brief Memory estimation context.
*/
typedef struct zone_estim {
hattrie_t *node_table; /*!< Same trie is in actual zone. */
hattrie_t *dname_table; /*!< RDATA section DNAMEs. */
size_t rdata_size; /*!< Estimated RDATA size. */
size_t dname_size; /*!< Estimated DNAME size. */
size_t node_size; /*!< Estimated node size. */
size_t ahtable_size; /*!< Estimated ahtable size. */
size_t rrset_size; /*!< Estimated RRSet size. */
size_t record_count; /*!< Total record count for zone. */
} zone_estim_t;
/*!
* \brief Size counting malloc wrapper.
*
* \param ctx Data for malloc wrapper.
* \param len Size to allocate.
*
* \retval Alloc'd data on succes.
* \retval NULL on error.
*/
void *estimator_malloc(void* ctx, size_t len);
/*!
* \brief Size counting free wrapper.
*
* \param p Data to free.
*/
void estimator_free(void *p);
/*!
* \brief Goes through trie's ahtables and estimates their memory requirements.
*
* \param table Trie to traverse.
*/
size_t estimator_trie_ahtable_memsize(hattrie_t *table);
/*!
* \brief For use with scanner - counts memsize of RRSets.
*
* \param scanner Scanner context.
*/
void estimator_rrset_memsize_wrap(const scanner_t *scanner);
/*!
* \brief Cleanup function for use with hattrie.
*
* \param p Data to free.
*/
void estimator_free_trie_node(value_t *val, void *data);
#endif /* _KNOT_ESTIMATOR_H_ */
......@@ -189,7 +189,7 @@ static void process_rrsigs_in_node(parser_context_t *parser,
}
}
static void process_error(const scanner_t *s)
void process_error(const scanner_t *s)
{
if (s->stop == true) {
log_zone_error("Fatal error in zone file %s:%"PRIu64": %s "
......
......@@ -101,6 +101,8 @@ knot_zone_t *knot_zload_load(zloader_t *loader);
*/
void knot_zload_close(zloader_t *loader);
void process_error(const scanner_t *scanner);
#endif /* _KNOTD_ZONELOAD_H_ */
/*! @} */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment