Commit 8e5493eb authored by Libor Peltan's avatar Libor Peltan

journal: added obsolete journal import; definitive configuration for journal

parent a62a293e
......@@ -258,10 +258,12 @@ src/knot/events/handlers/refresh.c
src/knot/events/handlers/update.c
src/knot/events/replan.c
src/knot/events/replan.h
src/knot/journal/serialization.c
src/knot/journal/serialization.h
src/knot/journal/journal.c
src/knot/journal/journal.h
src/knot/journal/old_journal.c
src/knot/journal/old_journal.h
src/knot/journal/serialization.c
src/knot/journal/serialization.h
src/knot/modules/dnsproxy/dnsproxy.c
src/knot/modules/dnsproxy/dnsproxy.h
src/knot/modules/dnstap/dnstap.c
......
......@@ -337,6 +337,8 @@ libknotd_la_SOURCES = \
knot/server/dthreads.h \
knot/journal/journal.c \
knot/journal/journal.h \
knot/journal/old_journal.c \
knot/journal/old_journal.h \
knot/journal/serialization.c \
knot/journal/serialization.h \
knot/server/server.c \
......
......@@ -1046,17 +1046,41 @@ char* conf_zonefile_txn(
return get_filename(conf, txn, zone, file);
}
char* conf_journalfile_txn(
char* conf_old_journalfile(
conf_t *conf,
knot_db_txn_t *txn)
const knot_dname_t *zone)
{
conf_val_t val;
if (zone == NULL) {
return NULL;
}
val = conf_default_get_txn(conf, txn, C_STORAGE);
char *storage = conf_abs_path(&val, NULL);
conf_val_t val = conf_zone_get(conf, C_JOURNAL, zone);
const char *journal = conf_str(&val);
val = conf_default_get_txn(conf, txn, C_JOURNAL);
// Use default journalfile name pattern if not specified.
if (journal == NULL) {
journal = "%s.db";
} else {
CONF_LOG_ZONE(LOG_NOTICE, zone, "obsolete configuration 'journal', "
"use 'template.journal-db'' instead");
}
val = conf_zone_get(conf, C_MAX_JOURNAL_SIZE, zone);
if (val.code == KNOT_EOK) {
CONF_LOG_ZONE(LOG_NOTICE, zone, "obsolete configuration 'max-journal-size', "
"use 'max-journal-usage' and 'template.journal-db' instead");
}
return get_filename(conf, &conf->read_txn, zone, journal);
}
char* conf_journalfile_txn(
conf_t *conf,
knot_db_txn_t *txn)
{
conf_val_t val = conf_default_get_txn(conf, txn, C_STORAGE);
char *storage = conf_abs_path(&val, NULL);
val = conf_default_get_txn(conf, txn, C_JOURNAL_DB);
char *journaldir = conf_abs_path(&val, storage);
free(storage);
......
......@@ -580,11 +580,11 @@ static inline char* conf_zonefile(
*
* \param[in] conf Configuration.
* \param[in] txn Configuration DB transaction.
* \param[in] zone Zone name.
*
* \return Absolute journal file path string pointer.
*/
char* conf_journalfile_txn(conf_t *conf,
char* conf_journalfile_txn(
conf_t *conf,
knot_db_txn_t *txn);
static inline char* conf_journalfile(
conf_t *conf)
......@@ -592,6 +592,11 @@ static inline char* conf_journalfile(
return conf_journalfile_txn(conf, &conf->read_txn);
}
char* conf_old_journalfile(
conf_t *conf,
const knot_dname_t *zone
);
/*!
* Gets the configured number of UDP threads.
*
......
......@@ -45,11 +45,14 @@
#define HOURS(x) ((x) * 3600)
#define DAYS(x) ((x) * HOURS(24))
#define GIGA (1024LLU * 1024 * 1024)
#define TERA (1024 * GIGA)
#define KILO(x) (1024LLU * (x))
#define MEGA(x) (KILO(1024) * (x))
#define GIGA(x) (MEGA(1024) * (x))
#define TERA(x) (GIGA(1024) * (x))
#define VIRT_MEM_TOP_32BIT (2 * GIGA)
#define VIRT_MEM_LIMIT(x) (((sizeof(void *) < 8) && ((x) > VIRT_MEM_TOP_32BIT)) ? VIRT_MEM_TOP_32BIT : (x))
#define VIRT_MEM_TOP_32BIT GIGA(2)
#define VIRT_MEM_LIMIT(x) (((sizeof(void *) < 8) && ((x) > VIRT_MEM_TOP_32BIT)) \
? VIRT_MEM_TOP_32BIT : (x))
#define FMOD (YP_FMULTI | CONF_IO_FRLD_MOD | CONF_IO_FRLD_ZONES)
......@@ -242,11 +245,9 @@ static const yp_item_t desc_remote[] = {
{ C_DISABLE_ANY, YP_TBOOL, YP_VNONE }, \
{ C_ZONEFILE_SYNC, YP_TINT, YP_VINT = { -1, INT32_MAX, 0, YP_STIME } }, \
{ C_IXFR_DIFF, YP_TBOOL, YP_VNONE }, \
{ C_MAX_ZONE_SIZE, YP_TINT, YP_VINT = { 0, INT64_MAX, INT64_MAX, YP_SSIZE }, \
FLAGS }, \
{ C_MAX_JOURNAL_USAGE, YP_TINT, YP_VINT = { 40 * 1024, INT64_MAX, 100 * 1024 * 1024, \
YP_SSIZE } }, \
{ C_MAX_JOURNAL_DEPTH, YP_TINT, YP_VINT = { 2, INT64_MAX, INT64_MAX, YP_SSIZE } }, \
{ C_MAX_ZONE_SIZE, YP_TINT, YP_VINT = { 0, INT64_MAX, INT64_MAX, YP_SSIZE }, FLAGS }, \
{ C_MAX_JOURNAL_USAGE, YP_TINT, YP_VINT = { KILO(40), INT64_MAX, MEGA(100), YP_SSIZE } }, \
{ C_MAX_JOURNAL_DEPTH, YP_TINT, YP_VINT = { 2, INT64_MAX, INT64_MAX } }, \
{ C_KASP_DB, YP_TSTR, YP_VSTR = { "keys" }, FLAGS }, \
{ C_DNSSEC_SIGNING, YP_TBOOL, YP_VNONE, FLAGS }, \
{ C_DNSSEC_POLICY, YP_TREF, YP_VREF = { C_POLICY }, FLAGS, { check_ref_dflt } }, \
......@@ -254,18 +255,21 @@ static const yp_item_t desc_remote[] = {
{ C_REQUEST_EDNS_OPTION, YP_TDATA, YP_VDATA = { 0, NULL, edns_opt_to_bin, edns_opt_to_txt } }, \
{ C_MODULE, YP_TDATA, YP_VDATA = { 0, NULL, mod_id_to_bin, mod_id_to_txt }, \
YP_FMULTI | FLAGS, { check_modref } }, \
{ C_COMMENT, YP_TSTR, YP_VNONE },
{ C_COMMENT, YP_TSTR, YP_VNONE }, \
/* Obsolete, old journal items. */ \
{ C_JOURNAL, YP_TSTR, YP_VNONE, FLAGS }, \
{ C_MAX_JOURNAL_SIZE, YP_TINT, YP_VINT = { 0, INT64_MAX, INT64_MAX, YP_SSIZE }, FLAGS }, \
static const yp_item_t desc_template[] = {
{ C_ID, YP_TSTR, YP_VNONE, CONF_IO_FREF },
ZONE_ITEMS(CONF_IO_FRLD_ZONES)
{ C_TIMER_DB, YP_TSTR, YP_VSTR = { "timers" }, CONF_IO_FRLD_ZONES }, \
{ C_GLOBAL_MODULE, YP_TDATA, YP_VDATA = { 0, NULL, mod_id_to_bin, mod_id_to_txt }, \
YP_FMULTI | CONF_IO_FRLD_MOD, { check_modref } }, \
{ C_JOURNAL, YP_TSTR, YP_VSTR = { "journal.db" }, CONF_IO_FRLD_SRV }, \
{ C_MAX_JOURNAL_SIZE, YP_TINT, YP_VINT = { 1024 * 1024, VIRT_MEM_LIMIT(100 * TERA), \
VIRT_MEM_LIMIT(20 * GIGA), YP_SSIZE }, \
CONF_IO_FRLD_SRV }, \
{ C_TIMER_DB, YP_TSTR, YP_VSTR = { "timers" }, CONF_IO_FRLD_ZONES },
{ C_GLOBAL_MODULE, YP_TDATA, YP_VDATA = { 0, NULL, mod_id_to_bin, mod_id_to_txt },
YP_FMULTI | CONF_IO_FRLD_MOD, { check_modref } },
{ C_JOURNAL_DB, YP_TSTR, YP_VSTR = { "journal" }, CONF_IO_FRLD_SRV },
{ C_MAX_JOURNAL_DB_SIZE, YP_TINT, YP_VINT = { JOURNAL_MIN_FSLIMIT, VIRT_MEM_LIMIT(TERA(100)),
VIRT_MEM_LIMIT(GIGA(20)), YP_SSIZE },
CONF_IO_FRLD_SRV },
{ NULL }
};
......
......@@ -53,7 +53,8 @@
#define C_IDENT "\x08""identity"
#define C_INCL "\x07""include"
#define C_IXFR_DIFF "\x15""ixfr-from-differences"
#define C_JOURNAL "\x07""journal"
#define C_JOURNAL "\x07""journal" /* obsolete, old journal compat */
#define C_JOURNAL_DB "\x0A""journal-db"
#define C_KASP_DB "\x07""kasp-db"
#define C_KEY "\x03""key"
#define C_KEYSTORE "\x08""keystore"
......@@ -62,7 +63,8 @@
#define C_LOG "\x03""log"
#define C_MANUAL "\x06""manual"
#define C_MASTER "\x06""master"
#define C_MAX_JOURNAL_SIZE "\x10""max-journal-size"
#define C_MAX_JOURNAL_SIZE "\x10""max-journal-size" /* obsolete, old journal compat */
#define C_MAX_JOURNAL_DB_SIZE "\x13""max-journal-db-size"
#define C_MAX_JOURNAL_USAGE "\x11""max-journal-usage"
#define C_MAX_JOURNAL_DEPTH "\x11""max-journal-depth"
#define C_MAX_TCP_CLIENTS "\x0F""max-tcp-clients"
......
......@@ -495,42 +495,19 @@ int check_template(
return KNOT_EOK;
}
// Check global-module.
conf_val_t g_module = conf_rawid_get_txn(args->conf, args->txn, C_TPL,
C_GLOBAL_MODULE, args->id,
args->id_len);
if (g_module.code == KNOT_EOK) {
args->err_str = "global module in non-default template";
return KNOT_EINVAL;
}
// Check timer-db.
conf_val_t timer_db = conf_rawid_get_txn(args->conf, args->txn, C_TPL,
C_TIMER_DB, args->id, args->id_len);
if (timer_db.code == KNOT_EOK) {
args->err_str = "timer database location in non-default template";
return KNOT_EINVAL;
}
// Check journal.
conf_val_t journal = conf_rawid_get_txn(args->conf, args->txn, C_TPL,
C_JOURNAL, args->id, args->id_len);
if (journal.code == KNOT_EOK) {
args->err_str = "journal location in non-default template";
return KNOT_EINVAL;
}
// Check max-journal-size.
conf_val_t max_journal_size = conf_rawid_get_txn(args->conf, args->txn, C_TPL,
C_MAX_JOURNAL_SIZE, args->id, args->id_len);
if (max_journal_size.code == KNOT_EOK) {
args->err_str = "journal size in non-default template";
return KNOT_EINVAL;
}
conf_val_t val;
#define CHECK_DFLT(item, name) \
val = conf_rawid_get_txn(args->conf, args->txn, C_TPL, item, \
args->id, args->id_len); \
if (val.code == KNOT_EOK) { \
args->err_str = name " in non-default template"; \
return KNOT_EINVAL; \
}
CHECK_DFLT(C_TIMER_DB, "timer database");
CHECK_DFLT(C_GLOBAL_MODULE, "global module");
CHECK_DFLT(C_JOURNAL_DB, "journal database path");
CHECK_DFLT(C_MAX_JOURNAL_DB_SIZE, "journal database maximum size");
return KNOT_EOK;
}
......
This diff is collapsed.
/* Copyright (C) 2016 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <stdint.h>
#include "contrib/ucw/lists.h"
/*!
* \brief Check if the journal file is used or not.
*
* \param path Journal file.
*
* \return true or false
*/
bool old_journal_exists(const char *path);
/*!
* \brief Load changesets from journal.
*
* \param path Path to journal file.
* \param zone Corresponding zone.
* \param dst Store changesets here.
* \param from Start serial.
* \param to End serial.
*
* \retval KNOT_EOK on success.
* \retval KNOT_ERANGE if given entry was not found.
* \return < KNOT_EOK on error.
*/
int old_journal_load_changesets(const char *path, const knot_dname_t *zone,
list_t *dst, uint32_t from, uint32_t to);
......@@ -380,7 +380,7 @@ int server_init(server_t *server, int bg_workers)
}
char * journal_dir = conf_journalfile(conf());
conf_val_t journal_size = conf_default_get(conf(), C_MAX_JOURNAL_SIZE);
conf_val_t journal_size = conf_default_get(conf(), C_MAX_JOURNAL_DB_SIZE);
int ret = init_journal_db(&server->journal_db, journal_dir, conf_int(&journal_size));
free(journal_dir);
if (ret != KNOT_EOK) {
......
......@@ -16,6 +16,7 @@
#include "knot/common/log.h"
#include "knot/journal/journal.h"
#include "knot/journal/old_journal.h"
#include "knot/zone/zone-diff.h"
#include "knot/zone/zone-load.h"
#include "knot/zone/zonefile.h"
......@@ -90,6 +91,68 @@ int zone_load_check(conf_t *conf, zone_contents_t *contents)
return KNOT_EOK;
}
/*!
* \brief If old journal exists, warn the user and append the changes to chgs
*
* \todo Remove in the future together with journal/old_journal.[ch] and conf_old_journalfile()
*/
static void try_old_journal(conf_t *conf, zone_t *zone, uint32_t zone_c_serial, list_t *chgs)
{
list_t old_chgs;
init_list(&old_chgs);
// fetch old journal name
char *jfile = conf_old_journalfile(conf, zone->name);
if (jfile == NULL) {
return;
}
if (!old_journal_exists(jfile)) {
goto toj_end;
}
log_zone_notice(zone->name, "journal, obsolete exists, file '%s'", jfile);
// determine serial to load from
if (!EMPTY_LIST(*chgs)) {
changeset_t * lastch = TAIL(*chgs);
zone_c_serial = knot_soa_serial(&lastch->soa_to->rrs);
}
// load changesets from old journal
int ret = old_journal_load_changesets(jfile, zone->name, &old_chgs,
zone_c_serial, zone_c_serial - 1);
if (ret != KNOT_ERANGE && ret != KNOT_ENOENT && ret != KNOT_EOK) {
log_zone_warning(zone->name, "journal, failed to load obsolete history (%s)",
knot_strerror(ret));
goto toj_end;
}
if (EMPTY_LIST(old_chgs)) {
goto toj_end;
}
log_zone_notice(zone->name, "journal, loaded obsolete history since serial '%u'",
zone_c_serial);
// store them to new journal
ret = zone_changes_store(conf, zone, &old_chgs);
if (ret != KNOT_EOK) {
log_zone_warning(zone->name, "journal, failed to store obsolete history (%s)",
knot_strerror(ret));
goto toj_end;
}
// append them to chgs
changeset_t *ch, *nxt;
WALK_LIST_DELSAFE(ch, nxt, old_chgs) {
rem_node(&ch->n);
add_tail(chgs, &ch->n);
}
toj_end:
changesets_free(&old_chgs);
free(jfile);
}
int zone_load_journal(conf_t *conf, zone_t *zone, zone_contents_t *contents)
{
if (conf == NULL || zone == NULL || contents == NULL) {
......@@ -104,19 +167,20 @@ int zone_load_journal(conf_t *conf, zone_t *zone, zone_contents_t *contents)
/* Fetch SOA serial. */
uint32_t serial = zone_contents_serial(contents);
/*! \todo Check what should be the upper bound. */
/* Load journal */
list_t chgs;
init_list(&chgs);
int ret = zone_changes_load(conf, zone, &chgs, serial);
if (ret != KNOT_EOK) {
if (ret != KNOT_EOK && ret != KNOT_ENOENT) {
changesets_free(&chgs);
/* Absence of records is not an error. */
if (ret == KNOT_ENOENT) {
return KNOT_EOK;
} else {
return ret;
}
/* Load old journal (to be obsoleted) */
try_old_journal(conf, zone, serial, &chgs);
if (EMPTY_LIST(chgs)) {
return KNOT_EOK;
}
/* Apply changesets. */
......
......@@ -118,8 +118,8 @@ class Server(object):
self.max_udp6_payload = None
self.disable_any = None
self.disable_notify = None
self.zonefile_sync = None
self.journal_size = 20 * 1024 * 1024
self.zonefile_sync = "1d"
self.journal_db_size = 20 * 1024 * 1024
self.zone_size_limit = None
self.inquirer = None
......@@ -1044,10 +1044,8 @@ class Knot(Server):
s.id_item("id", "default")
s.item_str("storage", self.dir)
s.item_str("kasp-db", self.keydir)
if self.zonefile_sync:
s.item_str("zonefile-sync", self.zonefile_sync)
else:
s.item_str("zonefile-sync", "1d")
s.item_str("max-journal-db-size", self.journal_db_size)
s.item_str("semantic-checks", "on")
if self.disable_any:
s.item_str("disable-any", "on")
......@@ -1060,7 +1058,6 @@ class Knot(Server):
s.item("global-module", "[%s]" % modules)
if self.zone_size_limit:
s.item("max-zone-size", self.zone_size_limit)
s.item_str("max-journal-usage", self.journal_size)
s.end()
s.begin("zone")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment