Commit 5508a3c6 authored by Daniel Salzman's avatar Daniel Salzman

Merge branch 'journal_bootstrap_evo' into 'master'

Journal bootstrap evolution

See merge request !750
parents ffd9084d dc9209d0
......@@ -924,6 +924,7 @@ zone:
semantic\-checks: BOOL
disable\-any: BOOL
zonefile\-sync: TIME
zone\-in\-journal: BOOL
ixfr\-from\-differences: BOOL
max\-journal\-usage: SIZE
max\-journal\-depth: INT
......@@ -1090,6 +1091,28 @@ Otherwise the journal can\(aqt be applied.
.UNINDENT
.sp
\fIDefault:\fP 0 (immediate)
.SS zone\-in\-journal
.sp
If enabled, the server stores the zone file contents in journal. When starting server,
the zone is loaded preferably from journal, the changes from zone file are applied
afterwards.
.sp
\fBNOTE:\fP
.INDENT 0.0
.INDENT 3.5
Implies \fI\%ixfr\-from\-differences\fP\&.
.UNINDENT
.UNINDENT
.sp
\fBWARNING:\fP
.INDENT 0.0
.INDENT 3.5
If you modify the zone file, expecting the changes to be applied to the zone,
you shall set the SOA serial significatnly higher than current zone serial.
.UNINDENT
.UNINDENT
.sp
\fIDefault:\fP off
.SS ixfr\-from\-differences
.sp
If enabled, the server creates zone differences from changes you made to the
......
......@@ -1059,6 +1059,7 @@ Definition of zones served by the server.
semantic-checks: BOOL
disable-any: BOOL
zonefile-sync: TIME
zone-in-journal: BOOL
ixfr-from-differences: BOOL
max-journal-usage: SIZE
max-journal-depth: INT
......@@ -1232,6 +1233,24 @@ using the ``-f`` option.
*Default:* 0 (immediate)
.. _zone_zone-in-journal:
zone-in-journal
---------------
If enabled, the server stores the zone file contents in journal. When starting server,
the zone is loaded preferably from journal, the changes from zone file are applied
afterwards.
.. NOTE::
Implies :ref:`ixfr-from-differences<zone_ixfr-from-differences>`.
.. WARNING::
If you modify the zone file, expecting the changes to be applied to the zone,
you shall set the SOA serial significatnly higher than current zone serial.
*Default:* off
.. _zone_ixfr-from-differences:
ixfr-from-differences
......
......@@ -260,6 +260,7 @@ static const yp_item_t desc_policy[] = {
{ C_SEM_CHECKS, YP_TBOOL, YP_VNONE, FLAGS }, \
{ C_DISABLE_ANY, YP_TBOOL, YP_VNONE }, \
{ C_ZONEFILE_SYNC, YP_TINT, YP_VINT = { -1, INT32_MAX, 0, YP_STIME } }, \
{ C_ZONE_IN_JOURNAL, YP_TBOOL, YP_VNONE }, \
{ C_IXFR_DIFF, YP_TBOOL, YP_VNONE }, \
{ C_MAX_ZONE_SIZE, YP_TINT, YP_VINT = { 0, INT64_MAX, INT64_MAX, YP_SSIZE }, FLAGS }, \
{ C_MAX_JOURNAL_USAGE, YP_TINT, YP_VINT = { KILO(40), INT64_MAX, MEGA(100), YP_SSIZE } }, \
......
......@@ -124,6 +124,7 @@
#define C_VERSION "\x07""version"
#define C_VIA "\x03""via"
#define C_ZONE "\x04""zone"
#define C_ZONE_IN_JOURNAL "\x0F""zone-in-journal"
#define C_ZONEFILE_SYNC "\x0D""zonefile-sync"
#define C_ZSK_LIFETIME "\x0C""zsk-lifetime"
#define C_ZSK_SIZE "\x08""zsk-size"
......
......@@ -23,6 +23,7 @@
#include "knot/events/handlers.h"
#include "knot/events/log.h"
#include "knot/events/replan.h"
#include "knot/zone/zone-diff.h"
#include "knot/zone/zone-load.h"
#include "knot/zone/zone.h"
#include "knot/zone/zonefile.h"
......@@ -56,18 +57,25 @@ int event_load(conf_t *conf, zone_t *zone)
conf_val_t val;
zone_contents_t *contents = NULL;
bool load_from_journal = false;
bool contents_in_update = true;
zone_sign_reschedule_t dnssec_refresh = { 0 };
dnssec_refresh.allow_rollover = true;
/* Take zone file mtime and load it. */
time_t mtime;
char *filename = conf_zonefile(conf, zone->name);
int ret = zonefile_exists(filename, &mtime);
bool load_from_journal = (ret != KNOT_EOK);
free(filename);
if (ret != KNOT_EOK) {
load_from_journal = true;
bool load_journal_first = false;
bool loaded_from_journal = false;
if (zone->contents == NULL) {
conf_val_t val = conf_zone_get(conf, C_ZONE_IN_JOURNAL, zone->name);
load_journal_first = conf_bool(&val);
}
if (load_from_journal) {
ret = zone_load_from_journal(conf, zone, &contents);
if (ret != KNOT_EOK) {
if (zone_load_can_bootstrap(conf, zone->name)) {
......@@ -78,6 +86,15 @@ int event_load(conf_t *conf, zone_t *zone)
goto fail;
}
goto load_post;
} else if (load_journal_first) {
ret = zone_load_from_journal(conf, zone, &contents);
if (ret != KNOT_EOK && ret != KNOT_ENOENT) {
goto fail;
}
loaded_from_journal = (ret == KNOT_EOK);
if (loaded_from_journal) {
goto load_post;
}
}
ret = zone_load_contents(conf, zone->name, &contents);
......@@ -85,6 +102,14 @@ int event_load(conf_t *conf, zone_t *zone)
goto fail;
}
if (load_journal_first && !loaded_from_journal) {
ret = zone_in_journal_store(conf, zone, contents);
if (ret != KNOT_EOK) {
log_zone_warning(zone->name, "failed to write zone-in-journal (%s)",
knot_strerror(ret));
}
}
/* Store the zonefile SOA serial. */
zone->zonefile.serial = zone_contents_serial(contents);
......@@ -105,6 +130,10 @@ load_post:
bool dnssec_enable = conf_bool(&val);
val = conf_zone_get(conf, C_IXFR_DIFF, zone->name);
bool build_diffs = conf_bool(&val);
if (!build_diffs) {
val = conf_zone_get(conf, C_ZONE_IN_JOURNAL, zone->name);
build_diffs = conf_bool(&val);
}
bool old_contents = (zone->contents != NULL);
const bool contents_changed = old_contents && (contents != zone->contents);
......@@ -114,8 +143,8 @@ load_post:
if (old_contents) {
if (build_diffs && contents_changed) {
ret = zone_update_from_differences(&post_load, zone, contents, UPDATE_INCREMENTAL);
if (ret == KNOT_ERANGE || ret == KNOT_ENODIFF) {
log_zone_warning(zone->name, (ret == KNOT_ENODIFF ?
if (ret == KNOT_ERANGE || ret == KNOT_ESEMCHECK) {
log_zone_warning(zone->name, (ret == KNOT_ESEMCHECK ?
"failed to create journal entry, zone file changed without "
"SOA serial update" : "IXFR history will be lost, "
"zone file changed, but SOA serial decreased"));
......@@ -184,6 +213,11 @@ load_post:
zone_events_schedule_now(zone, ZONE_EVENT_NOTIFY);
}
if (loaded_from_journal) {
// this enforces further load from zone file and applying ixfr from diff
zone_events_schedule_now(zone, ZONE_EVENT_LOAD);
}
return KNOT_EOK;
fail:
......
......@@ -248,7 +248,7 @@ static int axfr_finalize(struct refresh_data *data)
}
ret = zone_in_journal_store(data->conf, data->zone, new_zone);
if (ret != KNOT_EOK && ret != KNOT_EEXIST && ret != KNOT_ENOTSUP) {
if (ret != KNOT_EOK && ret != KNOT_ENOTSUP) {
IXFRIN_LOG(LOG_WARNING, data->zone->name, data->remote,
"failed to write zone contents to journal (%s)",
knot_strerror(ret));
......
......@@ -914,6 +914,15 @@ static int load_bootstrap_changeset(journal_t *j, txn_t *_txn, changeset_t **ch)
return txn->ret;
}
static bool has_bootstrap_changeset(journal_t *j, txn_t *_txn)
{
reuse_txn(txn, j, _txn, false);
txn_key_str_u32(txn, j->zone, KEY_BOOTSTRAP_CHANGESET, 0);
bool res = txn_find(txn);
unreuse_txn(txn, _txn);
return res;
}
int journal_load_bootstrap(journal_t *j, list_t *dst)
{
if (j == NULL || j->db == NULL || dst == NULL) return KNOT_EINVAL;
......@@ -1175,7 +1184,7 @@ static int merge_itercb(iteration_ctx_t *ctx)
return ret;
}
static int merge_unflushed_changesets(journal_t *j, txn_t *_txn, changeset_t **mch)
static int merge_unflushed_changesets(journal_t *j, txn_t *_txn, changeset_t **mch, bool *merged_bootstrap)
{
reuse_txn(txn, j, _txn, false);
txn_check_ret(txn);
......@@ -1183,18 +1192,24 @@ static int merge_unflushed_changesets(journal_t *j, txn_t *_txn, changeset_t **m
if (md_flushed(txn)) {
goto m_u_ch_end;
}
bool was_merged = md_flag(txn, MERGED_SERIAL_VALID);
bool was_flushed = md_flag(txn, LAST_FLUSHED_VALID);
uint32_t from = was_merged ? txn->shadow_md.merged_serial :
(was_flushed ? txn->shadow_md.last_flushed :
txn->shadow_md.first_serial);
txn->ret = load_one(j, txn, from, mch);
if (!was_merged && was_flushed && txn->ret == KNOT_EOK) {
// we have to jump to ONE AFTER last_flushed
from = knot_soa_serial(&(*mch)->soa_to->rrs);
changeset_free(*mch);
*mch = NULL;
uint32_t from;
txn->ret = load_bootstrap_changeset(j, txn, mch);
*merged_bootstrap = (txn->ret == KNOT_EOK);
if (txn->ret == KNOT_ENOENT) { // no bootstrap changeset (normal operation)
bool was_merged = md_flag(txn, MERGED_SERIAL_VALID);
bool was_flushed = md_flag(txn, LAST_FLUSHED_VALID);
txn->ret = KNOT_EOK;
from = was_merged ? txn->shadow_md.merged_serial :
(was_flushed ? txn->shadow_md.last_flushed :
txn->shadow_md.first_serial);
txn->ret = load_one(j, txn, from, mch);
if (!was_merged && was_flushed && txn->ret == KNOT_EOK) {
// we have to jump to ONE AFTER last_flushed
from = knot_soa_serial(&(*mch)->soa_to->rrs);
changeset_free(*mch);
*mch = NULL;
txn->ret = load_one(j, txn, from, mch);
}
}
if (txn->ret != KNOT_EOK) {
goto m_u_ch_end;
......@@ -1220,7 +1235,7 @@ m_u_ch_end:
if (!md_flushed(txn)) { \
if (journal_merge_allowed(j)) { \
changeset_t *merged; \
merge_unflushed_changesets(j, txn, &merged); \
merge_unflushed_changesets(j, txn, &merged, &merged_into_bootstrap); \
if (txn->ret != KNOT_EOK) { \
goto store_changeset_cleanup; \
} \
......@@ -1248,18 +1263,26 @@ static int store_changesets(journal_t *j, list_t *changesets)
size_t *chunksizes = NULL;
knot_db_val_t *vals = NULL;
int inserting_merged = false;
bool inserting_merged = false;
bool merged_into_bootstrap = false;
bool inserting_bootstrap = false;
size_t occupied_last, occupied_now = knot_db_lmdb_get_usage(j->db->db);
WALK_LIST(ch, *changesets) {
nchs++;
serialized_size_total += changeset_serialized_size(ch);
if (ch->soa_from == NULL) {
inserting_bootstrap = true;
}
}
local_txn_t(txn, j);
txn_begin(txn, true);
bool zone_in_journal = has_bootstrap_changeset(j, txn);
bool merge_allowed = journal_merge_allowed(j);
// if you're tempted to add dirty_serial deletion somewhere here, you're wrong. Don't do it.
// PART 2 : recalculating the previous insert's occupy change
......@@ -1289,12 +1312,26 @@ static int store_changesets(journal_t *j, list_t *changesets)
md_set_common_last_inserter_zone(txn, j->zone);
}
// PART 3 : check if we exceeded designed occupation and delete some
// PART 3a : delete all if inserting bootstrap changeset
if (inserting_bootstrap) {
drop_journal(j, txn);
txn_restart(txn);
}
// PART 3b : check if we exceeded designed occupation and delete some
uint64_t occupied = 0, occupied_max;
md_get(txn, j->zone, MDKEY_PERZONE_OCCUPIED, &occupied);
occupied_max = journal_max_usage(j);
occupied += serialized_size_total;
if (occupied > occupied_max) {
if (zone_in_journal && occupied > occupied_max) {
if (merge_allowed) {
try_flush // decrease journal occupation by merging all into bootstrap changeset
} else {
txn->ret = KNOT_ESPACE;
log_zone_warning(j->zone, "journal, unable to make free space for insert");
goto store_changeset_cleanup;
}
} else if (occupied > occupied_max) {
size_t freed;
size_t tofree = (occupied - occupied_max) * journal_tofree_factor(j);
size_t free_min = tofree * journal_minfree_factor(j);
......@@ -1312,10 +1349,18 @@ static int store_changesets(journal_t *j, list_t *changesets)
}
}
// PART 3.5 : check if we exceeded history depth
// PART 3c : check if we exceeded history depth
long over_limit = (long)txn->shadow_md.changeset_count - journal_max_changesets(j) +
list_size(changesets) - (inserting_merged ? 1 : 0);
if (over_limit > 0) {
if (zone_in_journal && over_limit > 0) {
if (merge_allowed) {
try_flush // decrease journal occupation by merging all into bootstrap changeset
} else {
txn->ret = KNOT_ESPACE;
log_zone_warning(j->zone, "journal, unable to make free slot for insert");
goto store_changeset_cleanup;
}
} else if (over_limit > 0) {
size_t deled;
delete_count(j, txn, over_limit, &deled);
over_limit -= deled;
......@@ -1328,22 +1373,27 @@ static int store_changesets(journal_t *j, list_t *changesets)
// PART 4: continuity and duplicity check
changeset_t * chs_head = (HEAD(*changesets));
bool is_bootstrap = (chs_head->soa_from == NULL);
uint32_t serial = is_bootstrap ? 0 : knot_soa_serial(&chs_head->soa_from->rrs);
if (!is_bootstrap && md_flag(txn, SERIAL_TO_VALID) &&
serial_compare(txn->shadow_md.last_serial_to, serial) != 0) {
bool is_first_bootstrap = (chs_head->soa_from == NULL);
uint32_t serial = is_first_bootstrap ? 0 : knot_soa_serial(&chs_head->soa_from->rrs);
if (md_flag(txn, SERIAL_TO_VALID) && (is_first_bootstrap ||
serial_compare(txn->shadow_md.last_serial_to, serial) != 0) &&
!inserting_bootstrap /* if inserting bootstrap, drop_journal() was called, so no discontinuity */) {
log_zone_warning(j->zone, "journal, discontinuity in changes history (%u -> %u), dropping older changesets",
txn->shadow_md.last_serial_to, serial);
if (!md_flushed(txn) && !journal_merge_allowed(j)) {
txn->ret = KNOT_EBUSY; // aka try_flush, but don't merge
if (zone_in_journal) {
txn->ret = KNOT_ERANGE; // we can't drop history if zone-in-journal, so this is forbidden
goto store_changeset_cleanup;
} else if (merge_allowed) {
// flush would only merge and drop would delete the merge, so skip it
} else {
try_flush
}
drop_journal(j, txn);
txn_restart(txn);
}
WALK_LIST(ch, *changesets) {
uint32_t serial_to = knot_soa_serial(&ch->soa_to->rrs);
bool is_this_bootstrap = (is_bootstrap && ch == HEAD(*changesets));
bool is_this_bootstrap = (ch->soa_from == NULL);
bool is_this_merged = (inserting_merged && ch == TAIL(*changesets));
if (is_this_bootstrap || is_this_merged) {
continue;
......@@ -1352,9 +1402,18 @@ static int store_changesets(journal_t *j, list_t *changesets)
if (txn_find(txn)) {
log_zone_warning(j->zone, "journal, duplicate changeset serial (%u), dropping older changesets",
serial_to);
try_flush
delete_upto(j, txn, txn->shadow_md.first_serial, serial_to);
txn_restart(txn);
if (zone_in_journal) {
if (merge_allowed) {
try_flush // merge will get rid of the duplicity => OK
} else {
txn->ret = KNOT_EEXIST; // we can't fix it in this case, refuse to do it
goto store_changeset_cleanup;
}
} else {
try_flush
delete_upto(j, txn, txn->shadow_md.first_serial, serial_to);
txn_restart(txn);
}
}
}
......@@ -1383,8 +1442,8 @@ static int store_changesets(journal_t *j, list_t *changesets)
break;
}
bool is_this_bootstrap = is_bootstrap && (ch == HEAD(*changesets));
bool is_this_merged = (inserting_merged && ch == TAIL(*changesets));
bool is_this_bootstrap = (ch->soa_from == NULL);
uint32_t serial = is_this_bootstrap ? 0 : knot_soa_serial(&ch->soa_from->rrs);
uint32_t serial_to = knot_soa_serial(&ch->soa_to->rrs);
......@@ -1420,7 +1479,7 @@ static int store_changesets(journal_t *j, list_t *changesets)
if (txn->ret != KNOT_EOK) {
break;
}
if (is_this_merged) {
if (is_this_merged && !merged_into_bootstrap) {
txn->shadow_md.flags |= MERGED_SERIAL_VALID;
txn->shadow_md.merged_serial = serial;
}
......
......@@ -574,7 +574,7 @@ static int commit_full(conf_t *conf, zone_update_t *update, zone_contents_t **co
/* Store new zone contents in journal. */
if (update->zone->journal_db != NULL) { // TODO this should be better
ret = zone_in_journal_store(conf, update->zone, update->new_cont);
if (ret != KNOT_EOK && ret != KNOT_EEXIST && ret != KNOT_ENOTSUP) {
if (ret != KNOT_EOK && ret != KNOT_ENOTSUP) {
return ret;
}
}
......
......@@ -352,17 +352,26 @@ int zone_contents_diff(const zone_contents_t *zone1, const zone_contents_t *zone
return KNOT_EINVAL;
}
int ret = load_soas(zone1, zone2, changeset);
int ret_soa = load_soas(zone1, zone2, changeset);
if (ret_soa != KNOT_EOK && ret_soa != KNOT_ENODIFF) {
return ret_soa;
}
int ret = load_trees(zone1->nodes, zone2->nodes, changeset);
if (ret != KNOT_EOK) {
return ret;
}
ret = load_trees(zone1->nodes, zone2->nodes, changeset);
ret = load_trees(zone1->nsec3_nodes, zone2->nsec3_nodes, changeset);
if (ret != KNOT_EOK) {
return ret;
}
return load_trees(zone1->nsec3_nodes, zone2->nsec3_nodes, changeset);
if (ret_soa == KNOT_ENODIFF && !changeset_empty(changeset)) {
return KNOT_ESEMCHECK;
}
return ret_soa;
}
int zone_tree_add_diff(zone_tree_t *t1, zone_tree_t *t2, changeset_t *changeset)
......
......@@ -370,12 +370,6 @@ int zone_in_journal_store(conf_t *conf, zone_t *zone, zone_contents_t *new_conte
return KNOT_ENOTSUP;
}
if (journal_exists(zone->journal_db, zone->name)) {
// for now we refuse storing zone-in-journal into existing journal
// TODO maybe store the diff in such case as a normal changeset ?
return KNOT_EEXIST;
}
changeset_t *co_ch = changeset_from_contents(new_contents);
int ret = co_ch ? zone_change_store(conf, zone, co_ch) : KNOT_ENOMEM;
changeset_from_contents_free(co_ch);
......
......@@ -482,6 +482,9 @@ static void test_store_load(void)
init_list(&l);
init_list(&k);
ret = journal_scrape(j);
ok(ret == KNOT_EOK, "journal: scrape must be ok");
unset_conf();
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment