stats: stats module for mean/variance RTT

Slow start evaluates all NS in the preference list, using mean as
a sorting function. Invalidated NSs are pushed to the back of the
preference list.

knot-resolver.files

@@ -21,3 +21,4 @@ lib/rplan.h
 lib/rplan.c
 lib/layer/stats.h
 lib/layer/stats.c
+lib/defines.h

lib/context.c

@@ -54,7 +54,6 @@ int kr_result_init(struct kr_context *ctx, struct kr_result *result)
 	knot_wire_set_qr(ans->wire);
 
 	result->ans = ans;
-	gettimeofday(&result->t_start, NULL);
 
 	return 0;
 }

lib/context.h

@@ -27,6 +27,7 @@ struct kr_result {
 	knot_pkt_t *ans;
 	unsigned flags;
 	struct timeval t_start, t_end;
+	unsigned total_rtt;
 	unsigned nr_queries;
 };
 

lib/defines.h

+/* Copyright 2014 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#pragma once
+
+/*
+ * Connection limits.
+ */
+#define KR_CONN_RTT_MAX 10000

lib/delegpt.c

 #include "lib/delegpt.h"
+#include "lib/defines.h"
 #include <common/mempool.h>
 
 static void delegpt_free(struct kr_ns *dp, mm_ctx_t *mm)
@@ -104,6 +105,22 @@ struct kr_ns *kr_ns_find(list_t *list, const knot_dname_t *name)
 	return NULL;
 }
 
+void kr_ns_invalidate(struct kr_ns *ns)
+{
+	/* Slow start. */
+	ns->flags = DP_LAME;
+	ns->stat.M = KR_CONN_RTT_MAX;
+	ns->stat.S = 0;
+	ns->stat.n = 1;
+
+	/* Move to the end of the preference list. */
+	node_t *next = ns->node.next;
+	if (next->next) {
+		rem_node(&ns->node);
+		insert_node(&ns->node, next);
+	}
+}
+
 void kr_ns_remove(struct kr_ns *ns, mm_ctx_t *mm)
 {
 	rem_node((node_t *)ns);

lib/delegpt.h

@@ -35,7 +35,10 @@ struct kr_ns {
 	knot_dname_t *name;
 	struct sockaddr_storage addr;
 	unsigned valid_until;
-	unsigned mean_rtt;
+	struct {
+		double M, S; /* Mean, Variance S/n */
+		unsigned n;
+	} stat;
 	unsigned flags;
 };
 
@@ -56,4 +59,6 @@ list_t *kr_delegmap_find(struct kr_delegmap *map, const knot_dname_t *name);
 
 struct kr_ns *kr_ns_get(list_t *list, const knot_dname_t *name, mm_ctx_t *mm);
 struct kr_ns *kr_ns_find(list_t *list, const knot_dname_t *name);
+void kr_ns_invalidate(struct kr_ns *ns);
 void kr_ns_remove(struct kr_ns *ns, mm_ctx_t *mm);
+

lib/layer/stats.c

@@ -20,6 +20,44 @@ limitations under the License.
 
 #define DEBUG_MSG(fmt, ...) fprintf(stderr, "[stats] " fmt, ## __VA_ARGS__)
 
+static void update_ns_preference(struct kr_ns *ns, struct kr_ns *next)
+{
+	assert(ns);
+	assert(next);
+
+	/* Push down if next has better score. */
+	if (next->stat.M < ns->stat.M) {
+		rem_node(&ns->node);
+		insert_node(&ns->node, &next->node);
+	}
+}
+
+static void update_ns_preference_list(struct kr_ns *cur)
+{
+	assert(cur);
+	struct kr_ns *next = (struct kr_ns *)cur->node.next;
+
+	/* O(n), walk the list (shouldn't be too large). */
+	/* TODO: cut on first swap? random swaps? */
+	while (next->node.next != NULL) {
+		update_ns_preference(cur, next);
+		cur  = next;
+		next = (struct kr_ns *)cur->node.next;
+	}
+}
+
+static void update_stats(struct kr_ns *ns, double rtt)
+{
+	/* Knuth, TAOCP, p.232 (Welford running variance/mean). */
+	double d_mean = (rtt - ns->stat.M);
+	ns->stat.n += 1;
+	ns->stat.M += d_mean / ns->stat.n;
+	ns->stat.S += d_mean * (rtt - ns->stat.M);
+
+	/* Update NS position in preference list. */
+	update_ns_preference_list(ns);
+}
+
 static int begin(knot_layer_t *ctx, void *param)
 {
 	ctx->data = param;
@@ -38,7 +76,7 @@ static int finish(knot_layer_t *ctx)
 
 	DEBUG_MSG("rcode: %d (%u RRs)\n", knot_wire_get_rcode(result->ans->wire), result->ans->rrset_count);
 	DEBUG_MSG("queries: %u\n", result->nr_queries);
-	DEBUG_MSG("total time: %.02f msecs\n", time_diff(&result->t_start, &result->t_end));
+	DEBUG_MSG("total time: %u msecs\n", result->total_rtt);
 #endif
 
 	return ctx->state;
@@ -51,6 +89,9 @@ static int query(knot_layer_t *ctx, knot_pkt_t *pkt)
 
 	result->nr_queries += 1;
 
+	/* Store stats. */
+	gettimeofday(&result->t_start, NULL);
+
 	return ctx->state;
 }
 
@@ -60,19 +101,27 @@ static int answer(knot_layer_t *ctx, knot_pkt_t *pkt)
 	struct kr_layer_param *param = ctx->data;
 	struct kr_context* resolve = param->ctx;
 	struct kr_result *result = param->result;
+	struct kr_ns *ns = resolve->current_ns;
 
 	/* Store stats. */
 	gettimeofday(&result->t_end, NULL);
 
+	/* Update NS statistics. */
+	double rtt = time_diff(&result->t_start, &result->t_end);
+	if (rtt > 0.0) {
+		update_stats(ns, rtt);
+		result->total_rtt += rtt;
+	}
+
 #ifndef NDEBUG
-	char *ns_name = knot_dname_to_str(resolve->current_ns->name);
+	char *ns_name = knot_dname_to_str(ns->name);
 	char pad[16];
 	memset(pad, '-', sizeof(pad));
 	pad[MIN(sizeof(pad) - 1, list_size(&resolve->rplan.q) * 2)] = '\0';
-	DEBUG_MSG("#%s %s ... RC=%d, AA=%d, cumulative time: %.02f msecs\n",
+	DEBUG_MSG("#%s %s ... RC=%d, AA=%d, RTT: %.02f msecs\n",
 	          pad, ns_name, knot_wire_get_rcode(pkt->wire),
 	          knot_wire_get_aa(pkt->wire) != 0,
-	          time_diff(&result->t_start, &result->t_end));
+	          rtt);
 	free(ns_name);
 #endif
 

lib/resolve.c

@@ -4,6 +4,7 @@
 #include <libknot/processing/requestor.h>
 #include <libknot/descriptor.h>
 #include "lib/resolve.h"
+#include "lib/defines.h"
 #include "lib/layer/iterate.h"
 #include "lib/layer/static.h"
 #include "lib/layer/stats.h"
@@ -29,7 +30,7 @@ static int resolve_ns(struct kr_context *resolve, struct kr_ns *ns)
 
 static void iterate(struct knot_requestor *requestor, struct kr_context* ctx)
 {
-	struct timeval timeout = { 5, 0 };
+	struct timeval timeout = { KR_CONN_RTT_MAX / 1000, 0 };
 	const struct kr_query *next = kr_rplan_next(&ctx->rplan);
 	assert(next);
 
@@ -66,7 +67,8 @@ static void iterate(struct knot_requestor *requestor, struct kr_context* ctx)
 	knot_requestor_enqueue(requestor, tx);
 	int ret = knot_requestor_exec(requestor, &timeout);
 	if (ret != 0) {
-		kr_ns_remove(ns, ctx->dp_map.pool);
+		/* Resolution failed, invalidate current resolver. */
+		kr_ns_invalidate(ns);
 	}
 
 	/* Pop resolved query. */