worker.c 69.2 KB
Newer Older
1
/*  Copyright (C) 2014-2017 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
Marek Vavruša's avatar
Marek Vavruša committed
2 3 4 5 6 7 8 9 10 11 12 13

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
14
    along with this program.  If not, see <https://www.gnu.org/licenses/>.
Marek Vavruša's avatar
Marek Vavruša committed
15 16
 */

17
#include <uv.h>
18
#include <lua.h>
19
#include <libknot/packet/pkt.h>
20
#include <libknot/descriptor.h>
21 22
#include <contrib/ucw/lib.h>
#include <contrib/ucw/mempool.h>
23
#include <contrib/wire.h>
Marek Vavruša's avatar
Marek Vavruša committed
24 25 26
#if defined(__GLIBC__) && defined(_GNU_SOURCE)
#include <malloc.h>
#endif
27
#include <assert.h>
28 29
#include <sys/types.h>
#include <unistd.h>
30
#include <gnutls/gnutls.h>
31
#include "lib/utils.h"
32
#include "lib/layer.h"
33
#include "daemon/worker.h"
34
#include "daemon/bindings.h"
35
#include "daemon/engine.h"
36
#include "daemon/io.h"
37
#include "daemon/tls.h"
38

39 40
#define VERBOSE_MSG(qry, fmt...) QRVERBOSE(qry, "wrkr", fmt)

41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
/** Client request state. */
struct request_ctx
{
	struct kr_request req;
	struct {
		union inaddr addr;
		union inaddr dst_addr;
		/* uv_handle_t *handle; */

		/** NULL if the request didn't come over network. */
		struct session *session;
	} source;
	struct worker_ctx *worker;
	qr_tasklist_t tasks;
};

/** Query resolution task. */
struct qr_task
{
	struct request_ctx *ctx;
	knot_pkt_t *pktbuf;
	qr_tasklist_t waiting;
	uv_handle_t *pending[MAX_PENDING];
	uint16_t pending_count;
	uint16_t addrlist_count;
	uint16_t addrlist_turn;
	uint16_t timeouts;
	uint16_t iter_count;
	uint16_t bytes_remaining;
	struct sockaddr *addrlist;
	uint32_t refs;
	bool finished : 1;
	bool leading  : 1;
74 75
};

76

77 78 79 80
/* Convenience macros */
#define qr_task_ref(task) \
	do { ++(task)->refs; } while(0)
#define qr_task_unref(task) \
81
	do { if (task && --(task)->refs == 0) { qr_task_free(task); } } while (0)
82
#define qr_valid_handle(task, checked) \
83 84 85 86 87 88
	(!uv_is_closing((checked)) || (task)->ctx->source.session->handle == (checked))

/** @internal get key for tcp session
 *  @note kr_straddr() return pointer to static string
 */
#define tcpsess_key(addr) kr_straddr(addr)
89 90

/* Forward decls */
91
static void qr_task_free(struct qr_task *task);
92 93 94 95 96 97 98 99 100 101 102 103 104
static int qr_task_step(struct qr_task *task,
			const struct sockaddr *packet_source,
			knot_pkt_t *packet);
static int qr_task_send(struct qr_task *task, uv_handle_t *handle,
			struct sockaddr *addr, knot_pkt_t *pkt);
static int qr_task_finalize(struct qr_task *task, int state);
static void qr_task_complete(struct qr_task *task);
static int worker_add_tcp_connected(struct worker_ctx *worker,
				    const struct sockaddr *addr,
				    struct session *session);
static int worker_del_tcp_connected(struct worker_ctx *worker,
				    const struct sockaddr *addr);
static struct session* worker_find_tcp_connected(struct worker_ctx *worker,
105
						 const struct sockaddr *addr);
106 107 108 109 110 111
static int worker_add_tcp_waiting(struct worker_ctx *worker,
				  const struct sockaddr *addr,
				  struct session *session);
static int worker_del_tcp_waiting(struct worker_ctx *worker,
				  const struct sockaddr *addr);
static struct session* worker_find_tcp_waiting(struct worker_ctx *worker,
112
					       const struct sockaddr *addr);
113 114 115 116 117 118 119 120 121 122
static int session_add_waiting(struct session *session, struct qr_task *task);
static int session_del_waiting(struct session *session, struct qr_task *task);
static int session_add_tasks(struct session *session, struct qr_task *task);
static int session_del_tasks(struct session *session, struct qr_task *task);
static void session_close(struct session *session);
static void on_session_idle_timeout(uv_timer_t *timer);
static int timer_start(struct session *session, uv_timer_cb cb,
		       uint64_t timeout, uint64_t repeat);
static void on_tcp_connect_timeout(uv_timer_t *timer);
static void on_tcp_watchdog_timeout(uv_timer_t *timer);
123 124 125 126 127 128 129

/** @internal Get singleton worker. */
static inline struct worker_ctx *get_worker(void)
{
	return uv_default_loop()->data;
}

130 131 132 133
static inline void *iohandle_borrow(struct worker_ctx *worker)
{
	void *h = NULL;

134
	const size_t size = sizeof(uv_handles_t);
135 136 137 138 139 140 141 142 143 144 145 146
	if (worker->pool_iohandles.len > 0) {
		h = array_tail(worker->pool_iohandles);
		array_pop(worker->pool_iohandles);
		kr_asan_unpoison(h, size);
	} else {
		h = malloc(size);
	}

	return h;
}

static inline void iohandle_release(struct worker_ctx *worker, void *h)
147
{
148 149 150 151
	assert(h);

	if (worker->pool_iohandles.len < MP_FREELIST_SIZE) {
		array_push(worker->pool_iohandles, h);
152
		kr_asan_poison(h, sizeof(uv_handles_t));
153
	} else {
154
		free(h);
155 156 157
	}
}

158
void *worker_iohandle_borrow(struct worker_ctx *worker)
159
{
160 161 162 163 164 165 166 167 168
	return iohandle_borrow(worker);
}

void worker_iohandle_release(struct worker_ctx *worker, void *h)
{
	iohandle_release(worker, h);
}

static inline void *iorequest_borrow(struct worker_ctx *worker)
169
{
170 171
	void *r = NULL;

172
	const size_t size = sizeof(uv_reqs_t);
173 174 175 176
	if (worker->pool_ioreqs.len > 0) {
		r = array_tail(worker->pool_ioreqs);
		array_pop(worker->pool_ioreqs);
		kr_asan_unpoison(r, size);
177
	} else {
178
		r = malloc(size);
179
	}
180 181

	return r;
182 183
}

184
static inline void iorequest_release(struct worker_ctx *worker, void *r)
185
{
186 187 188 189
	assert(r);

	if (worker->pool_ioreqs.len < MP_FREELIST_SIZE) {
		array_push(worker->pool_ioreqs, r);
190
		kr_asan_poison(r, sizeof(uv_reqs_t));
191
	} else {
192
		free(r);
193 194 195
	}
}

196

197 198 199
/*! @internal Create a UDP/TCP handle for an outgoing AF_INET* connection.
 *  socktype is SOCK_* */
static uv_handle_t *ioreq_spawn(struct qr_task *task, int socktype, sa_family_t family)
200
{
201 202 203
	bool precond = (socktype == SOCK_DGRAM || socktype == SOCK_STREAM)
			&& (family == AF_INET  || family == AF_INET6);
	if (!precond) {
204 205
		/* assert(false); see #245 */
		kr_log_verbose("[work] ioreq_spawn: pre-condition failed\n");
206 207 208
		return NULL;
	}

209 210 211 212
	if (task->pending_count >= MAX_PENDING) {
		return NULL;
	}
	/* Create connection for iterative query */
213
	struct worker_ctx *worker = task->ctx->worker;
214 215
	void *h = iohandle_borrow(worker);
	uv_handle_t *handle = (uv_handle_t *)h;
216 217 218
	if (!handle) {
		return NULL;
	}
219
	io_create(worker->loop, handle, socktype);
220 221 222 223

	/* Bind to outgoing address, according to IP v4/v6. */
	union inaddr *addr;
	if (family == AF_INET) {
224
		addr = (union inaddr *)&worker->out_addr4;
225
	} else {
226
		addr = (union inaddr *)&worker->out_addr6;
227 228 229 230 231
	}
	int ret = 0;
	if (addr->ip.sa_family != AF_UNSPEC) {
		assert(addr->ip.sa_family == family);
		if (socktype == SOCK_DGRAM) {
232 233 234 235 236
			uv_udp_t *udp = (uv_udp_t *)handle;
			ret = uv_udp_bind(udp, &addr->ip, 0);
		} else if (socktype == SOCK_STREAM){
			uv_tcp_t *tcp = (uv_tcp_t *)handle;
			ret = uv_tcp_bind(tcp, &addr->ip, 0);
237 238 239
		}
	}

240 241
	/* Set current handle as a subrequest type. */
	struct session *session = handle->data;
242 243
	if (ret == 0) {
		session->outgoing = true;
244
		ret = session_add_tasks(session, task);
245
	}
246
	if (ret < 0) {
247
		io_deinit(handle);
248
		iohandle_release(worker, h);
249 250 251
		return NULL;
	}
	/* Connect or issue query datagram */
252
	task->pending[task->pending_count] = handle;
253
	task->pending_count += 1;
254
	return handle;
255 256
}

257
static void on_session_close(uv_handle_t *handle)
258
{
259 260
	uv_loop_t *loop = handle->loop;
	struct worker_ctx *worker = loop->data;
261
	struct session *session = handle->data;
262 263
	assert(session->handle == handle);
	session->handle = NULL;
264
	io_deinit(handle);
265
	iohandle_release(worker, handle);
266 267 268 269 270 271
}

static void on_session_timer_close(uv_handle_t *timer)
{
	struct session *session = timer->data;
	uv_handle_t *handle = session->handle;
272 273
	assert(handle && handle->data == session);
	assert (session->outgoing || handle->type == UV_TCP);
274 275 276
	if (!uv_is_closing(handle)) {
		uv_close(handle, on_session_close);
	}
277 278
}

279
static void ioreq_kill_udp(uv_handle_t *req, struct qr_task *task)
280 281
{
	assert(req);
282 283 284 285
	struct session *session = req->data;
	assert(session->outgoing);
	if (session->closing) {
		return;
286
	}
287 288 289 290
	uv_timer_stop(&session->timeout);
	session_del_tasks(session, task);
	assert(session->tasks.len == 0);
	session_close(session);
291 292
}

293
static void ioreq_kill_tcp(uv_handle_t *req, struct qr_task *task)
294
{
295 296 297 298 299 300 301 302 303 304 305 306 307
	assert(req);
	struct session *session = req->data;
	assert(session->outgoing);
	if (session->closing) {
		return;
	}

	session_del_waiting(session, task);
	session_del_tasks(session, task);

	int res = 0;

	if (session->outgoing && session->peer.ip.sa_family != AF_UNSPEC &&
308
	    session->tasks.len == 0 && session->waiting.len == 0 && !session->closing) {
309 310
		assert(session->peer.ip.sa_family == AF_INET ||
		       session->peer.ip.sa_family == AF_INET6);
311 312 313 314 315 316 317 318 319 320
		res = 1;
		if (session->connected) {
			/* This is outbound TCP connection which can be reused.
			* Close it after timeout */
			uv_timer_t *timer = &session->timeout;
			timer->data = session;
			uv_timer_stop(timer);
			res = uv_timer_start(timer, on_session_idle_timeout,
					     KR_CONN_RTT_MAX, 0);
		}
321 322 323 324 325
	}

	if (res != 0) {
		/* if any errors, close the session immediately */
		session_close(session);
326 327 328
	}
}

329
static void ioreq_kill_pending(struct qr_task *task)
330
{
331 332 333 334 335 336 337 338
	for (uint16_t i = 0; i < task->pending_count; ++i) {
		if (task->pending[i]->type == UV_UDP) {
			ioreq_kill_udp(task->pending[i], task);
		} else if (task->pending[i]->type == UV_TCP) {
			ioreq_kill_tcp(task->pending[i], task);
		} else {
			assert(false);
		}
339 340 341 342
	}
	task->pending_count = 0;
}

343 344 345 346 347 348 349 350
static void session_close(struct session *session)
{
	assert(session->tasks.len == 0 && session->waiting.len == 0);

	if (session->closing) {
		return;
	}

Grigorii Demidov's avatar
Grigorii Demidov committed
351
	if (!session->outgoing && session->buffering != NULL) {
352 353
		qr_task_complete(session->buffering);
	}
Grigorii Demidov's avatar
Grigorii Demidov committed
354
	session->buffering = NULL;
355

356 357
	uv_handle_t *handle = session->handle;
	io_stop_read(handle);
358 359 360 361 362 363 364 365 366 367 368 369
	session->closing = true;
	if (session->outgoing &&
	    session->peer.ip.sa_family != AF_UNSPEC) {
		struct worker_ctx *worker = get_worker();
		struct sockaddr *peer = &session->peer.ip;
		worker_del_tcp_connected(worker, peer);
		session->connected = false;
	}

	if (!uv_is_closing((uv_handle_t *)&session->timeout)) {
		uv_timer_stop(&session->timeout);
		if (session->tls_client_ctx) {
370
			tls_close(&session->tls_client_ctx->c);
371
		}
372
		if (session->tls_ctx) {
373
			tls_close(&session->tls_ctx->c);
374 375
		}

376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436
		session->timeout.data = session;
		uv_close((uv_handle_t *)&session->timeout, on_session_timer_close);
	}
}

static int session_add_waiting(struct session *session, struct qr_task *task)
{
	for (int i = 0; i < session->waiting.len; ++i) {
		if (session->waiting.at[i] == task) {
			return i;
		}
	}
	int ret = array_push(session->waiting, task);
	if (ret >= 0) {
		qr_task_ref(task);
	}
	return ret;
}

static int session_del_waiting(struct session *session, struct qr_task *task)
{
	int ret = kr_error(ENOENT);
	for (int i = 0; i < session->waiting.len; ++i) {
		if (session->waiting.at[i] == task) {
			array_del(session->waiting, i);
			qr_task_unref(task);
			ret = kr_ok();
			break;
		}
	}
	return ret;
}

static int session_add_tasks(struct session *session, struct qr_task *task)
{
	for (int i = 0; i < session->tasks.len; ++i) {
		if (session->tasks.at[i] == task) {
			return i;
		}
	}
	int ret = array_push(session->tasks, task);
	if (ret >= 0) {
		qr_task_ref(task);
	}
	return ret;
}

static int session_del_tasks(struct session *session, struct qr_task *task)
{
	int ret = kr_error(ENOENT);
	for (int i = 0; i < session->tasks.len; ++i) {
		if (session->tasks.at[i] == task) {
			array_del(session->tasks, i);
			qr_task_unref(task);
			ret = kr_ok();
			break;
		}
	}
	return ret;
}

437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460
/** @cond This memory layout is internal to mempool.c, use only for debugging. */
#if defined(__SANITIZE_ADDRESS__)
struct mempool_chunk {
  struct mempool_chunk *next;
  size_t size;
};
static void mp_poison(struct mempool *mp, bool poison)
{
	if (!poison) { /* @note mempool is part of the first chunk, unpoison it first */
		kr_asan_unpoison(mp, sizeof(*mp));
	}
	struct mempool_chunk *chunk = mp->state.last[0];
	void *chunk_off = (void *)chunk - chunk->size;
	if (poison) {
		kr_asan_poison(chunk_off, chunk->size);
	} else {
		kr_asan_unpoison(chunk_off, chunk->size);
	}
}
#else
#define mp_poison(mp, enable)
#endif
/** @endcond */

461
/** Get a mempool.  (Recycle if possible.)  */
462
static inline struct mempool *pool_borrow(struct worker_ctx *worker)
463 464
{
	struct mempool *mp = NULL;
465 466 467 468
	if (worker->pool_mp.len > 0) {
		mp = array_tail(worker->pool_mp);
		array_pop(worker->pool_mp);
		mp_poison(mp, 0);
469 470 471 472 473 474
	} else { /* No mempool on the freelist, create new one */
		mp = mp_new (4 * CPU_PAGE_SIZE);
	}
	return mp;
}

475
/** Return a mempool.  (Cache them up to some count.) */
476 477
static inline void pool_release(struct worker_ctx *worker, struct mempool *mp)
{
478
	if (worker->pool_mp.len < MP_FREELIST_SIZE) {
479
		mp_flush(mp);
480
		array_push(worker->pool_mp, mp);
481
		mp_poison(mp, 1);
482 483 484 485 486
	} else {
		mp_delete(mp);
	}
}

487 488 489 490 491 492 493 494
/** Create and initialize a request_ctx (on a fresh mempool).
 *
 * handle and addr point to the source of the request, and they are NULL
 * in case the request didn't come from network.
 */
static struct request_ctx *request_create(struct worker_ctx *worker,
					  uv_handle_t *handle,
					  const struct sockaddr *addr)
495
{
496
	knot_mm_t pool = {
497
		.ctx = pool_borrow(worker),
498
		.alloc = (knot_mm_alloc_t) mp_alloc
499
	};
500

501 502 503 504
	/* Create request context */
	struct request_ctx *ctx = mm_alloc(&pool, sizeof(*ctx));
	if (!ctx) {
		pool_release(worker, pool.ctx);
505 506
		return NULL;
	}
507

508 509 510 511 512
	memset(ctx, 0, sizeof(*ctx));

	/* TODO Relocate pool to struct request */
	ctx->worker = worker;
	array_init(ctx->tasks);
513 514 515
	struct session *session = handle ? handle->data : NULL;
	if (session) {
		assert(session->outgoing == false);
516
	}
517
	ctx->source.session = session;
518 519 520 521

	struct kr_request *req = &ctx->req;
	req->pool = pool;

522
	/* Remember query source addr */
523 524 525
	if (!addr || (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)) {
		ctx->source.addr.ip.sa_family = AF_UNSPEC;
	} else {
526 527 528
		size_t addr_len = sizeof(struct sockaddr_in);
		if (addr->sa_family == AF_INET6)
			addr_len = sizeof(struct sockaddr_in6);
529 530
		memcpy(&ctx->source.addr.ip, addr, addr_len);
		ctx->req.qsource.addr = &ctx->source.addr.ip;
531
	}
532 533 534 535 536

	worker->stats.rconcurrent += 1;

	if (!handle) {
		return ctx;
537
	}
538

539
	/* Remember the destination address. */
540 541 542 543 544 545
	int addr_len = sizeof(ctx->source.dst_addr);
	struct sockaddr *dst_addr = &ctx->source.dst_addr.ip;
	ctx->source.dst_addr.ip.sa_family = AF_UNSPEC;
	if (handle->type == UV_UDP) {
		if (uv_udp_getsockname((uv_udp_t *)handle, dst_addr, &addr_len) == 0) {
			req->qsource.dst_addr = dst_addr;
546
		}
547 548 549 550
		req->qsource.tcp = false;
	} else if (handle->type == UV_TCP) {
		if (uv_tcp_getsockname((uv_tcp_t *)handle, dst_addr, &addr_len) == 0) {
			req->qsource.dst_addr = dst_addr;
551
		}
552
		req->qsource.tcp = true;
553
	}
554 555

	return ctx;
556 557
}

558 559
/** More initialization, related to the particular incoming query/packet. */
static int request_start(struct request_ctx *ctx, knot_pkt_t *query)
560
{
561 562 563 564 565 566 567 568 569 570 571
	assert(query && ctx);
	size_t answer_max = KNOT_WIRE_MIN_PKTSIZE;
	struct kr_request *req = &ctx->req;

	/* source.session can be empty if request was generated by kresd itself */
	if (!ctx->source.session ||
	     ctx->source.session->handle->type == UV_TCP) {
		answer_max = KNOT_WIRE_MAX_PKTSIZE;
	} else if (knot_pkt_has_edns(query)) { /* EDNS */
		answer_max = MAX(knot_edns_get_payload(query->opt_rr),
				 KNOT_WIRE_MIN_PKTSIZE);
572
	}
573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
	req->qsource.size = query->size;

	req->answer = knot_pkt_new(NULL, answer_max, &req->pool);
	if (!req->answer) {
		return kr_error(ENOMEM);
	}

	/* Remember query source TSIG key */
	if (query->tsig_rr) {
		req->qsource.key = knot_rrset_copy(query->tsig_rr, &req->pool);
	}

	/* Remember query source EDNS data */
	if (query->opt_rr) {
		req->qsource.opt = knot_rrset_copy(query->opt_rr, &req->pool);
	}
	/* Start resolution */
	struct worker_ctx *worker = ctx->worker;
	struct engine *engine = worker->engine;
	kr_resolve_begin(req, &engine->resolver, req->answer);
	worker->stats.queries += 1;
	/* Throttle outbound queries only when high pressure */
	if (worker->stats.concurrent < QUERY_RATE_THRESHOLD) {
		req->options.NO_THROTTLE = true;
	}
	return kr_ok();
}

static void request_free(struct request_ctx *ctx)
{
	struct worker_ctx *worker = ctx->worker;
604
	/* Return mempool to ring or free it if it's full */
605
	pool_release(worker, ctx->req.pool.ctx);
606
	/* @note The 'task' is invalidated from now on. */
Marek Vavruša's avatar
Marek Vavruša committed
607
	/* Decommit memory every once in a while */
608
	static int mp_delete_count = 0;
609 610 611
	if (++mp_delete_count == 100000) {
		lua_gc(worker->engine->L, LUA_GCCOLLECT, 0);
#if defined(__GLIBC__) && defined(_GNU_SOURCE)
Marek Vavruša's avatar
Marek Vavruša committed
612
		malloc_trim(0);
613
#endif
Marek Vavruša's avatar
Marek Vavruša committed
614
		mp_delete_count = 0;
615
	}
616
	worker->stats.rconcurrent -= 1;
617
}
618

619
static int request_add_tasks(struct request_ctx *ctx, struct qr_task *task)
620
{
621 622 623 624 625 626 627 628
	for (int i = 0; i < ctx->tasks.len; ++i) {
		if (ctx->tasks.at[i] == task) {
			return i;
		}
	}
	int ret = array_push(ctx->tasks, task);
	if (ret >= 0) {
		qr_task_ref(task);
629
	}
630 631
	return ret;
}
632

633 634 635 636 637 638 639 640 641 642
static int request_del_tasks(struct request_ctx *ctx, struct qr_task *task)
{
	int ret = kr_error(ENOENT);
	for (int i = 0; i < ctx->tasks.len; ++i) {
		if (ctx->tasks.at[i] == task) {
			array_del(ctx->tasks, i);
			qr_task_unref(task);
			ret = kr_ok();
			break;
		}
643
	}
644 645
	return ret;
}
646

647 648 649 650 651 652 653 654 655

static struct qr_task *qr_task_create(struct request_ctx *ctx)
{
	/* How much can client handle? */
	struct engine *engine = ctx->worker->engine;
	size_t pktbuf_max = KR_EDNS_PAYLOAD;
	if (engine->resolver.opt_rr) {
		pktbuf_max = MAX(knot_edns_get_payload(engine->resolver.opt_rr),
				 pktbuf_max);
656 657
	}

658 659 660 661
	/* Create resolution task */
	struct qr_task *task = mm_alloc(&ctx->req.pool, sizeof(*task));
	if (!task) {
		return NULL;
662
	}
663
	memset(task, 0, sizeof(*task)); /* avoid accidentally unitialized fields */
664

665 666 667 668 669
	/* Create packet buffers for answer and subrequests */
	knot_pkt_t *pktbuf = knot_pkt_new(NULL, pktbuf_max, &ctx->req.pool);
	if (!pktbuf) {
		mm_free(&ctx->req.pool, task);
		return NULL;
670
	}
671
	pktbuf->size = 0;
672

673 674 675 676 677 678 679 680 681
	task->ctx = ctx;
	task->pktbuf = pktbuf;
	array_init(task->waiting);
	task->refs = 0;
	int ret = request_add_tasks(ctx, task);
	if (ret < 0) {
		mm_free(&ctx->req.pool, task);
		mm_free(&ctx->req.pool, pktbuf);
		return NULL;
682
	}
683 684
	ctx->worker->stats.concurrent += 1;
	return task;
685 686
}

687 688 689 690 691 692 693 694 695 696 697 698
/* This is called when the task refcount is zero, free memory. */
static void qr_task_free(struct qr_task *task)
{
	struct request_ctx *ctx = task->ctx;

	assert(ctx);

	/* Process outbound session. */
	struct session *source_session = ctx->source.session;
	struct worker_ctx *worker = ctx->worker;

	/* Process source session. */
Grigorii Demidov's avatar
Grigorii Demidov committed
699 700 701 702
	if (source_session &&
	    source_session->tasks.len < worker->tcp_pipeline_max/2 &&
	    !source_session->closing && source_session->throttled) {
		uv_handle_t *handle = source_session->handle;
703 704
		/* Start reading again if the session is throttled and
		 * the number of outgoing requests is below watermark. */
Grigorii Demidov's avatar
Grigorii Demidov committed
705 706 707
		if (handle) {
			io_start_read(handle);
			source_session->throttled = false;
708 709 710 711 712 713 714 715 716 717 718 719 720
		}
	}

	if (ctx->tasks.len == 0) {
		array_clear(ctx->tasks);
		request_free(ctx);
	}

	/* Update stats */
	worker->stats.concurrent -= 1;
}

/*@ Register new qr_task within session. */
721 722
static int qr_task_register(struct qr_task *task, struct session *session)
{
723
	assert(session->outgoing == false && session->handle->type == UV_TCP);
724

725 726 727 728
	int ret = array_reserve(session->tasks, session->tasks.len + 1);
	if (ret != 0) {
		return kr_error(ENOMEM);
	}
729 730 731 732 733 734

	session_add_tasks(session, task);

	struct request_ctx *ctx = task->ctx;
	assert(ctx && (ctx->source.session == NULL || ctx->source.session == session));
	ctx->source.session = session;
735 736 737 738 739
	/* Soft-limit on parallel queries, there is no "slow down" RCODE
	 * that we could use to signalize to client, but we can stop reading,
	 * an in effect shrink TCP window size. To get more precise throttling,
	 * we would need to copy remainder of the unread buffer and reassemble
	 * when resuming reading. This is NYI.  */
740 741
	if (session->tasks.len >= task->ctx->worker->tcp_pipeline_max) {
		uv_handle_t *handle = session->handle;
Grigorii Demidov's avatar
Grigorii Demidov committed
742
		if (handle && !session->throttled && !session->closing) {
743 744 745 746
			io_stop_read(handle);
			session->throttled = true;
		}
	}
747

748 749 750 751
	return 0;
}

static void qr_task_complete(struct qr_task *task)
752
{
753
	struct request_ctx *ctx = task->ctx;
754

755
	/* Kill pending I/O requests */
756
	ioreq_kill_pending(task);
757 758
	assert(task->waiting.len == 0);
	assert(task->leading == false);
759

Grigorii Demidov's avatar
Grigorii Demidov committed
760 761 762 763 764 765
	struct session *source_session = ctx->source.session;
	if (source_session) {
		assert(source_session->outgoing == false &&
		       source_session->waiting.len == 0);
		session_del_tasks(source_session, task);
	}
766

767
	/* Release primary reference to task. */
768
	request_del_tasks(ctx, task);
769 770
}

771
/* This is called when we send subrequest / answer */
772
static int qr_task_on_send(struct qr_task *task, uv_handle_t *handle, int status)
773
{
774 775 776 777 778 779 780
	if (task->finished) {
		assert(task->leading == false);
		qr_task_complete(task);
		if (!handle || handle->type != UV_TCP) {
			return status;
		}
		struct session* session = handle->data;
Grigorii Demidov's avatar
Grigorii Demidov committed
781
		assert(session);
782 783 784 785 786 787
		if (!session->outgoing ||
		    session->waiting.len == 0) {
			return status;
		}
	}

Grigorii Demidov's avatar
Grigorii Demidov committed
788
	if (handle) {
789
		struct session* session = handle->data;
Grigorii Demidov's avatar
Grigorii Demidov committed
790 791 792
		if (!session->outgoing && task->ctx->source.session) {
			assert (task->ctx->source.session->handle == handle);
		}
793 794 795
		if (handle->type == UV_TCP && session->outgoing &&
		    session->waiting.len > 0) {
			session_del_waiting(session, task);
Grigorii Demidov's avatar
Grigorii Demidov committed
796 797 798
			if (session->closing) {
				return status;
			}
799 800 801 802 803
			/* Finalize the task, if any errors.
			 * We can't add it to the end of waiting list for retrying
			 * since it may lead endless loop in some circumstances
			 * (for instance: tls; send->tls_push->too many non-critical errors->
			 * on_send with nonzero status->re-add to waiting->send->etc).*/
Grigorii Demidov's avatar
Grigorii Demidov committed
804
			if (status != 0) {
805 806 807 808 809 810 811
				if (session->outgoing) {
					qr_task_finalize(task, KR_STATE_FAIL);
				} else {
					assert(task->ctx->source.session == session);
					task->ctx->source.session = NULL;
				}
				session_del_tasks(session, task);
812
			}
813 814
			if (session->waiting.len > 0) {
				struct qr_task *t = session->waiting.at[0];
815
				int ret = qr_task_send(t, handle, &session->peer.ip, t->pktbuf);
Grigorii Demidov's avatar
Grigorii Demidov committed
816 817 818 819 820 821
				if (ret == kr_ok()) {
					uv_timer_t *timer = &session->timeout;
					uv_timer_stop(timer);
					session->timeout.data = session;
					timer_start(session, on_tcp_watchdog_timeout, MAX_TCP_INACTIVITY, 0);
				} else {
822 823
					uv_timer_t *timer = &session->timeout;
					uv_timer_stop(timer);
824
					while (session->waiting.len > 0) {
Grigorii Demidov's avatar
Grigorii Demidov committed
825
						struct qr_task *t = session->waiting.at[0];
826
						if (session->outgoing) {
Grigorii Demidov's avatar
Grigorii Demidov committed
827
							qr_task_finalize(t, KR_STATE_FAIL);
828
						} else {
Grigorii Demidov's avatar
Grigorii Demidov committed
829 830
							assert(t->ctx->source.session == session);
							t->ctx->source.session = NULL;
831
						}
832
						array_del(session->waiting, 0);
Grigorii Demidov's avatar
Grigorii Demidov committed
833 834
						session_del_tasks(session, t);
						qr_task_unref(t);
835 836
					}
					while (session->tasks.len > 0) {
Grigorii Demidov's avatar
Grigorii Demidov committed
837
						struct qr_task *t = session->tasks.at[0];
838
						if (session->outgoing) {
Grigorii Demidov's avatar
Grigorii Demidov committed
839
							qr_task_finalize(t, KR_STATE_FAIL);
840
						} else {
Grigorii Demidov's avatar
Grigorii Demidov committed
841 842
							assert(t->ctx->source.session == session);
							t->ctx->source.session = NULL;
843
						}
Grigorii Demidov's avatar
Grigorii Demidov committed
844
						session_del_tasks(session, t);
845 846 847 848
					}
					session_close(session);
					return status;
				}
849
			}
850
		}
Grigorii Demidov's avatar
Grigorii Demidov committed
851
		if (!session->closing) {
Daniel Kahn Gillmor's avatar
Daniel Kahn Gillmor committed
852
			io_start_read(handle); /* Start reading new query */
853 854
		}
	}
855
	return status;
856 857
}

858 859
static void on_send(uv_udp_send_t *req, int status)
{
860 861 862 863
	uv_handle_t *handle = (uv_handle_t *)(req->handle);
	uv_loop_t *loop = handle->loop;
	struct worker_ctx *worker = loop->data;
	assert(worker == get_worker());
864
	struct qr_task *task = req->data;
Grigorii Demidov's avatar
Grigorii Demidov committed
865
	qr_task_on_send(task, handle, status);
866
	qr_task_unref(task);
867
	iorequest_release(worker, req);
868 869
}

870
static void on_task_write(uv_write_t *req, int status)
871
{
872 873 874 875
	uv_handle_t *handle = (uv_handle_t *)(req->handle);
	uv_loop_t *loop = handle->loop;
	struct worker_ctx *worker = loop->data;
	assert(worker == get_worker());
876
	struct qr_task *task = req->data;
Grigorii Demidov's avatar
Grigorii Demidov committed
877
	qr_task_on_send(task, handle, status);
878
	qr_task_unref(task);
879
	iorequest_release(worker, req);
880 881
}

882 883 884 885 886 887 888 889 890
static void on_nontask_write(uv_write_t *req, int status)
{
	uv_handle_t *handle = (uv_handle_t *)(req->handle);
	uv_loop_t *loop = handle->loop;
	struct worker_ctx *worker = loop->data;
	assert(worker == get_worker());
	iorequest_release(worker, req);
}

891 892
ssize_t worker_gnutls_push(gnutls_transport_ptr_t h, const void *buf, size_t len)
{
893
	struct tls_common_ctx *t = (struct tls_common_ctx *)h;
894 895 896
	const uv_buf_t uv_buf[1] = {
		{ (char *)buf, len }
	};
897 898 899 900 901 902

	if (t == NULL) {
		errno = EFAULT;
		return -1;
	}

903 904 905
	assert(t->session && t->session->handle &&
	       t->session->handle->type == UV_TCP);

906
	VERBOSE_MSG(NULL,"[%s] push %zu <%p>\n",
907
		    t->client_side ? "tls_client" : "tls", len, h);
908 909

	struct worker_ctx *worker = t->worker;
910
	assert(worker);
911 912 913 914 915 916 917 918

	void *ioreq = worker_iohandle_borrow(worker);
	if (!ioreq) {
		errno = EFAULT;
		return -1;
	}

	uv_write_t *write_req = (uv_write_t *)ioreq;
919 920 921 922 923 924 925 926 927

	struct qr_task *task = t->task;
	uv_write_cb write_cb = on_task_write;
	if (t->handshake_state == TLS_HS_DONE) {
		assert(task);
	} else {
		task = NULL;
		write_cb = on_nontask_write;
	}
928 929 930 931

	write_req->data = task;

	ssize_t ret = -1;
932
	int res = uv_write(write_req, (uv_stream_t *)t->session->handle, uv_buf, 1, write_cb);
933
	if (res == 0) {
934 935 936
		if (task) {
			qr_task_ref(task); /* Pending ioreq on current task */
		}
937 938 939 940 941 942
		if (worker->too_many_open &&
		    worker->stats.rconcurrent <
			worker->rconcurrent_highwatermark - 10) {
			worker->too_many_open = false;
		}
		ret = len;
943 944 945 946 947 948
		struct sockaddr *addr = &t->session->peer.ip;
		worker->stats.tcp += 1;
		if (addr->sa_family == AF_INET6)
			worker->stats.ipv6 += 1;
		else if (addr->sa_family == AF_INET)
			worker->stats.ipv4 += 1;
949
	} else {
950
		VERBOSE_MSG(NULL,"[%s] uv_write: %s\n",
951
			    t->client_side ? "tls_client" : "tls", uv_strerror(res));
952 953 954 955 956 957
		iorequest_release(worker, ioreq);
		errno = EIO;
	}
	return ret;
}

958 959
static int qr_task_send(struct qr_task *task, uv_handle_t *handle,
			struct sockaddr *addr, knot_pkt_t *pkt)
960
{
961
	if (!handle) {
962
		return qr_task_on_send(task, handle, kr_error(EIO));
963
	}
964 965 966

	/* Synchronous push to TLS context, bypassing event loop. */
	struct session *session = handle->data;
967
	assert(session->closing == false);
968
	if (session->has_tls) {
969
		struct kr_request *req = &task->ctx->req;
970 971 972
		if (session->outgoing) {
			int ret = kr_resolve_checkout(req, NULL, addr,
						      SOCK_STREAM, pkt);
973 974 975 976
			if (ret != kr_ok()) {
				return ret;
			}
		}
977
		return tls_push(task, handle, pkt);
978
	}
979

980
	int ret = 0;
981 982 983
	struct request_ctx *ctx = task->ctx;
	struct worker_ctx *worker = ctx->worker;
	struct kr_request *req = &ctx->req;
984 985
	void *ioreq = iorequest_borrow(worker);
	if (!ioreq) {
986 987
		return qr_task_on_send(task, handle, kr_error(ENOMEM));
	}
988
	if (knot_wire_get_qr(pkt->wire) == 0) {
989 990 991 992 993 994 995 996 997 998 999 1000
		/*
		 * Query must be finalised using destination address before
		 * sending.
		 *
		 * Libuv does not offer a convenient way how to obtain a source
		 * IP address from a UDP handle that has been initialised using
		 * uv_udp_init(). The uv_udp_getsockname() fails because of the
		 * lazy socket initialisation.
		 *
		 * @note -- A solution might be opening a separate socket and
		 * trying to obtain the IP address from it.
		 */
1001
		ret = kr_resolve_checkout(req, NULL, addr,
1002 1003
		                          handle->type == UV_UDP ? SOCK_DGRAM : SOCK_STREAM,
		                          pkt);
1004
		if (ret != 0) {
1005
			iorequest_release(worker, ioreq);
1006
			return ret;
1007
		}
1008 1009 1010
	}
	/* Send using given protocol */
	if (handle->type == UV_UDP) {
1011
		uv_udp_send_t *send_req = (uv_udp_send_t *)ioreq;
1012
		uv_buf_t buf = { (char *)pkt->wire, pkt->size };
1013 1014 1015 1016
		send_req->data = task;
		ret = uv_udp_send(send_req, (uv_udp_t *)handle, &buf, 1, addr, &on_send);
	} else if (handle->type == UV_TCP) {
		uv_write_t *write_req = (uv_write_t *)ioreq;
1017 1018 1019 1020 1021
		uint16_t pkt_size = htons(pkt->size);
		uv_buf_t buf[2] = {
			{ (char *)&pkt_size, sizeof(pkt_size) },
			{ (char *)pkt->wire, pkt->size }
		};
1022
		write_req->data = task;
1023
		ret = uv_write(write_req, (uv_stream_t *)handle, buf, 2, &on_task_write);
1024 1025
	} else {
		assert(false);
1026
	}
1027

1028
	if (ret == 0) {
1029
		qr_task_ref(task); /* Pending ioreq on current task */
1030 1031
		if (worker->too_many_open &&
		    worker->stats.rconcurrent <
1032
			worker->rconcurrent_highwatermark - 10) {
1033 1034
			worker->too_many_open = false;
		}
1035
	} else {
1036
		iorequest_release(worker, ioreq);
1037 1038 1039 1040
		if (ret == UV_EMFILE) {
			worker->too_many_open = true;
			worker->rconcurrent_highwatermark = worker->stats.rconcurrent;
		}
1041
	}
1042

1043
	/* Update statistics */
1044 1045 1046
	if (ctx->source.session &&
	    handle != ctx->source.session->handle &&
	    addr) {
1047
		if (handle->type == UV_UDP)
1048
			worker->stats.udp += 1;
1049
		else
1050
			worker->stats.tcp += 1;
1051
		if (addr->sa_family == AF_INET6)
1052
			worker->stats.ipv6 += 1;
1053
		else if (addr->sa_family == AF_INET)
1054
			worker->stats.ipv4 += 1;
1055
	}
1056
	return ret;
1057 1058
}

1059 1060 1061 1062 1063 1064 1065 1066
static int session_next_waiting_send(struct session *session)
{
	union inaddr *peer = &session->peer;
	int ret = kr_ok();
	if (session->waiting.len > 0) {
		struct qr_task *task = session->waiting.at[0];
		ret = qr_task_send(task, session->handle, &peer->ip, task->pktbuf);
	}
Grigorii Demidov's avatar
Grigorii Demidov committed
1067 1068
	session->timeout.data = session;
	timer_start(session, on_tcp_watchdog_timeout, MAX_TCP_INACTIVITY, 0);
1069 1070 1071 1072 1073
	return ret;
}

static int session_tls_hs_cb(struct session *session, int status)
{
1074 1075 1076 1077
	struct worker_ctx *worker = get_worker();
	union inaddr *peer = &session->peer;
	int deletion_res = worker_del_tcp_waiting(worker, &peer->ip);

1078
	if (status) {
1079
		kr_nsrep_update_rtt(NULL, &peer->ip, KR_NS_DEAD,
1080 1081
				    worker->engine->resolver.cache_rtt,
				    KR_NS_UPDATE_NORESET);
1082 1083
		return kr_ok();
	}
1084

1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102
	int ret = worker_add_tcp_connected(worker, &peer->ip, session);
	if (deletion_res == kr_ok() && ret == kr_ok()) {
		ret = session_next_waiting_send(session);
	} else {
		ret = kr_error(EINVAL);
	}

	if (ret != kr_ok()) {
		/* Something went wrong.
		 * Session isn't in the list of waiting sessions,
		 * or addition to the list of connected sessions failed,
		 * or write to upstream failed. */
		while (session->waiting.len > 0) {
			struct qr_task *task = session->waiting.at[0];
			session_del_tasks(session, task);
			array_del(session->waiting, 0);
			qr_task_finalize(task, KR_STATE_FAIL);
			qr_task_unref(task);
1103
		}
1104 1105 1106
		worker_del_tcp_connected(worker, &peer->ip);
		assert(session->tasks.len == 0);
		session_close(session);
1107 1108 1109 1110
	}
	return kr_ok();
}

1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124
static struct kr_query *session_current_query(struct session *session)
{
	if (session->waiting.len == 0) {
		return NULL;
	}

	struct qr_task *task = session->waiting.at[0];
	if (task->ctx->req.rplan.pending.len == 0) {
		return NULL;
	}

	return array_tail(task->ctx->req.rplan.pending);
}

1125
static void on_connect(uv_connect_t *req, int status)
1126
{
1127
	struct worker_ctx *worker = get_worker();
1128
	uv_stream_t *handle = req->handle;
1129 1130 1131
	struct session *session = handle->data;

	union inaddr *peer = &session->peer;
1132
	uv_timer_stop(&session->timeout);
1133 1134 1135 1136

	if (status == UV_ECANCELED) {
		worker_del_tcp_waiting(worker, &peer->ip);
		assert(session->closing && session->waiting.len == 0 && session->tasks.len == 0);
1137
		iorequest_release(worker, req);
1138 1139 1140 1141 1142 1143
		return;
	}

	if (session->closing) {
		worker_del_tcp_waiting(worker, &peer->ip);
		assert(session->waiting.len == 0 && session->tasks.len == 0);
1144
		iorequest_release(worker, req);
1145 1146 1147 1148
		return;
	}

	if (status != 0) {
1149
		worker_del_tcp_waiting(worker, &peer->ip);
1150 1151 1152 1153
		while (session->waiting.len > 0) {
			struct qr_task *task = session->waiting.at[0];
			session_del_tasks(session, task);
			array_del(session->waiting, 0);
1154
			assert(task->refs > 1);
1155
			qr_task_unref(task);
1156
			qr_task_step(task, NULL, NULL);
1157 1158
		}
		assert(session->tasks.len == 0);
1159
		iorequest_release(worker, req);
1160 1161 1162 1163
		session_close(session);
		return;
	}

1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183
	if (!session->has_tls) {
		/* if there is a TLS, session still waiting for handshake,
		 * otherwise remove it from waiting list */
		if (worker_del_tcp_waiting(worker, &peer->ip) != 0) {
			/* session isn't in list of waiting queries, *
			 * something gone wrong */
			while (session->waiting.len > 0) {
				struct qr_task *task = session->waiting.at[0];
				session_del_tasks(session, task);
				array_del(session->waiting, 0);
				qr_task_finalize(task, KR_STATE_FAIL);
				qr_task_unref(task);
			}
			assert(session->tasks.len == 0);
			iorequest_release(worker, req);
			session_close(session);
			return;
		}
	}

1184 1185
	struct kr_query *qry = session_current_query(session);
	WITH_VERBOSE (qry) {
1186 1187 1188
		char addr_str[INET6_ADDRSTRLEN];
		inet_ntop(session->peer.ip.sa_family, kr_inaddr(&session->peer.ip),
			  addr_str, sizeof(addr_str));
1189
		VERBOSE_MSG(qry, "=> connected to '%s'\n", addr_str);
1190 1191
	}

1192 1193 1194 1195 1196 1197 1198 1199
	session->connected = true;
	session->handle = (uv_handle_t *)handle;

	int ret = kr_ok();
	if (session->has_tls) {
		ret = tls_client_connect_start(session->tls_client_ctx,
					       session, session_tls_hs_cb);
		if (ret == kr_error(EAGAIN)) {
1200
			iorequest_release(worker, req);
1201
			io_start_read(session->handle);
1202
			timer_start(session, on_tcp_watchdog_timeout, MAX_TCP_INACTIVITY, 0);
1203 1204 1205 1206 1207 1208 1209 1210
			return;
		}
	}

	if (ret == kr_ok()) {
		ret = session_next_waiting_send(session);
		if (ret == kr_ok()) {
			worker_add_tcp_connected(worker, &session->peer.ip, session);
1211
			iorequest_release(worker, req);
1212 1213
			return;
		}
1214
	}
1215

1216 1217 1218 1219 1220 1221
	while (session->waiting.len > 0) {
		struct qr_task *task = session->waiting.at[0];
		session_del_tasks(session, task);
		array_del(session->waiting, 0);
		qr_task_finalize(task, KR_STATE_FAIL);
		qr_task_unref(task);
1222
	}
1223 1224 1225

	assert(session->tasks.len == 0);

1226
	iorequest_release(worker, req);
1227
	session_close(session);
1228 1229
}

1230
static void on_tcp_connect_timeout(uv_timer_t *timer)
1231
{
1232 1233 1234
	struct session *session = timer->data;

	uv_timer_stop(timer);
1235
	struct worker_ctx *worker = get_worker();
1236 1237 1238

	assert (session->waiting.len == session->tasks.len);

1239 1240 1241
	union inaddr *peer = &session->peer;
	worker_del_tcp_waiting(worker, &peer->ip);

1242 1243
	struct kr_query *qry = session_current_query(session);
	WITH_VERBOSE (qry) {
1244 1245
		char addr_str[INET6_ADDRSTRLEN];
		inet_ntop(peer->ip.sa_family, kr_inaddr(&peer->ip), addr_str, sizeof(addr_str));
1246
		VERBOSE_MSG(qry, "=> connection to '%s' failed\n", addr_str);
1247
	}
1248

1249
	kr_nsrep_update_rtt(NULL, &peer->ip, KR_NS_DEAD,
1250 1251
			    worker->engine->resolver.cache_rtt,
			    KR_NS_UPDATE_NORESET);
1252

1253 1254 1255
	while (session->waiting.len > 0) {
		struct qr_task *task = session->waiting.at[0];
		struct request_ctx *ctx = task->ctx;
1256
		assert(ctx);
1257 1258 1259 1260
		task->timeouts += 1;
		worker->stats.timeout += 1;
		session_del_tasks(session, task);
		array_del(session->waiting, 0);
1261
		assert(task->refs > 1);
1262
		qr_task_unref(task);
1263
		qr_task_step(task, NULL, NULL);
1264 1265 1266 1267
	}

	assert (session->tasks.len == 0);
	session_close(session);
1268 1269
}

1270
static void on_tcp_watchdog_timeout(uv_timer_t *timer)
1271
{
1272 1273 1274 1275 1276 1277
	struct session *session = timer->data;

	assert(session->outgoing);
	uv_timer_stop(timer);
	struct worker_ctx *worker = get_worker();

Grigorii Demidov's avatar
Grigorii Demidov committed
1278
	if (session->outgoing) {
1279 1280 1281
		if (session->has_tls) {
			worker_del_tcp_waiting(worker, &session->peer.ip);
		}
Grigorii Demidov's avatar
Grigorii Demidov committed
1282
		worker_del_tcp_connected(worker, &session->peer.ip);
1283

Grigorii Demidov's avatar
Grigorii Demidov committed
1284 1285 1286 1287 1288 1289 1290 1291 1292
		while (session->waiting.len > 0) {
			struct qr_task *task = session->waiting.at[0];
			task->timeouts += 1;
			worker->stats.timeout += 1;
			array_del(session->waiting, 0);
			session_del_tasks(session, task);
			qr_task_finalize(task, KR_STATE_FAIL);
			qr_task_unref(task);
		}
1293 1294 1295 1296 1297 1298 1299 1300 1301
	}

	while (session->tasks.len > 0) {
		struct qr_task *task = session->tasks.at[0];
		task->timeouts += 1;
		worker->stats.timeout += 1;
		assert(task->refs > 1);
		array_del(session->tasks, 0);
		qr_task_finalize(task, KR_STATE_FAIL);
Grigorii Demidov's avatar
Grigorii Demidov committed