Fixed bug in REST server that prevented sending large data, added...

Fixed bug in REST server that prevented sending large data, added schema-digest operation, included utility for reading Knot zone data
parent 28a6561d
......@@ -611,7 +611,10 @@ class BaseDatastore:
"status": "OK",
"conf-changed": True
}
elif rpc.op_name == "schema-digest":
ret_data = self._dm.schema_digest()
else:
# User-defined operation
op_handler = OP_HANDLERS.get_handler(rpc.op_name)
if op_handler is None:
raise NoHandlerForOpError(rpc.op_name)
......
......@@ -443,7 +443,10 @@ def create_api_op(ds: BaseDatastore):
if ret_data is None:
http_resp = HttpResponse.empty(HttpStatus.NoContent, status_in_body=False)
else:
response = json.dumps(ret_data, indent=4)
if not isinstance(ret_data, str):
response = json.dumps(ret_data, indent=4)
else:
response = ret_data
http_resp = HttpResponse(HttpStatus.Ok, response.encode(), CT_YANG_JSON)
except NacmForbiddenError as e:
warn(epretty(e))
......
......@@ -2,22 +2,25 @@ import asyncio
import ssl
from io import BytesIO
from collections import OrderedDict
from colorlog import error, warning as warn, info
from typing import List, Tuple, Dict, Any, Callable, Optional
from h2.connection import H2Connection
from h2.errors import PROTOCOL_ERROR, ENHANCE_YOUR_CALM
from h2.events import DataReceived, RequestReceived, RemoteSettingsChanged, StreamEnded
from h2.exceptions import ProtocolError
from h2.events import DataReceived, RequestReceived, RemoteSettingsChanged, StreamEnded, WindowUpdated
from . import http_handlers as handlers
from .http_handlers import HttpResponse, HttpStatus
from .config import CONFIG_HTTP, API_ROOT_data, API_ROOT_STAGING_data, API_ROOT_ops
from .data import BaseDatastore
from .helpers import SSLCertT
from .helpers import SSLCertT, LogHelpers
HandlerConditionT = Callable[[str, str], bool] # Function(method, path) -> bool
HttpHandlerT = Callable[[OrderedDict, Optional[str], SSLCertT], handlers.HttpResponse]
debug_srv = LogHelpers.create_module_dbg_logger(__name__)
h2_handlers = None # type: HttpHandlerList
......@@ -28,6 +31,11 @@ class RequestData:
self.data = data
self.data_overflow = False
class ResponseData:
def __init__(self, data: bytes):
self.data = data
self.bytes_sent = 0
class HttpHandlerList:
def __init__(self):
......@@ -53,6 +61,7 @@ class H2Protocol(asyncio.Protocol):
self.conn = H2Connection(client_side=False)
self.transport = None
self.stream_data = {} # type: Dict[int, RequestData]
self.resp_stream_data = {} # type: Dict[int, ResponseData]
self.client_cert = None # type: SSLCertT
def connection_made(self, transport: asyncio.Transport):
......@@ -103,11 +112,18 @@ class H2Protocol(asyncio.Protocol):
else:
warn("Unknown http method \"{}\"".format(headers[":method"]))
self.send_response(HttpResponse.empty(HttpStatus.MethodNotAllowed), event.stream_id)
# elif isinstance(event, RemoteSettingsChanged):
# changed_settings = {}
# for s in event.changed_settings.items():
# changed_settings[s[0]] = s[1].new_value
# self.conn.update_settings(changed_settings)
elif isinstance(event, RemoteSettingsChanged):
changed_settings = {}
for s in event.changed_settings.items():
changed_settings[s[0]] = s[1].new_value
self.conn.update_settings(changed_settings)
elif isinstance(event, WindowUpdated):
try:
debug_srv("str {} nw={}".format(event.stream_id, self.conn.local_flow_control_window(event.stream_id)))
self.send_response_continue(event.stream_id)
except (ProtocolError, KeyError) as e:
# debug_srv("wupdexception strid={}: {}".format(event.stream_id, str(e)))
pass
# else:
# print(type(event))
......@@ -127,12 +143,15 @@ class H2Protocol(asyncio.Protocol):
resp = h(headers, data, self.client_cert)
self.send_response(resp, stream_id)
def max_chunk_size(self, stream_id: int):
return min(self.conn.max_outbound_frame_size, self.conn.local_flow_control_window(stream_id))
def send_response(self, resp: HttpResponse, stream_id: int):
resp_headers = (
(':status', resp.status_code),
('content-type', resp.content_type),
('content-length', len(resp.data)),
('server', CONFIG_HTTP["SERVER_NAME"]),
(":status", resp.status_code),
("content-type", resp.content_type),
("content-length", str(len(resp.data))),
("server", CONFIG_HTTP["SERVER_NAME"]),
)
if resp.extra_headers:
......@@ -143,16 +162,38 @@ class H2Protocol(asyncio.Protocol):
self.conn.send_headers(stream_id, resp_headers)
# Do this for optimization
if len(resp.data) > self.conn.max_outbound_frame_size:
def split_arr(arr, chunk_size):
for i in range(0, len(arr), chunk_size):
yield arr[i:i + chunk_size]
for data_chunk in split_arr(resp.data, self.conn.max_outbound_frame_size):
self.conn.send_data(stream_id, data_chunk, end_stream=False)
self.conn.send_data(stream_id, bytes(), end_stream=True)
else:
if len(resp.data) <= self.max_chunk_size(stream_id):
self.conn.send_data(stream_id, resp.data, end_stream=True)
else:
self.resp_stream_data[stream_id] = ResponseData(resp.data)
self.send_response_continue(stream_id)
def send_response_continue(self, stream_id: int):
resp_data = self.resp_stream_data[stream_id]
debug_srv("Continuing...")
while self.max_chunk_size(stream_id) != 0:
if resp_data.bytes_sent >= len(resp_data.data):
self.send_response_end(stream_id)
return
# Get available window
chunk_size = self.max_chunk_size(stream_id)
data_chunk = resp_data.data[resp_data.bytes_sent:resp_data.bytes_sent + chunk_size]
resp_data.bytes_sent += chunk_size
debug_srv("len = {}, max = {}, sent={}, dlen={}, strid={}".format(
len(data_chunk),
chunk_size,
resp_data.bytes_sent,
len(resp_data.data),
stream_id
))
self.conn.send_data(stream_id, data_chunk, end_stream=False)
def send_response_end(self, stream_id: int):
debug_srv("Ending stream {}...".format(stream_id))
self.conn.send_data(stream_id, bytes(), end_stream=True)
del self.resp_stream_data[stream_id]
class RestServer:
......
......@@ -28,4 +28,5 @@ Now you should have the following files:
output_filename.pem - the client certificate
output_filename.key - the client private key
output_filename_curl.pem - the combination of previous 2 files containing both
certificate and key
certificate and key. Some utilities like CURL expect the client certificate
to be in this combined form.
# This script reads zone data from KnotDNS socket and converts them
# to YANG model compliant data tree in JSON formatting.
# Only SOA, A, AAAA, NS, MX, TXT, TLSA and CNAME records are currently
# supported.
import sys
import json
from typing import Dict, Any
from libknot.control import KnotCtl, KnotCtlType
# Edit this to match your actual KnotDNS control socket
KNOT_SOCKET = "/home/pspirek/knot-conf/knot.sock"
def main(args):
if len(args) != 2:
print("Usage: {} [domain]".format(args[0]))
exit(1)
domain = args[1] # type: str
if domain[-1] != '.':
domain += "."
ctl = KnotCtl()
ctl.connect(KNOT_SOCKET)
ctl.send_block("zone-read", zone=domain)
resp = ctl.receive_block() # type: Dict[str, Any]
ctl.send(KnotCtlType.END)
ctl.close()
resp = resp[domain]
zone_template = {
"dns-zones:zone-data": {
"zone": [
{
"name": domain,
"class": "IN",
"default-ttl": 3600,
"SOA": {},
"rrset": []
}
]
}
}
zone_out = zone_template["dns-zones:zone-data"]["zone"][0]
soa_out = zone_out["SOA"]
soa = resp[domain]["SOA"]
soa_data = soa["data"][0].split()
try:
soa_out["ttl"] = int(soa["ttl"])
soa_out["mname"] = soa_data[0]
soa_out["rname"] = soa_data[1]
soa_out["serial"] = int(soa_data[2])
soa_out["refresh"] = int(soa_data[3])
soa_out["retry"] = int(soa_data[4])
soa_out["expire"] = int(soa_data[5])
soa_out["minimum"] = int(soa_data[6])
except (IndexError, ValueError) as e:
print(str(e))
rrset_out = zone_out["rrset"]
for owner, rrs in resp.items():
# print("rrs={}".format(rrs))
for rr_type, rr in rrs.items():
# print("rr={}".format(rr))
if rr_type not in ("A", "AAAA", "NS", "MX", "TXT", "TLSA", "CNAME"):
continue
ttl = int(rr["ttl"])
rr_data_list = rr["data"]
new_rr_out_rdata_list = []
new_rr_out = {
"owner": owner,
"type": "iana-dns-parameters:" + rr_type,
"ttl": ttl,
"rdata": new_rr_out_rdata_list
}
id_int = 0
for rr_data in rr_data_list:
new_rr_out_rdata_values = {}
new_rr_out_rdata = {
"id": str(id_int),
rr_type: new_rr_out_rdata_values
}
if rr_type in ("A", "AAAA"):
new_rr_out_rdata_values["address"] = rr_data
elif rr_type == "NS":
new_rr_out_rdata_values["nsdname"] = rr_data
elif rr_type == "MX":
rr_data = rr_data.split()
new_rr_out_rdata_values["preference"] = rr_data[0]
new_rr_out_rdata_values["exchange"] = rr_data[1]
elif rr_type == "TXT":
new_rr_out_rdata_values["txt-data"] = rr_data.strip(" \"")
elif rr_type == "TLSA":
cert_usage_enum = {
"0": "PKIX-TA",
"1": "PKIX-EE",
"2": "DANE-TA",
"3": "DANE-EE",
"255": "PrivCert"
}
sel_enum = {
"0": "Cert",
"1": "SPKI",
"255": "PrivSel"
}
match_type_enum = {
"0": "Full",
"1": "SHA2-256",
"2": "SHA2-512",
"255": "PrivMatch"
}
rr_data = rr_data.split()
new_rr_out_rdata_values["certificate-usage"] = cert_usage_enum[rr_data[0]]
new_rr_out_rdata_values["selector"] = sel_enum[rr_data[1]]
new_rr_out_rdata_values["matching-type"] = match_type_enum[rr_data[2]]
new_rr_out_rdata_values["certificate-association-data"] = rr_data[3]
elif rr_type == "CNAME":
new_rr_out_rdata_values["cname"] = rr_data
new_rr_out_rdata_list.append(new_rr_out_rdata)
id_int += 1
rrset_out.append(new_rr_out)
print(json.dumps(zone_template, indent=4, sort_keys=True))
if __name__ == "__main__":
main(sys.argv)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment