Reworked zone data handlers, improved logging, fixed bugs and stability issues

parent 53550b6b
This diff is collapsed.
import os
import yaml
from colorlog import warning as warn, info
from yaml.parser import ParserError
from colorlog import error, warning as warn, info
CONFIG_GLOBAL = {
"TIMEZONE": "GMT",
"LOGFILE": "-",
"PIDFILE": "/tmp/jetconf.pid"
"PIDFILE": "/tmp/jetconf.pid",
"PERSISTENT_CHANGES": True,
"LOG_LEVEL": "info",
"LOG_DBG_MODULES": ["*"]
}
CONFIG_HTTP = {
......@@ -59,6 +64,9 @@ def load_config(filename: str):
except FileNotFoundError:
warn("Configuration file does not exist")
except ParserError as e:
error("Configuration syntax error: " + str(e))
exit()
# Shortcuts
NACM_ADMINS = CONFIG["NACM"]["ALLOWED_USERS"]
......
......@@ -2,6 +2,9 @@ GLOBAL:
TIMEZONE: "Europe/Prague"
LOGFILE: "-"
PIDFILE: "/tmp/jetconf.pid"
PERSISTENT_CHANGES: true
LOG_LEVEL: "debug"
LOG_DBG_MODULES: ["usr_conf_data_handlers", "knot_api"]
HTTP_SERVER:
DOC_ROOT: "jetconf/doc-root"
......
import json
from threading import Lock
from enum import Enum
from colorlog import error, warning as warn, info, debug
from colorlog import error, warning as warn, info
from typing import List, Any, Dict, Callable
from yangson.schema import SchemaNode, NonexistentSchemaNode, ListNode, LeafListNode
from yangson.schema import SchemaNode, NonexistentSchemaNode, ListNode, LeafListNode, SchemaError, SemanticError
from yangson.datamodel import DataModel
from yangson.enumerations import ContentType
from yangson.instance import (
InstanceNode,
NonexistentInstance,
InstanceTypeError,
InstanceValueError,
ArrayValue,
ObjectValue,
MemberName,
......@@ -20,7 +21,11 @@ from yangson.instance import (
InstanceRoute
)
from .helpers import PathFormat
from .helpers import PathFormat, ErrorHelpers, LogHelpers
from .config import CONFIG
epretty = ErrorHelpers.epretty
debug_data = LogHelpers.create_module_dbg_logger(__name__)
class ChangeType(Enum):
......@@ -157,25 +162,34 @@ class UsrChangeJournal:
return chl_json
def commit(self, ds: "BaseDatastore"):
# ds.lock_data()
# Set new data root
if hash(ds.get_data_root()) == hash(self.root_origin):
info("Commiting new configuration (swapping roots)")
# Set new root
nr = self.get_root_head()
else:
info("Commiting new configuration (re-applying changes)")
nr = ds.get_data_root()
for cl in self.clists:
for change in cl.journal:
if change.change_type == ChangeType.CREATE:
nr = ds.create_node_rpc(nr, change.rpc_info, change.data)
elif change.change_type == ChangeType.REPLACE:
nr = ds.update_node_rpc(nr, change.rpc_info, change.data)
elif change.change_type == ChangeType.DELETE:
nr = ds.delete_node_rpc(nr, change.rpc_info)
try:
nr.validate(ContentType.config)
new_data_valid = True
except (SchemaError, SemanticError) as e:
error("Data validation error:")
error(epretty(e))
new_data_valid = False
if new_data_valid:
# Set new data root
if hash(ds.get_data_root()) == hash(self.root_origin):
info("Commiting new configuration (swapping roots)")
# Set new root
ds.set_data_root(self.get_root_head())
else:
info("Commiting new configuration (re-applying changes)")
nr = ds.get_data_root()
for cl in self.clists:
for change in cl.journal:
if change.change_type == ChangeType.CREATE:
nr = ds.create_node_rpc(nr, change.rpc_info, change.data)
elif change.change_type == ChangeType.REPLACE:
nr = ds.update_node_rpc(nr, change.rpc_info, change.data)
elif change.change_type == ChangeType.DELETE:
nr = ds.delete_node_rpc(nr, change.rpc_info)
ds.set_data_root(nr)
ds.set_data_root(nr)
# Notify schema node observers
for cl in self.clists:
......@@ -189,9 +203,6 @@ class UsrChangeJournal:
# Clear user changelists
self.clists.clear()
finally:
# ds.unlock_data()
pass
class BaseDatastore:
......@@ -300,7 +311,7 @@ class BaseDatastore:
sdh = STATE_DATA_HANDLES.get_handler(state_node_pth)
if sdh is not None:
root_val = sdh.update_node(ii, root, True)
root = self._data.update_from_raw(root_val)
root = self._data.update(root_val, raw=True)
else:
raise NoHandlerForStateDataError()
self.commit_end_callback()
......@@ -402,7 +413,7 @@ class BaseDatastore:
n = root.goto(ii)
sn = n.schema_node
sch_member_name = sn.iname2qname(input_member_name)
sch_member_name = sn._iname2qname(input_member_name)
member_sn = sn.get_data_child(*sch_member_name)
if isinstance(member_sn, ListNode):
......@@ -476,7 +487,7 @@ class BaseDatastore:
if nrpc.check_data_node_path(root, ii, Permission.NACM_ACCESS_UPDATE) == Action.DENY:
raise NacmForbiddenError()
new_n = n.update_from_raw(value)
new_n = n.update(value, raw=True)
return new_n.top()
......@@ -502,7 +513,7 @@ class BaseDatastore:
if isinstance(last_isel, MemberName):
new_n = n_parent.delete_member(last_isel.name)
else:
raise InstanceTypeError(n, "Invalid target node type")
raise InstanceValueError(n, "Invalid target node type")
return new_n.top()
......@@ -550,7 +561,16 @@ class BaseDatastore:
if usr_journal is not None:
if self.commit_begin_callback is not None:
self.commit_begin_callback()
usr_journal.commit(self)
try:
self.lock_data(rpc.username)
old_root = self._data
usr_journal.commit(self)
if CONFIG["GLOBAL"]["PERSISTENT_CHANGES"] is True:
self.save()
finally:
self.unlock_data()
if self.commit_end_callback is not None:
self.commit_end_callback()
del self._usr_journals[rpc.username]
......@@ -591,7 +611,7 @@ class BaseDatastore:
ret = self._data_lock.acquire(blocking=blocking, timeout=1)
if ret:
self._lock_username = username or "(unknown)"
debug("Acquired lock in datastore \"{}\" for user \"{}\"".format(self.name, username))
debug_data("Acquired lock in datastore \"{}\" for user \"{}\"".format(self.name, username))
else:
raise DataLockError(
"Failed to acquire lock in datastore \"{}\" for user \"{}\", already locked by \"{}\"".format(
......@@ -604,22 +624,26 @@ class BaseDatastore:
# Unlock datastore data
def unlock_data(self):
self._data_lock.release()
debug("Released lock in datastore \"{}\" for user \"{}\"".format(self.name, self._lock_username))
debug_data("Released lock in datastore \"{}\" for user \"{}\"".format(self.name, self._lock_username))
self._lock_username = None
# Load data from persistent storage
def load(self, filename: str):
def load(self):
raise NotImplementedError("Not implemented in base class")
# Save data to persistent storage
def save(self, filename: str):
def save(self):
raise NotImplementedError("Not implemented in base class")
class JsonDatastore(BaseDatastore):
def load(self, filename: str):
def __init__(self, dm: DataModel, json_file: str, name: str = ""):
super().__init__(dm, name)
self.json_file = json_file
def load(self):
self._data = None
with open(filename, "rt") as fp:
with open(self.json_file, "rt") as fp:
self._data = self._dm.from_raw(json.load(fp))
def load_yl_data(self, filename: str):
......@@ -627,11 +651,9 @@ class JsonDatastore(BaseDatastore):
with open(filename, "rt") as fp:
self._yang_lib_data = self._dm.from_raw(json.load(fp))
def save(self, filename: str):
with open(filename, "w") as jfd:
self.lock_data("json_save")
json.dump(self._data, jfd)
self.unlock_data()
def save(self):
with open(self.json_file, "w") as jfd:
json.dump(self._data.raw_value(), jfd, indent=4)
def test():
......
......@@ -308,7 +308,7 @@
},
{
"name": "permit-zone-access",
"path": "/dns-server:dns-server/zones/zone",
"path": "/dns-server:dns-server/zones",
"access-operations": "*",
"comment": "Users can write other zones.",
"action": "permit"
......
import logging
from colorlog import debug, getLogger
from enum import Enum
from typing import Dict, Any
from datetime import datetime
......@@ -5,6 +8,8 @@ from pytz import timezone
from yangson.instance import InstanceRoute, MemberName, EntryKeys, InstanceIdParser, ResourceIdParser
from yangson.datamodel import DataModel
from .config import CONFIG
class PathFormat(Enum):
URL = 0
......@@ -77,3 +82,18 @@ class ErrorHelpers:
return "In module " + module_name + ": " + err_str
else:
return err_str
class LogHelpers:
@staticmethod
def create_module_dbg_logger(module_name: str):
module_name_simple = module_name.split(".")[-1]
def module_dbg_logger(msg: str):
if ({module_name_simple, "*"} & set(CONFIG["GLOBAL"]["LOG_DBG_MODULES"])) and (CONFIG["GLOBAL"]["LOG_LEVEL"] == "debug"):
logger = getLogger()
logger.setLevel(logging.DEBUG)
debug(module_name_simple + ": " + msg)
logger.setLevel(logging.INFO)
return module_dbg_logger
......@@ -2,17 +2,17 @@ import json
import os
import mimetypes
from collections import OrderedDict
from colorlog import error, warning as warn, info, debug
from colorlog import error, warning as warn, info
from urllib.parse import parse_qs
from typing import Dict, List
from yangson.schema import NonexistentSchemaNode
from yangson.instance import NonexistentInstance, InstanceTypeError, DuplicateMember
from yangson.instance import NonexistentInstance, InstanceValueError
from yangson.datatype import YangTypeError
from jetconf.knot_api import KnotError
from .config import CONFIG_GLOBAL, CONFIG_HTTP, NACM_ADMINS, API_ROOT_data, API_ROOT_STAGING_data, API_ROOT_ops
from .helpers import CertHelpers, DataHelpers, DateTimeHelpers, ErrorHelpers
from .helpers import CertHelpers, DataHelpers, DateTimeHelpers, ErrorHelpers, LogHelpers
from .data import (
BaseDatastore,
RpcInfo,
......@@ -26,6 +26,7 @@ from .data import (
QueryStrT = Dict[str, List[str]]
epretty = ErrorHelpers.epretty
debug_httph = LogHelpers.create_module_dbg_logger(__name__)
def unknown_req_handler(prot: "H2Protocol", stream_id: int, headers: OrderedDict, data: bytes=None):
......@@ -91,7 +92,6 @@ def _get(prot: "H2Protocol", stream_id: int, ds: BaseDatastore, pth: str, yl_dat
response_headers.append(("content-length", len(response_bytes)))
prot.conn.send_headers(stream_id, response_headers)
# prot.conn.send_data(stream_id, response_bytes, end_stream=True)
def split_arr(arr, chunk_size):
for i in range(0, len(arr), chunk_size):
......@@ -111,7 +111,7 @@ def _get(prot: "H2Protocol", stream_id: int, ds: BaseDatastore, pth: str, yl_dat
except NonexistentInstance as e:
warn(epretty(e))
prot.send_empty(stream_id, "404", "Not Found")
except InstanceTypeError as e:
except InstanceValueError as e:
warn(epretty(e))
prot.send_empty(stream_id, "400", "Bad Request")
except KnotError as e:
......@@ -242,7 +242,7 @@ def _get_staging(prot: "H2Protocol", stream_id: int, ds: BaseDatastore, pth: str
except NonexistentInstance as e:
warn(epretty(e))
prot.send_empty(stream_id, "404", "Not Found")
except InstanceTypeError as e:
except InstanceValueError as e:
warn(epretty(e))
prot.send_empty(stream_id, "400", "Bad Request")
except NoHandlerError as e:
......@@ -278,7 +278,7 @@ def create_get_staging_api(ds: BaseDatastore):
def _post(prot: "H2Protocol", data: bytes, stream_id: int, ds: BaseDatastore, pth: str):
data_str = data.decode("utf-8")
debug("HTTP data received: " + data_str)
debug_httph("HTTP data received: " + data_str)
url_split = pth.split("?")
url_path = url_split[0]
......@@ -319,12 +319,12 @@ def _post(prot: "H2Protocol", data: bytes, stream_id: int, ds: BaseDatastore, pt
except NonexistentInstance as e:
warn(epretty(e))
prot.send_empty(stream_id, "404", "Not Found")
except DuplicateMember as e:
warn(epretty(e))
prot.send_empty(stream_id, "409", "Conflict")
except (InstanceTypeError, YangTypeError, InstanceAlreadyPresent, NoHandlerError, ValueError) as e:
except (InstanceValueError, YangTypeError, NoHandlerError, ValueError) as e:
warn(epretty(e))
prot.send_empty(stream_id, "400", "Bad Request")
except InstanceAlreadyPresent as e:
warn(epretty(e))
prot.send_empty(stream_id, "409", "Conflict")
finally:
ds.unlock_data()
......@@ -352,7 +352,7 @@ def create_post_api(ds: BaseDatastore):
def _put(prot: "H2Protocol", data: bytes, stream_id: int, ds: BaseDatastore, pth: str):
data_str = data.decode("utf-8")
debug("HTTP data received: " + data_str)
debug_httph("HTTP data received: " + data_str)
url_split = pth.split("?")
url_path = url_split[0]
......
from enum import Enum
from typing import List, Union, Dict, Any
from threading import Lock
from colorlog import debug
from colorlog import info
from .libknot.control import KnotCtl, KnotCtlType
from .config import CONFIG
from .helpers import LogHelpers
KNOT = None # type: KnotConfig
JsonNodeT = Union[Dict[str, Any], List]
debug_knot = LogHelpers.create_module_dbg_logger(__name__)
class KnotError(Exception):
......@@ -113,32 +115,42 @@ class KnotConfig(KnotCtl):
def begin(self):
if self.conf_state == KnotConfState.NONE:
self.send_block("conf-begin")
self.receive_block()
# print(">>> CONF BEGIN")
self.conf_state = KnotConfState.CONF
try:
self.receive_block()
# print(">>> CONF BEGIN")
self.conf_state = KnotConfState.CONF
except Exception as e:
raise KnotInternalError(str(e))
def begin_zone(self):
if self.conf_state == KnotConfState.NONE:
self.send_block("zone-begin")
self.receive_block()
# print(">>> ZONE BEGIN")
self.conf_state = KnotConfState.ZONE
try:
self.receive_block()
# print(">>> ZONE BEGIN")
self.conf_state = KnotConfState.ZONE
except Exception as e:
raise KnotInternalError(str(e))
def commit(self):
if self.conf_state == KnotConfState.CONF:
self.send_block("conf-commit")
self.receive_block()
# print(">>> CONF COMMIT")
self.conf_state = KnotConfState.NONE
try:
self.receive_block()
self.conf_state = KnotConfState.NONE
except Exception as e:
raise KnotInternalError(str(e))
else:
raise KnotApiStateError()
def commit_zone(self):
if self.conf_state == KnotConfState.ZONE:
self.send_block("zone-commit")
self.receive_block()
# print(">>> ZONE COMMIT")
self.conf_state = KnotConfState.NONE
try:
self.receive_block()
self.conf_state = KnotConfState.NONE
except Exception as e:
raise KnotInternalError(str(e))
else:
raise KnotApiStateError()
......@@ -195,6 +207,17 @@ class KnotConfig(KnotCtl):
raise KnotInternalError(str(e))
return resp
def zone_remove(self, domain_name: str) -> JsonNodeT:
if not self.connected:
raise KnotApiError("Knot socket is closed")
try:
self.send_block("conf-unset", section="zone", item="domain", zone=domain_name)
resp = self.receive_block()
except Exception as e:
raise KnotInternalError(str(e))
return resp
def zone_add_record(self, domain_name: str, rr: RRecordBase) -> JsonNodeT:
if not self.connected:
raise KnotApiError("Knot socket is closed")
......@@ -202,7 +225,7 @@ class KnotConfig(KnotCtl):
try:
res_data = rr.rrdata_format()
self.set_zone_item(zone=domain_name, owner=rr.owner, ttl=str(rr.ttl), rtype=rr.type, data=res_data)
debug("Inserting zone \"{}\" RR, type=\"{}\", owner=\"{}\", ttl=\"{}\", data=\"{}\"".format(
debug_knot("Inserting zone \"{}\" RR, type=\"{}\", owner=\"{}\", ttl=\"{}\", data=\"{}\"".format(
domain_name, rr.type, rr.owner, rr.ttl, res_data
))
resp = self.receive_block()
......@@ -210,6 +233,27 @@ class KnotConfig(KnotCtl):
raise KnotInternalError(str(e))
return resp
def zone_del_record(self, domain_name: str, owner: str, rr_type: str) -> JsonNodeT:
if not self.connected:
raise KnotApiError("Knot socket is closed")
try:
self.set_zone_item(zone=domain_name, owner=owner, rtype=rr_type, data=None)
resp = self.receive_block()
except Exception as e:
raise KnotInternalError(str(e))
return resp
def knot_connect():
debug_knot("Connecting to KNOT socket")
KNOT.knot_connect()
def knot_disconnect():
debug_knot("Disonnecting from KNOT socket")
KNOT.knot_disconnect()
def knot_api_init():
global KNOT
......
......@@ -2,7 +2,7 @@ import collections
import copy
from threading import Lock
from enum import Enum
from colorlog import error, warning as warn, info, debug
from colorlog import error, warning as warn, info
from typing import List, Set
from yangson.instance import (
......@@ -18,9 +18,10 @@ from yangson.instance import (
EntryKeys
)
from .helpers import DataHelpers, PathFormat, ErrorHelpers
from .helpers import DataHelpers, ErrorHelpers, LogHelpers
epretty = ErrorHelpers.epretty
debug_nacm = LogHelpers.create_module_dbg_logger(__name__)
class Action(Enum):
......@@ -377,10 +378,10 @@ class UserNacm:
mii = copy.copy(ii)
mii.append(nsel)
debug("checking mii {}".format(mii))
debug_nacm("checking mii {}".format(mii))
if self.check_data_node_path(root, mii, Permission.NACM_ACCESS_READ) == Action.DENY:
# info("Pruning node {} {}".format(id(node.value[child_key]), node.value[child_key]))
debug("Pruning node {}".format(mii))
debug_nacm("Pruning node {}".format(mii))
node = node.delete_member(child_key, validate=False)
else:
node = self._check_data_read_path(m, root, mii).up()
......@@ -394,9 +395,9 @@ class UserNacm:
eii = copy.copy(ii)
eii.append(nsel)
debug("checking eii {}".format(eii))
debug_nacm("checking eii {}".format(eii))
if self.check_data_node_path(root, eii, Permission.NACM_ACCESS_READ) == Action.DENY:
debug("Pruning node {} {}".format(id(node.value[i]), node.value[i]))
debug_nacm("Pruning node {} {}".format(id(node.value[i]), node.value[i]))
node = node.delete_entry(i)
arr_len -= 1
else:
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment