Improved error handling in configuration data handlers

parent d89c6b9a
......@@ -14,7 +14,7 @@ from .config import CONFIG, load_config, print_config
from .data import JsonDatastore
from .helpers import DataHelpers
from .handler_list import OP_HANDLERS, STATE_DATA_HANDLES, CONF_DATA_HANDLES
from .knot_api import knot_api_init, knot_connect, knot_disconnect
from .knot_api import knot_global_init, knot_connect, knot_disconnect
from .usr_conf_data_handlers import (
KnotConfServerListener,
KnotConfLogListener,
......@@ -115,7 +115,7 @@ def main():
datamodel = DataHelpers.load_data_model("data/", "data/yang-library-data.json")
# Datastore init
datastore = JsonDatastore(datamodel, "jetconf/example-data.json", "DNS data", with_nacm=True)
datastore = JsonDatastore(datamodel, "jetconf/example-data.json", "DNS data", with_nacm=False)
datastore.load()
datastore.load_yl_data("data/yang-library-data.json")
......@@ -136,7 +136,7 @@ def main():
usr_state_data_handlers.create_zone_state_handlers(STATE_DATA_HANDLES, datamodel)
# Initialize Knot control interface
knot_api_init()
knot_global_init()
datastore.commit_begin_callback = knot_connect
datastore.commit_end_callback = knot_disconnect
......
......@@ -2,9 +2,9 @@ GLOBAL:
TIMEZONE: "Europe/Prague"
LOGFILE: "-"
PIDFILE: "/tmp/jetconf.pid"
PERSISTENT_CHANGES: true
PERSISTENT_CHANGES: false
LOG_LEVEL: "debug"
LOG_DBG_MODULES: ["usr_conf_data_handlers", "knot_api", "nacm"]
LOG_DBG_MODULES: ["usr_conf_data_handlers", "knot_api", "nacm", "data"]
HTTP_SERVER:
DOC_ROOT: "jetconf/doc-root"
......
......@@ -25,8 +25,8 @@ from yangson.instance import (
EntryKeys,
EntryIndex,
InstanceRoute,
ArrayEntry
)
ArrayEntry,
RootNode)
from .helpers import PathFormat, ErrorHelpers, LogHelpers, DataHelpers, JsonNodeT
from .config import CONFIG
......@@ -80,6 +80,10 @@ class NoHandlerError(HandlerError):
pass
class ConfHandlerFailedError(HandlerError):
pass
class NoHandlerForOpError(NoHandlerError):
def __init__(self, op_name: str):
self.op_name = op_name
......@@ -92,19 +96,13 @@ class NoHandlerForStateDataError(NoHandlerError):
pass
class ConfHandlerResult(Enum):
PASS = 0,
OK = 1,
ERROR = 2
class BaseDataListener:
def __init__(self, ds: "BaseDatastore", sch_pth: str):
self.ds = ds
self.schema_path = sch_pth # type: str
self.schema_node = ds.get_schema_node(sch_pth) # type: SchemaNode
def process(self, sn: SchemaNode, ii: InstanceRoute, ch: "DataChange") -> ConfHandlerResult:
def process(self, sn: SchemaNode, ii: InstanceRoute, ch: "DataChange"):
raise NotImplementedError("Not implemented in base class")
def __str__(self):
......@@ -202,33 +200,61 @@ class UsrChangeJournal:
new_data_valid = False
if new_data_valid:
# Call commit begin hook
if ds.commit_begin_callback is not None:
ds.commit_begin_callback(self.transaction_opts)
# Set new data root
ds.set_data_root(nr)
# Call commit begin hook
begin_hook_failed = False
try:
ds.commit_begin_callback(self.transaction_opts)
except Exception as e:
error("Exception occured in commit_begin handler: {}".format(epretty(e)))
begin_hook_failed = True
# Run schema node handlers
for cl in self.clists:
for change in cl.journal:
ii = DataHelpers.parse_ii(change.rpc_info.path, change.rpc_info.path_format)
try:
ds.run_conf_edit_handler(ii, change)
except Exception as e:
ds.data_root_rollback(1, False)
raise e
conf_handler_failed = False
if not begin_hook_failed:
try:
for cl in self.clists:
for change in cl.journal:
ii = DataHelpers.parse_ii(change.rpc_info.path, change.rpc_info.path_format)
ds.run_conf_edit_handler(ii, change)
except Exception as e:
error("Exception occured in edit handler: {}".format(epretty(e)))
conf_handler_failed = True
# Call commit end hook
end_hook_failed = False
end_hook_abort_failed = False
if not (begin_hook_failed or conf_handler_failed):
try:
ds.commit_end_callback(self.transaction_opts, failed=False)
except Exception as e:
error("Exception occured in commit_end handler: {}".format(epretty(e)))
end_hook_failed = True
if begin_hook_failed or conf_handler_failed or end_hook_failed:
try:
# Call commit_end callback again with "failed" argument set to True
ds.commit_end_callback(self.transaction_opts, failed=True)
except Exception as e:
error("Exception occured in commit_end handler (abort): {}".format(epretty(e)))
end_hook_abort_failed = True
# Clear user changelists
self.clists.clear()
# Call commit end hook
if ds.commit_end_callback is not None:
ds.commit_end_callback(self.transaction_opts)
# Return to previous version of data and raise an exception if something went wrong
if begin_hook_failed or conf_handler_failed or end_hook_failed or end_hook_abort_failed:
ds.data_root_rollback(history_steps=1, store_current=False)
raise ConfHandlerFailedError("(see logged)")
class BaseDatastore:
def __init__(self, dm: DataModel, name: str="", with_nacm: bool=False):
def _blankfn(*args, **kwargs):
pass
self.name = name
self.nacm = None # type: NacmConfig
self._data = None # type: InstanceNode
......@@ -238,8 +264,8 @@ class BaseDatastore:
self._data_lock = Lock()
self._lock_username = None # type: str
self._usr_journals = {} # type: Dict[str, UsrChangeJournal]
self.commit_begin_callback = None # type: Callable[..., None]
self.commit_end_callback = None # type: Callable[..., None]
self.commit_begin_callback = _blankfn # type: Callable[..., bool]
self.commit_end_callback = _blankfn # type: Callable[..., bool]
if with_nacm:
self.nacm = NacmConfig(self)
......@@ -283,9 +309,7 @@ class BaseDatastore:
return sn
# Notify data observers about change in datastore
def run_conf_edit_handler(self, ii: InstanceRoute, ch: DataChange) -> Optional[ConfHandlerResult]:
h_res = None
def run_conf_edit_handler(self, ii: InstanceRoute, ch: DataChange):
try:
sch_pth_list = filter(lambda n: isinstance(n, MemberName), ii)
sch_pth = DataHelpers.ii2str(sch_pth_list)
......@@ -294,23 +318,11 @@ class BaseDatastore:
while sn is not None:
h = CONF_DATA_HANDLES.get_handler(str(id(sn)))
if h is not None:
h_res = h.process(sn, ii, ch)
if h_res == ConfHandlerResult.OK:
# Edit successfully handled
break
elif h_res == ConfHandlerResult.ERROR:
# Error occured in handler
warn("Error occured in handler for sch_node \"{}\"".format(sch_pth))
break
else:
# Pass edit to superior handler
pass
h.process(sn, ii, ch)
sn = sn.parent
except NonexistentInstance:
warn("Cannnot notify {}, parent container removed".format(ii))
return h_res
# Get data node, evaluate NACM if required
def get_node_rpc(self, rpc: RpcInfo, yl_data=False, staging=False) -> InstanceNode:
ii = DataHelpers.parse_ii(rpc.path, rpc.path_format)
......@@ -500,7 +512,15 @@ class BaseDatastore:
if nrpc.check_data_node_permission(root, ii, Permission.NACM_ACCESS_UPDATE) == Action.DENY:
raise NacmForbiddenError()
new_n = n.update(value, raw=True)
sn = n.schema_node
new_val = sn.from_raw([value])[0]
new_node = RootNode(new_val, sn, new_val.timestamp)
new_node_prunned = nrpc.prune_data_tree(new_node, self._data, ii, Permission.NACM_ACCESS_UPDATE).raw_value()
print(json.dumps(new_node_prunned, indent=4))
new_n = n.update(new_node_prunned, raw=True)
else:
new_n = n.update(value, raw=True)
return new_n.top()
......
This diff is collapsed.
......@@ -12,7 +12,6 @@ from yangson.schema import NonexistentSchemaNode
from yangson.instance import NonexistentInstance, InstanceValueError
from yangson.datatype import YangTypeError
from .knot_api import KnotError
from .config import CONFIG_GLOBAL, CONFIG_HTTP, NACM_ADMINS, API_ROOT_data, API_ROOT_STAGING_data, API_ROOT_ops
from .helpers import CertHelpers, DataHelpers, DateTimeHelpers, ErrorHelpers, LogHelpers, SSLCertT
from .data import (
......@@ -23,8 +22,8 @@ from .data import (
NoHandlerError,
NoHandlerForOpError,
InstanceAlreadyPresent,
ChangeType
)
ChangeType,
ConfHandlerFailedError)
QueryStrT = Dict[str, List[str]]
epretty = ErrorHelpers.epretty
......@@ -120,7 +119,7 @@ def _get(ds: BaseDatastore, pth: str, username: str, yl_data: bool=False, stagin
except InstanceValueError as e:
warn(epretty(e))
http_resp = HttpResponse.empty(HttpStatus.BadRequest)
except KnotError as e:
except ConfHandlerFailedError as e:
error(epretty(e))
http_resp = HttpResponse.empty(HttpStatus.InternalServerError)
finally:
......@@ -455,7 +454,7 @@ def create_api_op(ds: BaseDatastore):
except (InstanceAlreadyPresent, NoHandlerForOpError, ValueError) as e:
warn(epretty(e))
http_resp = HttpResponse.empty(HttpStatus.BadRequest)
except KnotError as e:
except ConfHandlerFailedError as e:
error(epretty(e))
http_resp = HttpResponse.empty(HttpStatus.InternalServerError)
......
......@@ -134,69 +134,107 @@ class KnotConfig(KnotCtl):
self.connected = False
self.socket_lock.release()
def flush_socket(self):
self.set_timeout(1)
while True:
try:
self.receive_block()
except Exception as e:
if str(e) == "connection timeout":
debug_knot("socket flushed")
break
# Starts a new transaction for configuration data
def begin(self):
if self.conf_state == KnotConfState.NONE:
self.send_block("conf-begin")
try:
self.receive_block()
# print(">>> CONF BEGIN")
self.conf_state = KnotConfState.CONF
except Exception as e:
raise KnotInternalError(str(e))
# Starts a new transaction for zone data
def begin_zone(self):
if self.conf_state == KnotConfState.NONE:
self.send_block("zone-begin")
try:
self.receive_block()
# print(">>> ZONE BEGIN")
self.conf_state = KnotConfState.ZONE
except Exception as e:
raise KnotInternalError(str(e))
# Commits the internal KnotDNS transaction
def commit(self):
if self.conf_state == KnotConfState.CONF:
self.send_block("conf-commit")
try:
self.receive_block()
self.conf_state = KnotConfState.NONE
except Exception as e:
raise KnotInternalError(str(e))
elif self.conf_state == KnotConfState.ZONE:
self.send_block("zone-commit")
else:
raise KnotApiStateError()
def commit_zone(self):
if self.conf_state == KnotConfState.ZONE:
self.send_block("zone-commit")
try:
self.receive_block()
self.conf_state = KnotConfState.NONE
except Exception as e:
raise KnotInternalError(str(e))
try:
self.receive_block()
self.conf_state = KnotConfState.NONE
except Exception as e:
raise KnotInternalError(str(e))
# Aborts the internal KnotDNS transaction
def abort(self):
if self.conf_state == KnotConfState.CONF:
self.send_block("conf-abort")
elif self.conf_state == KnotConfState.ZONE:
self.send_block("zone-abort")
else:
raise KnotApiStateError()
def set_item(self, item=None, section=None, identifier=None, zone=None, data: str=None):
try:
self.receive_block()
self.conf_state = KnotConfState.NONE
except Exception as e:
raise KnotInternalError(str(e))
def unset_section(self, section: str, identifier: str=None):
if not self.connected:
raise KnotApiError("Knot socket is closed")
self.send_block("conf-unset", section=section, identifier=identifier)
try:
resp = self.receive_block()
except Exception as e:
resp = {}
err_str = str(e)
if err_str != "not exists":
raise KnotInternalError(err_str)
def set_item(self, section=None, identifier=None, item=None, data: str=None) -> JsonNodeT:
if not self.connected:
raise KnotApiError("Knot socket is closed")
if data is not None:
if isinstance(data, (int, bool)):
data = str(data).lower()
self.send_block("conf-set", section=section, identifier=identifier, item=item, zone=zone, data=data)
self.send_block("conf-set", section=section, identifier=identifier, item=item, data=data)
try:
resp = self.receive_block()
except Exception as e:
raise KnotInternalError(str(e))
else:
self.send_block("conf-unset", section=section, identifier=identifier, item=item, zone=zone)
resp = {}
def set_item_list(self, item=None, section=None, identifier=None, zone=None, data: List[str]=None):
return resp
def set_item_list(self, section=None, identifier=None, item=None, data: List[str]=None):
if not self.connected:
raise KnotApiError("Knot socket is closed")
self.send_block("conf-unset", section=section, identifier=identifier, item=item, zone=zone)
if data is None:
return
for data_item in data:
self.send_block("conf-set", section=section, identifier=identifier, item=item, zone=zone, data=data_item)
if data is not None:
for data_item in data:
self.send_block("conf-set", section=section, identifier=identifier, item=item, data=data_item)
try:
resp = self.receive_block()
except Exception as e:
raise KnotInternalError(str(e))
# Returns a status data of all or one specific DNS zone
def zone_status(self, domain_name: str=None) -> JsonNodeT:
......@@ -266,7 +304,8 @@ class KnotConfig(KnotCtl):
return resp
def knot_connect(transaction_opts: Optional[JsonNodeT]):
# Connects to Knot control socket and begins a new transaction (config or zone)
def knot_connect(transaction_opts: Optional[JsonNodeT]) -> bool:
debug_knot("Connecting to KNOT socket")
KNOT.knot_connect()
......@@ -277,20 +316,29 @@ def knot_connect(transaction_opts: Optional[JsonNodeT]):
debug_knot("Starting new KNOT zone transaction")
KNOT.begin_zone()
return True
def knot_disconnect(transaction_opts: Optional[JsonNodeT]):
if transaction_opts in ("config", None):
debug_knot("Commiting KNOT config transaction")
# Commits current Knot internal transaction and disconnects from control socket
def knot_disconnect(transaction_opts: Optional[JsonNodeT], failed: bool=False) -> bool:
KNOT.flush_socket()
if failed:
debug_knot("Aborting KNOT transaction")
KNOT.abort()
retval = True
else:
debug_knot("Commiting KNOT transaction")
KNOT.commit()
elif transaction_opts == "zone":
debug_knot("Commiting KNOT zone transaction")
KNOT.commit_zone()
retval = True
debug_knot("Disonnecting from KNOT socket")
KNOT.knot_disconnect()
return retval
def knot_api_init():
def knot_global_init():
global KNOT
if KNOT is None:
KNOT = KnotConfig(CONFIG["KNOT"]["SOCKET"])
......
......@@ -3,7 +3,7 @@ from typing import List, Dict, Union, Any
from yangson.instance import InstanceRoute, ObjectValue, EntryKeys, MemberName
from . import knot_api
from .data import BaseDataListener, SchemaNode, ChangeType, DataChange, ConfHandlerResult
from .data import BaseDataListener, SchemaNode, ChangeType, DataChange
from .helpers import PathFormat, ErrorHelpers, LogHelpers, DataHelpers
from .knot_api import RRecordBase, SOARecord, ARecord, AAAARecord, NSRecord, MXRecord
......@@ -40,8 +40,6 @@ class KnotConfServerListener(BaseDataListener):
knot_api.KNOT.set_item(section="server", item="udp-workers", data=base_nv.get("resources", {}).get("knot-dns:udp-workers"))
knot_api.KNOT.set_item(section="server", item="rate-limit-table-size", data=base_nv.get("response-rate-limiting", {}).get("table-size"))
return ConfHandlerResult.OK
class KnotConfLogListener(BaseDataListener):
def process(self, sn: SchemaNode, ii: InstanceRoute, ch: DataChange):
......@@ -64,14 +62,11 @@ class KnotConfLogListener(BaseDataListener):
knot_api.KNOT.set_item(section="log", identifier=tgt, item="zone", data=logitem.get("zone"))
knot_api.KNOT.set_item(section="log", identifier=tgt, item="any", data=logitem.get("any"))
return ConfHandlerResult.OK
class KnotConfZoneListener(BaseDataListener):
def process(self, sn: SchemaNode, ii: InstanceRoute, ch: DataChange):
debug_confh(self.__class__.__name__ + " triggered")
# ii_str = "".join([str(seg) for seg in ii])
base_ii_str = self.schema_path
base_ii = DataHelpers.parse_ii(base_ii_str, PathFormat.URL)
......@@ -81,41 +76,41 @@ class KnotConfZoneListener(BaseDataListener):
debug_confh("Creating new zone \"{}\"".format(domain))
knot_api.KNOT.zone_new(domain)
# Delete zone
elif (len(ii) == (len(base_ii) + 2)) and isinstance(ii[len(base_ii) + 1], EntryKeys) and (ch.change_type == ChangeType.DELETE):
domain = ii[len(base_ii) + 1].keys["domain"]
elif (len(ii) == 4) and isinstance(ii[3], EntryKeys) and (ch.change_type == ChangeType.DELETE):
domain = ii[3].keys["domain"]
debug_confh("Deleting zone \"{}\"".format(domain))
knot_api.KNOT.zone_remove(domain)
# Edit particular zone
elif (len(ii) >= (len(base_ii) + 2)) and isinstance(ii[len(base_ii)], MemberName) and isinstance(ii[len(base_ii) + 1], EntryKeys):
domain = ii[len(base_ii) + 1].keys["domain"]
elif (len(ii) >= 4) and isinstance(ii[2], MemberName) and isinstance(ii[3], EntryKeys):
domain = ii[3].keys["domain"]
debug_confh("Editing config of zone \"{}\"".format(domain))
# Write whole zone config to Knot
zone_nv = self.ds.get_data_root().goto(ii[0:(len(base_ii) + 1)]).value
knot_api.KNOT.set_item(section="zone", zone=domain, item="comment", data=zone_nv.get("description"))
knot_api.KNOT.set_item(section="zone", zone=domain, item="file", data=zone_nv.get("file"))
knot_api.KNOT.set_item_list(section="zone", zone=domain, item="master", data=zone_nv.get("master"))
knot_api.KNOT.set_item_list(section="zone", zone=domain, item="notify", data=zone_nv.get("notify", {}).get("recipient"))
knot_api.KNOT.set_item_list(section="zone", zone=domain, item="acl", data=zone_nv.get("access-control-list"))
knot_api.KNOT.set_item(section="zone", zone=domain, item="serial-policy", data=zone_nv.get("serial-update-method"))
zone_nv = self.ds.get_data_root().goto(ii[0:4]).value
knot_api.KNOT.unset_section(section="zone", identifier=domain)
knot_api.KNOT.set_item(section="zone", item="domain", data=domain)
knot_api.KNOT.set_item(section="zone", identifier=domain, item="comment", data=zone_nv.get("description"))
knot_api.KNOT.set_item(section="zone", identifier=domain, item="file", data=zone_nv.get("file"))
knot_api.KNOT.set_item_list(section="zone", identifier=domain, item="master", data=zone_nv.get("master"))
knot_api.KNOT.set_item_list(section="zone", identifier=domain, item="notify", data=zone_nv.get("notify", {}).get("recipient"))
knot_api.KNOT.set_item_list(section="zone", identifier=domain, item="acl", data=zone_nv.get("access-control-list"))
knot_api.KNOT.set_item(section="zone", identifier=domain, item="serial-policy", data=zone_nv.get("serial-update-method"))
anytotcp = zone_nv.get("any-to-tcp")
disable_any_str = str(not anytotcp) if isinstance(anytotcp, bool) else None
knot_api.KNOT.set_item(section="zone", zone=domain, item="disable-any", data=disable_any_str)
knot_api.KNOT.set_item(section="zone", identifier=domain, item="disable-any", data=disable_any_str)
knot_api.KNOT.set_item(section="zone", zone=domain, item="max-journal-size", data=zone_nv.get("journal", {}).get("maximum-journal-size"))
knot_api.KNOT.set_item(section="zone", zone=domain, item="zonefile-sync", data=zone_nv.get("journal", {}).get("zone-file-sync-delay"))
knot_api.KNOT.set_item(section="zone", zone=domain, item="ixfr-from-differences", data=zone_nv.get("journal", {}).get("from-differences"))
knot_api.KNOT.set_item(section="zone", identifier=domain, item="max-journal-size", data=zone_nv.get("journal", {}).get("maximum-journal-size"))
knot_api.KNOT.set_item(section="zone", identifier=domain, item="zonefile-sync", data=zone_nv.get("journal", {}).get("zone-file-sync-delay"))
knot_api.KNOT.set_item(section="zone", identifier=domain, item="ixfr-from-differences", data=zone_nv.get("journal", {}).get("from-differences"))
qms = zone_nv.get("query-module")
if qms is not None:
qm_str_list = list(map(lambda n: n["name"] + "/" + n["type"], qms))
qm_list = zone_nv.get("query-module")
if qm_list is not None:
qm_str_list = list(map(lambda n: n["name"] + "/" + n["type"][0], qm_list))
else:
qm_str_list = None
knot_api.KNOT.set_item_list(section="zone", zone=domain, item="module", data=qm_str_list)
knot_api.KNOT.set_item(section="zone", zone=domain, item="semantic-checks", data=zone_nv.get("knot-dns:semantic-checks"))
return ConfHandlerResult.OK
knot_api.KNOT.set_item_list(section="zone", identifier=domain, item="module", data=qm_str_list)
knot_api.KNOT.set_item(section="zone", identifier=domain, item="semantic-checks", data=zone_nv.get("knot-dns:semantic-checks"))
class KnotConfControlListener(BaseDataListener):
......@@ -127,7 +122,6 @@ class KnotConfControlListener(BaseDataListener):
base_nv = self.ds.get_data_root().goto(base_ii).value
knot_api.KNOT.set_item(section="control", item="listen", data=base_nv.get("unix"))
return ConfHandlerResult.OK
class KnotConfAclListener(BaseDataListener):
......@@ -171,8 +165,6 @@ class KnotConfAclListener(BaseDataListener):
print("acl nv={}".format(acl_nv))
self._process_list_item(acl_nv)
return ConfHandlerResult.OK
class KnotZoneDataListener(BaseDataListener):
# Create RR object from "rdata" json node
......@@ -218,8 +210,6 @@ class KnotZoneDataListener(BaseDataListener):
def_ttl = ch.data["zone"]["default-ttl"]
soa = ch.data.get("zone", {}).get("SOA")
if soa is None:
return ConfHandlerResult.ERROR
soarr = SOARecord()
soarr.ttl = def_ttl
......@@ -319,8 +309,3 @@ class KnotZoneDataListener(BaseDataListener):
debug_confh("KnotApi: deleting {} RR from zone \"{}\"".format(rr_type, domain_name))
knot_api.KNOT.zone_del_record(domain_name, rr_owner, rr_type, selector=rr_sel.rrdata_format())
else:
return ConfHandlerResult.ERROR
return ConfHandlerResult.OK
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment