Reworked state data generation architecture, bug fixes, typing improvements, ...

parent a2676e5d
module ietf-yang-library {
namespace "urn:ietf:params:xml:ns:yang:ietf-yang-library";
prefix "yanglib";
import ietf-yang-types { prefix yang; }
import ietf-inet-types { prefix inet; }
organization
"IETF NETCONF (Network Configuration) Working Group";
contact
"WG Web: <http://tools.ietf.org/wg/netconf/>
WG List: <mailto:netconf@ietf.org>
WG Chair: Mehmet Ersue
<mailto:mehmet.ersue@nsn.com>
WG Chair: Mahesh Jethanandani
<mailto:mjethanandani@gmail.com>
Editor: Andy Bierman
<mailto:andy@yumaworks.com>
Editor: Martin Bjorklund
<mailto:mbj@tail-f.com>
Editor: Kent Watsen
<mailto:kwatsen@juniper.net>";
description
"This module contains monitoring information about the YANG
modules and submodules that are used within a YANG-based
server.
Copyright (c) 2015 IETF Trust and the persons identified as
authors of the code. All rights reserved.
Redistribution and use in source and binary forms, with or
without modification, is permitted pursuant to, and subject
to the license terms contained in, the Simplified BSD License
set forth in Section 4.c of the IETF Trust's Legal Provisions
Relating to IETF Documents
(http://trustee.ietf.org/license-info).
This version of this YANG module is part of RFC XXXX; see
the RFC itself for full legal notices.";
// RFC Ed.: replace XXXX with actual RFC number and remove this
// note.
// RFC Ed.: remove this note
// Note: extracted from draft-ietf-netconf-yang-library-00.txt
// RFC Ed.: update the date below with the date of RFC publication
// and remove this note.
revision 2015-01-30 {
description
"Initial revision.";
reference
"RFC XXXX: YANG Module Library.";
}
typedef revision-identifier {
type string {
pattern '\d{4}-\d{2}-\d{2}';
}
description
"Represents a specific date in YYYY-MM-DD format.
TBD: make pattern more precise to exclude leading zeros.";
}
grouping module {
description
"The module data structure is represented as a grouping
so it can be reused in configuration or another monitoring
data structure.";
grouping common-leafs {
description
"Common parameters for YANG modules and submodules.";
leaf name {
type yang:yang-identifier;
description "The YANG module or submodule name.";
}
leaf revision {
type union {
type revision-identifier;
type string { length 0; }
}
description
"The YANG module or submodule revision date.
An empty string is used if no revision statement
is present in the YANG module or submodule.";
}
leaf schema {
type inet:uri;
description
"Contains a URL that represents the YANG schema
resource for this module or submodule.
This leaf will only be present if there is a URL
available for retrieval of the schema for this entry.";
}
}
list module {
key "name revision";
description
"Each entry represents one module currently
supported by the server.";
uses common-leafs;
leaf namespace {
type inet:uri;
mandatory true;
description
"The XML namespace identifier for this module.";
}
leaf-list feature {
type yang:yang-identifier;
description
"List of YANG feature names from this module that are
supported by the server.";
}
leaf-list deviation {
type yang:yang-identifier;
description
"List of YANG deviation module names used by this
server to modify the conformance of the module
associated with this entry.";
}
leaf conformance {
type boolean;
mandatory true;
description
"If 'true', then the server is claiming conformance to
the YANG module identified in this entry.
If 'false', then the server is not claiming any
conformance for the YANG module identified by this
entry. The module may be needed for reusable definitions
such as extensions, features, identifies, typedefs,
or groupings.";
}
container submodules {
description
"Contains information about all the submodules used
by the parent module entry";
list submodule {
key "name revision";
description
"Each entry represents one submodule within the
parent module.";
uses common-leafs;
}
}
} // list module
} // grouping module
container modules {
config false;
description
"Contains YANG module monitoring information.";
leaf module-set-id {
type string;
description
"Contains a server-specific identifier representing
the current set of modules and submodules. The
server MUST change the value of this leaf if the
information represented by the 'module' list instances
has changed.";
}
uses module;
}
}
{
"ietf-yang-library:modules-state": {
"module-set-id": "-TO-BE-INVENTED-",
"module-set-id": "e595da11ace92c0d881995fa7e56bbe86f1f48e9",
"module": [
{
"name": "dns-parameters",
......@@ -51,7 +51,7 @@
},
{
"name": "ietf-yang-library",
"revision": "2016-02-01",
"revision": "2016-06-21",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-yang-library",
"conformance-type": "implement"
},
......
This diff is collapsed.
......@@ -2,19 +2,22 @@ import json
from threading import Lock
from enum import Enum
from colorlog import error, warning as warn, info, debug
from typing import List, Any, Dict, TypeVar, Tuple, Set, Callable
from typing import List, Any, Dict, Callable
from yangson.schema import SchemaRoute, SchemaNode, NonexistentSchemaNode, ListNode, LeafListNode
from yangson.schema import SchemaNode, NonexistentSchemaNode, ListNode, LeafListNode
from yangson.datamodel import DataModel, InstancePath
from yangson.instance import \
InstanceNode, \
NonexistentInstance, \
InstanceTypeError, \
ArrayValue, \
ObjectValue, \
MemberName, \
EntryKeys, \
EntryIndex
from yangson.instance import (
InstanceNode,
NonexistentInstance,
InstanceTypeError,
ArrayValue,
ObjectValue,
MemberName,
EntryKeys,
EntryIndex,
InstanceIdParser,
ResourceIdParser
)
from .helpers import DataHelpers
......@@ -200,10 +203,11 @@ class BaseDatastore:
self.name = name
self.nacm = None # type: NacmConfig
self._data = None # type: InstanceNode
self._yang_lib_data = None # type: InstanceNode
self._dm = dm # type: DataModel
self._data_lock = Lock()
self._lock_username = None # type: str
self._usr_journals = {} # type: Dict[str, UsrChangeJournal]
self._usr_journals = {} # type: Dict[str, UsrChangeJournal]
self.commit_begin_callback = None # type: Callable
self.commit_end_callback = None # type: Callable
......@@ -238,9 +242,9 @@ class BaseDatastore:
# Parse Instance Identifier from string
def parse_ii(self, path: str, path_format: PathFormat) -> InstancePath:
if path_format == PathFormat.URL:
ii = self._dm.parse_resource_id(path)
ii = ResourceIdParser(path).parse()
else:
ii = self._dm.parse_instance_id(path)
ii = InstanceIdParser(path).parse()
return ii
......@@ -276,24 +280,33 @@ class BaseDatastore:
return n
# Get data node, evaluate NACM if required
def get_node_rpc(self, rpc: RpcInfo) -> InstanceNode:
def get_node_rpc(self, rpc: RpcInfo, yl_data=False) -> InstanceNode:
ii = self.parse_ii(rpc.path, rpc.path_format)
root = self._data
n = root.goto(ii)
sn = n.schema_node
for state_node_pth in sn.state_roots():
sn_pth_str = "".join(["/" + pth_seg for pth_seg in state_node_pth])
# print(sn_pth_str)
sdh = STATE_DATA_HANDLES.get_handler(sn_pth_str)
if sdh is not None:
root = sdh.update_node(ii, root).top()
self._data = root
else:
raise NoHandlerForStateDataError()
if yl_data:
root = self._yang_lib_data
else:
root = self._data
# n = root.goto(ii)
# sn = n.schema_node
sch_pth = str(InstancePath(filter(lambda n: isinstance(n, MemberName), ii)))
sn = self.get_schema_node(sch_pth)
if not yl_data:
if sn.state_roots():
self.commit_begin_callback()
for state_node_pth in sn.state_roots():
sn_pth_str = "".join(["/" + pth_seg for pth_seg in state_node_pth])
# print(sn_pth_str)
sdh = STATE_DATA_HANDLES.get_handler(sn_pth_str)
if sdh is not None:
root_val = sdh.update_node(ii, root, True)
root = self._data.update_from_raw(root_val)
else:
raise NoHandlerForStateDataError()
self.commit_end_callback()
self._data = root
n = self._data.goto(ii)
n = root.goto(ii)
try:
with_defs = rpc.qs["with-defaults"][0]
......@@ -307,11 +320,11 @@ class BaseDatastore:
if self.nacm:
nrpc = self.nacm.get_user_nacm(rpc.username)
if nrpc.check_data_node_path(self._data, ii, Permission.NACM_ACCESS_READ) == Action.DENY:
if nrpc.check_data_node_path(root, ii, Permission.NACM_ACCESS_READ) == Action.DENY:
raise NacmForbiddenError()
else:
# Prun subtree data
n = nrpc.check_data_read_path(self._data, ii)
n = nrpc.check_data_read_path(root, ii)
try:
max_depth = int(rpc.qs["depth"][0])
......@@ -605,6 +618,11 @@ class JsonDatastore(BaseDatastore):
with open(filename, "rt") as fp:
self._data = self._dm.from_raw(json.load(fp))
def load_yl_data(self, filename: str):
self._yang_lib_data = None
with open(filename, "rt") as fp:
self._yang_lib_data = self._dm.from_raw(json.load(fp))
def save(self, filename: str):
with open(filename, "w") as jfd:
self.lock_data("json_save")
......
......@@ -257,90 +257,11 @@
]
}
},
"dns-server:dns-server-state": {
"server": {
"boot-time": "2015-05-21T08:02:14+02:00",
"config-time": "2015-06-25T16:22:07+02:00"
},
"zone": [
{
"domain": "example.com",
"class": "IN",
"serial": 2015060600,
"server-role": "slave",
"dnssec-signing:dnssec-signing": {
"key": [
{
"key-id": "d3a9fd3b36a6be275adea2b67c6e82b27ca30e90",
"key-tag": 30348,
"algorithm": "RSASHA256",
"size": 2048,
"flags": "zone-key secure-entry-point",
"created": "2015-06-18T18:02:45+02:00",
"publish": "2015-06-18T19:00:00+02:00",
"activate": "2015-06-20T10:00:00+02:00",
"retire": "2015-07-18T18:02:45+02:00",
"remove": "2015-07-25T00:00:00+02:00"
}
]
},
"statistics": {
"opcodes": {
"opcode-count": [
{
"opcode": "query",
"count": "1234"
},
{
"opcode": "iquery",
"count": "5678"
},
{
"opcode": "status",
"count": "9012"
},
{
"opcode": "notify",
"count": "3456"
}
]
},
"packets": {
"packet-count": [
{
"packet-type": "normal",
"count": "98765"
},
{
"packet-type": "axfr",
"count": "43210"
},
{
"packet-type": "ixfr",
"count": "12345"
},
{
"packet-type": "notify",
"count": "67890"
},
{
"packet-type": "update",
"count": "24680"
}
]
}
}
}
]
},
"ietf-netconf-acm:nacm": {
"enable-nacm": true,
"read-default": "permit",
"write-default": "deny",
"exec-default": "deny",
"denied-operations": 123,
"denied-data-writes": 456,
"denied-notifications": 0,
"groups": {
"group": [
{
......
from typing import List, Tuple, Callable, Any
from .data import BaseDatastore, PathFormat, BaseDataListener
from .usr_state_data_handlers import StateNodeHandlerBase
from yangson.context import Context
from yangson.schema import SchemaNode
......@@ -38,7 +36,7 @@ class ConfDataHandlerList:
def __init__(self):
self.handlers = [] # type: List[Tuple[HandlerSelectorT, BaseDataListener]]
def register_handler(self, handler: BaseDataListener):
def register_handler(self, handler: "BaseDataListener"):
schema_node = handler.schema_node # type: SchemaNode
sch_node_id = str(id(schema_node))
self.handlers.append((sch_node_id, handler))
......
from typing import Dict, Any
from datetime import datetime
from pytz import timezone
from yangson.instance import InstanceIdentifier, MemberName, EntryKeys
from yangson.instance import InstanceRoute, MemberName, EntryKeys
from yangson.datamodel import DataModel
......@@ -14,7 +14,7 @@ class CertHelpers:
class DataHelpers:
# Create parent data nodes to JSON subtree up to top level
@staticmethod
def node2doc(id: InstanceIdentifier, val: Any) -> Dict[str, Any]:
def node2doc(id: InstanceRoute, val: Any) -> Dict[str, Any]:
n = val
for isel in reversed(id):
if isinstance(isel, MemberName):
......
......@@ -12,15 +12,16 @@ from yangson.instance import NonexistentInstance, InstanceTypeError, DuplicateMe
from jetconf.knot_api import KnotError
from .config import CONFIG_GLOBAL, CONFIG_HTTP, NACM_ADMINS, API_ROOT_data, API_ROOT_STAGING_data, API_ROOT_ops
from .helpers import CertHelpers, DataHelpers, DateTimeHelpers, ErrorHelpers
from .data import \
BaseDatastore, \
RpcInfo, \
DataLockError, \
NacmForbiddenError, \
NoHandlerError, \
NoHandlerForOpError, \
InstanceAlreadyPresent, \
from .data import (
BaseDatastore,
RpcInfo,
DataLockError,
NacmForbiddenError,
NoHandlerError,
NoHandlerForOpError,
InstanceAlreadyPresent,
ChangeType
)
QueryStrT = Dict[str, List[str]]
epretty = ErrorHelpers.epretty
......@@ -52,7 +53,7 @@ def api_root_handler(prot: "H2Protocol", headers: OrderedDict, stream_id: int):
prot.conn.send_data(stream_id, response_bytes, end_stream=True)
def _get(prot: "H2Protocol", stream_id: int, ds: BaseDatastore, pth: str):
def _get(prot: "H2Protocol", stream_id: int, ds: BaseDatastore, pth: str, yl_data: bool=False):
username = CertHelpers.get_field(prot.client_cert, "emailAddress")
url_split = pth.split("?")
......@@ -69,7 +70,7 @@ def _get(prot: "H2Protocol", stream_id: int, ds: BaseDatastore, pth: str):
try:
ds.lock_data(username)
n = ds.get_node_rpc(rpc1)
n = ds.get_node_rpc(rpc1, yl_data)
response = json.dumps(n.value, indent=4) + "\n"
response_bytes = response.encode()
......@@ -124,6 +125,8 @@ def create_get_api(ds: BaseDatastore):
prot.send_empty(stream_id, "403", "Forbidden")
else:
_get(prot, stream_id, ds.nacm.nacm_ds, api_pth)
elif ns == "ietf-yang-library":
_get(prot, stream_id, ds, api_pth, yl_data=True)
else:
_get(prot, stream_id, ds, api_pth)
......
from enum import Enum
from typing import List
from typing import List, Union, Dict, Any
from threading import Lock
from .libknot.control import KnotCtl, KnotCtlType
from .config import CONFIG
KNOT = None # type: KnotConfig
JsonNodeT = Union[Dict[str, Any], List]
class KnotError(Exception):
......@@ -161,7 +163,18 @@ class KnotConfig(KnotCtl):
else:
self.send_block("zone-unset", section=section, identifier=identifier, item=item, zone=zone, owner=owner, ttl=ttl, rtype=rtype, data=data)
def zone_new(self, domain_name: str) -> str:
def zone_status(self, domain_name: str=None) -> JsonNodeT:
if not self.connected:
raise KnotApiError("Knot socket is closed")
try:
self.send_block("zone-status", zone=domain_name)
resp = self.receive_block()
except Exception as e:
raise KnotInternalError(str(e))
return resp
def zone_new(self, domain_name: str) -> JsonNodeT:
if not self.connected:
raise KnotApiError("Knot socket is closed")
......@@ -172,7 +185,7 @@ class KnotConfig(KnotCtl):
raise KnotInternalError(str(e))
return resp
def zone_add_record(self, domain_name: str, rr: RRecordBase) -> str:
def zone_add_record(self, domain_name: str, rr: RRecordBase) -> JsonNodeT:
if not self.connected:
raise KnotApiError("Knot socket is closed")
......@@ -184,3 +197,11 @@ class KnotConfig(KnotCtl):
except Exception as e:
raise KnotInternalError(str(e))
return resp
def knot_api_init():
global KNOT
if KNOT is None:
KNOT = KnotConfig(CONFIG["KNOT"]["SOCKET"])
else:
raise ValueError("Knot API already instantiated")
from datetime import datetime
from typing import Dict, Any
from typing import Dict, Any, List, Union
from yangson.datamodel import DataModel
from yangson.instance import InstanceIdentifier, InstanceNode
from yangson.instance import InstanceRoute, InstanceNode, Value, EntryKeys, NonexistentInstance
from jetconf.knot_api import KnotInternalError
from .libknot.control import KnotCtl
from .knot_api import KNOT, KnotConfig
from . import knot_api
from .helpers import DataHelpers
from .handler_list import StateDataHandlerList
JsonNodeT = Dict[str, Any]
JsonNodeT = Union[Dict[str, Any], List]
class StateNonexistentInstance(NonexistentInstance):
def __init__(self, ii: InstanceRoute, text: str) -> None:
self.ii = ii
self.text = text
def __str__(self):
return str(self.ii) + ": " + self.text
class StateNodeHandlerBase:
......@@ -21,9 +33,12 @@ class StateNodeHandlerBase:
def add_member_handler(self, member: str, handler: "StateNodeHandlerBase"):
self.member_handlers[member] = handler
def update_node(self, node_ii: InstanceIdentifier, data_root: InstanceNode) -> InstanceNode:
def update_node(self, node_ii: InstanceRoute, data_root: InstanceNode, with_container: bool) -> InstanceNode:
pass
def gen_container(self, ii: InstanceRoute, data: JsonNodeT) -> JsonNodeT:
return DataHelpers.node2doc(ii, data)
class ZoneSigningStateHandler(StateNodeHandlerBase):
def __init__(self, data_model: DataModel, ctl: KnotCtl):
......@@ -31,17 +46,18 @@ class ZoneSigningStateHandler(StateNodeHandlerBase):
self.sch_pth = "/dns-server:dns-server-state/zone/dnssec-signing:dnssec-signing"
self.schema_node = data_model.get_data_node(self.sch_pth)
def update_node(self, node_ii: InstanceIdentifier, data_root: InstanceNode) -> InstanceNode:
def update_node(self, node_ii: InstanceRoute, data_root: InstanceNode, with_container: bool) -> InstanceNode:
print("zone_state_signing_handler, ii = {}".format(node_ii))
zone_name = node_ii[2].keys.get("domain")
zone_signing = {
"enabled": True,
"key": [
{
"key-id": "d3a9fd3b36a6be275adea2b67c6e82b27ca30e90",
"key-tag": 30348,
"algorithm": "RSASHA256",
"length": 2048,
"size": 2048,
"flags": "zone-key secure-entry-point",
"created": "2015-06-18T18:02:45+02:00",
"publish": "2015-06-18T19:00:00+02:00",
......@@ -52,10 +68,12 @@ class ZoneSigningStateHandler(StateNodeHandlerBase):
]
}
old_node = data_root.goto(node_ii[0:4])
new_node = self.schema_node.from_raw(zone_signing)
new_inst = old_node.update(new_node)
return new_inst
if with_container:
retval = self.gen_container(node_ii[0:4], zone_signing)
else:
retval = zone_signing
return retval
class ZoneStateHandler(StateNodeHandlerBase):
......@@ -64,62 +82,66 @@ class ZoneStateHandler(StateNodeHandlerBase):
self.sch_pth = "/dns-server:dns-server-state/zone"
self.schema_node = data_model.get_data_node(self.sch_pth)
def update_node(self, node_ii: InstanceIdentifier, data_root: InstanceNode) -> InstanceNode:
def update_node(self, node_ii: InstanceRoute, data_root: InstanceNode, with_container: bool) -> InstanceNode:
print("zone_state_handler, ii = {}".format(node_ii))
# Request status of specific zone
if len(node_ii) > 2:
zone_name = node_ii[2].keys.get("domain")
zone_name = node_ii[2].keys.get("domain") + "."
self.knotctl.send_block("zone-status", zone=zone_name)
resp = self.knotctl.receive_block()
resp = resp.get(zone_name + ".")
try:
resp = knot_api.KNOT.zone_status(zone_name)
resp = resp.get(zone_name)
except KnotInternalError:
raise StateNonexistentInstance(node_ii, "No such zone")
zone_obj = {
"domain": zone_name,
"class": "IN",
"serial": int(resp.get("serial")[0]),
"server-role": resp.get("type")[0]
"serial": int(resp.get("serial")),
"server-role": resp.get("type")
}
old_node = data_root.goto(node_ii[0:3])
new_node = self.schema_node.from_raw([zone_obj])[0]
new_inst = old_node.update(new_node)
if with_container:
retval = self.gen_container(node_ii[0:3], zone_obj)
else:
retval = zone_obj
for m, h in self.member_handlers.items():
new_inst = new_inst.new_member(m, h.update_node(node_ii, data_root).value).up()
zone_obj[m] = h.update_node(node_ii, data_root, False)
# Request status of all zones
else:
self.knotctl.send_block("zone-status")
resp = self.knotctl.receive_block()
resp = knot_api.KNOT.zone_status()
zones_list = []
for zone_name, zone_status in resp.items():
zone_name = zone_name.rstrip(".")
zone_obj = {
"domain": zone_name[0:-1],
"domain": zone_name,
"class": "IN",
"serial": int(zone_status.get("serial")[0]),
"server-role": zone_status.get("type")[0]
"serial": int(zone_status.get("serial")),
"server-role": zone_status.get("type")
}
zones_list.append(zone_obj)
old_node = data_root.goto(node_ii[0:2])
new_node = self.schema_node.from_raw(zones_list)
new_inst = old_node.update(new_node)
for m, h in self.member_handlers.items():
zone_obj[m] = h.update_node(node_ii + [EntryKeys({"domain": zone_name})], data_root, False)
for m, h in self.member_handlers.items():
new_inst = new_inst.new_member(m, h.update_node(node_ii, data_root).value).up()
zones_list.append(zone_obj)
if with_container:
retval = self.gen_container(node_ii[0:2], zones_list)
else:
retval = zones_list
return new_inst
return retval
# Create handler hierarchy
def create_zone_state_handlers(handler_list: "StateDataHandlerList", dm: DataModel):
zssh = ZoneSigningStateHandler(dm, KNOT)
zssh = ZoneSigningStateHandler(dm, knot_api.KNOT)