Reworked state data generation architecture, bug fixes, typing improvements, ...

parent a2676e5d
module ietf-yang-library {
namespace "urn:ietf:params:xml:ns:yang:ietf-yang-library";
prefix "yanglib";
import ietf-yang-types { prefix yang; }
import ietf-inet-types { prefix inet; }
organization
"IETF NETCONF (Network Configuration) Working Group";
contact
"WG Web: <http://tools.ietf.org/wg/netconf/>
WG List: <mailto:netconf@ietf.org>
WG Chair: Mehmet Ersue
<mailto:mehmet.ersue@nsn.com>
WG Chair: Mahesh Jethanandani
<mailto:mjethanandani@gmail.com>
Editor: Andy Bierman
<mailto:andy@yumaworks.com>
Editor: Martin Bjorklund
<mailto:mbj@tail-f.com>
Editor: Kent Watsen
<mailto:kwatsen@juniper.net>";
description
"This module contains monitoring information about the YANG
modules and submodules that are used within a YANG-based
server.
Copyright (c) 2015 IETF Trust and the persons identified as
authors of the code. All rights reserved.
Redistribution and use in source and binary forms, with or
without modification, is permitted pursuant to, and subject
to the license terms contained in, the Simplified BSD License
set forth in Section 4.c of the IETF Trust's Legal Provisions
Relating to IETF Documents
(http://trustee.ietf.org/license-info).
This version of this YANG module is part of RFC XXXX; see
the RFC itself for full legal notices.";
// RFC Ed.: replace XXXX with actual RFC number and remove this
// note.
// RFC Ed.: remove this note
// Note: extracted from draft-ietf-netconf-yang-library-00.txt
// RFC Ed.: update the date below with the date of RFC publication
// and remove this note.
revision 2015-01-30 {
description
"Initial revision.";
reference
"RFC XXXX: YANG Module Library.";
}
typedef revision-identifier {
type string {
pattern '\d{4}-\d{2}-\d{2}';
}
description
"Represents a specific date in YYYY-MM-DD format.
TBD: make pattern more precise to exclude leading zeros.";
}
grouping module {
description
"The module data structure is represented as a grouping
so it can be reused in configuration or another monitoring
data structure.";
grouping common-leafs {
description
"Common parameters for YANG modules and submodules.";
leaf name {
type yang:yang-identifier;
description "The YANG module or submodule name.";
}
leaf revision {
type union {
type revision-identifier;
type string { length 0; }
}
description
"The YANG module or submodule revision date.
An empty string is used if no revision statement
is present in the YANG module or submodule.";
}
leaf schema {
type inet:uri;
description
"Contains a URL that represents the YANG schema
resource for this module or submodule.
This leaf will only be present if there is a URL
available for retrieval of the schema for this entry.";
}
}
list module {
key "name revision";
description
"Each entry represents one module currently
supported by the server.";
uses common-leafs;
leaf namespace {
type inet:uri;
mandatory true;
description
"The XML namespace identifier for this module.";
}
leaf-list feature {
type yang:yang-identifier;
description
"List of YANG feature names from this module that are
supported by the server.";
}
leaf-list deviation {
type yang:yang-identifier;
description
"List of YANG deviation module names used by this
server to modify the conformance of the module
associated with this entry.";
}
leaf conformance {
type boolean;
mandatory true;
description
"If 'true', then the server is claiming conformance to
the YANG module identified in this entry.
If 'false', then the server is not claiming any
conformance for the YANG module identified by this
entry. The module may be needed for reusable definitions
such as extensions, features, identifies, typedefs,
or groupings.";
}
container submodules {
description
"Contains information about all the submodules used
by the parent module entry";
list submodule {
key "name revision";
description
"Each entry represents one submodule within the
parent module.";
uses common-leafs;
}
}
} // list module
} // grouping module
container modules {
config false;
description
"Contains YANG module monitoring information.";
leaf module-set-id {
type string;
description
"Contains a server-specific identifier representing
the current set of modules and submodules. The
server MUST change the value of this leaf if the
information represented by the 'module' list instances
has changed.";
}
uses module;
}
}
{ {
"ietf-yang-library:modules-state": { "ietf-yang-library:modules-state": {
"module-set-id": "-TO-BE-INVENTED-", "module-set-id": "e595da11ace92c0d881995fa7e56bbe86f1f48e9",
"module": [ "module": [
{ {
"name": "dns-parameters", "name": "dns-parameters",
...@@ -51,7 +51,7 @@ ...@@ -51,7 +51,7 @@
}, },
{ {
"name": "ietf-yang-library", "name": "ietf-yang-library",
"revision": "2016-02-01", "revision": "2016-06-21",
"namespace": "urn:ietf:params:xml:ns:yang:ietf-yang-library", "namespace": "urn:ietf:params:xml:ns:yang:ietf-yang-library",
"conformance-type": "implement" "conformance-type": "implement"
}, },
......
This diff is collapsed.
...@@ -2,19 +2,22 @@ import json ...@@ -2,19 +2,22 @@ import json
from threading import Lock from threading import Lock
from enum import Enum from enum import Enum
from colorlog import error, warning as warn, info, debug from colorlog import error, warning as warn, info, debug
from typing import List, Any, Dict, TypeVar, Tuple, Set, Callable from typing import List, Any, Dict, Callable
from yangson.schema import SchemaRoute, SchemaNode, NonexistentSchemaNode, ListNode, LeafListNode from yangson.schema import SchemaNode, NonexistentSchemaNode, ListNode, LeafListNode
from yangson.datamodel import DataModel, InstancePath from yangson.datamodel import DataModel, InstancePath
from yangson.instance import \ from yangson.instance import (
InstanceNode, \ InstanceNode,
NonexistentInstance, \ NonexistentInstance,
InstanceTypeError, \ InstanceTypeError,
ArrayValue, \ ArrayValue,
ObjectValue, \ ObjectValue,
MemberName, \ MemberName,
EntryKeys, \ EntryKeys,
EntryIndex EntryIndex,
InstanceIdParser,
ResourceIdParser
)
from .helpers import DataHelpers from .helpers import DataHelpers
...@@ -200,10 +203,11 @@ class BaseDatastore: ...@@ -200,10 +203,11 @@ class BaseDatastore:
self.name = name self.name = name
self.nacm = None # type: NacmConfig self.nacm = None # type: NacmConfig
self._data = None # type: InstanceNode self._data = None # type: InstanceNode
self._yang_lib_data = None # type: InstanceNode
self._dm = dm # type: DataModel self._dm = dm # type: DataModel
self._data_lock = Lock() self._data_lock = Lock()
self._lock_username = None # type: str self._lock_username = None # type: str
self._usr_journals = {} # type: Dict[str, UsrChangeJournal] self._usr_journals = {} # type: Dict[str, UsrChangeJournal]
self.commit_begin_callback = None # type: Callable self.commit_begin_callback = None # type: Callable
self.commit_end_callback = None # type: Callable self.commit_end_callback = None # type: Callable
...@@ -238,9 +242,9 @@ class BaseDatastore: ...@@ -238,9 +242,9 @@ class BaseDatastore:
# Parse Instance Identifier from string # Parse Instance Identifier from string
def parse_ii(self, path: str, path_format: PathFormat) -> InstancePath: def parse_ii(self, path: str, path_format: PathFormat) -> InstancePath:
if path_format == PathFormat.URL: if path_format == PathFormat.URL:
ii = self._dm.parse_resource_id(path) ii = ResourceIdParser(path).parse()
else: else:
ii = self._dm.parse_instance_id(path) ii = InstanceIdParser(path).parse()
return ii return ii
...@@ -276,24 +280,33 @@ class BaseDatastore: ...@@ -276,24 +280,33 @@ class BaseDatastore:
return n return n
# Get data node, evaluate NACM if required # Get data node, evaluate NACM if required
def get_node_rpc(self, rpc: RpcInfo) -> InstanceNode: def get_node_rpc(self, rpc: RpcInfo, yl_data=False) -> InstanceNode:
ii = self.parse_ii(rpc.path, rpc.path_format) ii = self.parse_ii(rpc.path, rpc.path_format)
root = self._data if yl_data:
n = root.goto(ii) root = self._yang_lib_data
sn = n.schema_node else:
root = self._data
for state_node_pth in sn.state_roots():
sn_pth_str = "".join(["/" + pth_seg for pth_seg in state_node_pth]) # n = root.goto(ii)
# print(sn_pth_str) # sn = n.schema_node
sdh = STATE_DATA_HANDLES.get_handler(sn_pth_str) sch_pth = str(InstancePath(filter(lambda n: isinstance(n, MemberName), ii)))
if sdh is not None: sn = self.get_schema_node(sch_pth)
root = sdh.update_node(ii, root).top()
self._data = root if not yl_data:
else: if sn.state_roots():
raise NoHandlerForStateDataError() self.commit_begin_callback()
for state_node_pth in sn.state_roots():
sn_pth_str = "".join(["/" + pth_seg for pth_seg in state_node_pth])
# print(sn_pth_str)
sdh = STATE_DATA_HANDLES.get_handler(sn_pth_str)
if sdh is not None:
root_val = sdh.update_node(ii, root, True)
root = self._data.update_from_raw(root_val)
else:
raise NoHandlerForStateDataError()
self.commit_end_callback()
self._data = root n = root.goto(ii)
n = self._data.goto(ii)
try: try:
with_defs = rpc.qs["with-defaults"][0] with_defs = rpc.qs["with-defaults"][0]
...@@ -307,11 +320,11 @@ class BaseDatastore: ...@@ -307,11 +320,11 @@ class BaseDatastore:
if self.nacm: if self.nacm:
nrpc = self.nacm.get_user_nacm(rpc.username) nrpc = self.nacm.get_user_nacm(rpc.username)
if nrpc.check_data_node_path(self._data, ii, Permission.NACM_ACCESS_READ) == Action.DENY: if nrpc.check_data_node_path(root, ii, Permission.NACM_ACCESS_READ) == Action.DENY:
raise NacmForbiddenError() raise NacmForbiddenError()
else: else:
# Prun subtree data # Prun subtree data
n = nrpc.check_data_read_path(self._data, ii) n = nrpc.check_data_read_path(root, ii)
try: try:
max_depth = int(rpc.qs["depth"][0]) max_depth = int(rpc.qs["depth"][0])
...@@ -605,6 +618,11 @@ class JsonDatastore(BaseDatastore): ...@@ -605,6 +618,11 @@ class JsonDatastore(BaseDatastore):
with open(filename, "rt") as fp: with open(filename, "rt") as fp:
self._data = self._dm.from_raw(json.load(fp)) self._data = self._dm.from_raw(json.load(fp))
def load_yl_data(self, filename: str):
self._yang_lib_data = None
with open(filename, "rt") as fp:
self._yang_lib_data = self._dm.from_raw(json.load(fp))
def save(self, filename: str): def save(self, filename: str):
with open(filename, "w") as jfd: with open(filename, "w") as jfd:
self.lock_data("json_save") self.lock_data("json_save")
......
...@@ -257,90 +257,11 @@ ...@@ -257,90 +257,11 @@
] ]
} }
}, },
"dns-server:dns-server-state": {
"server": {
"boot-time": "2015-05-21T08:02:14+02:00",
"config-time": "2015-06-25T16:22:07+02:00"
},
"zone": [
{
"domain": "example.com",
"class": "IN",
"serial": 2015060600,
"server-role": "slave",
"dnssec-signing:dnssec-signing": {
"key": [
{
"key-id": "d3a9fd3b36a6be275adea2b67c6e82b27ca30e90",
"key-tag": 30348,
"algorithm": "RSASHA256",
"size": 2048,
"flags": "zone-key secure-entry-point",
"created": "2015-06-18T18:02:45+02:00",
"publish": "2015-06-18T19:00:00+02:00",
"activate": "2015-06-20T10:00:00+02:00",
"retire": "2015-07-18T18:02:45+02:00",
"remove": "2015-07-25T00:00:00+02:00"
}
]
},
"statistics": {
"opcodes": {
"opcode-count": [
{
"opcode": "query",
"count": "1234"
},
{
"opcode": "iquery",
"count": "5678"
},
{
"opcode": "status",
"count": "9012"
},
{
"opcode": "notify",
"count": "3456"
}
]
},
"packets": {
"packet-count": [
{
"packet-type": "normal",
"count": "98765"
},
{
"packet-type": "axfr",
"count": "43210"
},
{
"packet-type": "ixfr",
"count": "12345"
},
{
"packet-type": "notify",
"count": "67890"
},
{
"packet-type": "update",
"count": "24680"
}
]
}
}
}
]
},
"ietf-netconf-acm:nacm": { "ietf-netconf-acm:nacm": {
"enable-nacm": true, "enable-nacm": true,
"read-default": "permit", "read-default": "permit",
"write-default": "deny", "write-default": "deny",
"exec-default": "deny", "exec-default": "deny",
"denied-operations": 123,
"denied-data-writes": 456,
"denied-notifications": 0,
"groups": { "groups": {
"group": [ "group": [
{ {
......
from typing import List, Tuple, Callable, Any from typing import List, Tuple, Callable, Any
from .data import BaseDatastore, PathFormat, BaseDataListener
from .usr_state_data_handlers import StateNodeHandlerBase
from yangson.context import Context from yangson.context import Context
from yangson.schema import SchemaNode from yangson.schema import SchemaNode
...@@ -38,7 +36,7 @@ class ConfDataHandlerList: ...@@ -38,7 +36,7 @@ class ConfDataHandlerList:
def __init__(self): def __init__(self):
self.handlers = [] # type: List[Tuple[HandlerSelectorT, BaseDataListener]] self.handlers = [] # type: List[Tuple[HandlerSelectorT, BaseDataListener]]
def register_handler(self, handler: BaseDataListener): def register_handler(self, handler: "BaseDataListener"):
schema_node = handler.schema_node # type: SchemaNode schema_node = handler.schema_node # type: SchemaNode
sch_node_id = str(id(schema_node)) sch_node_id = str(id(schema_node))
self.handlers.append((sch_node_id, handler)) self.handlers.append((sch_node_id, handler))
......
from typing import Dict, Any from typing import Dict, Any
from datetime import datetime from datetime import datetime
from pytz import timezone from pytz import timezone
from yangson.instance import InstanceIdentifier, MemberName, EntryKeys from yangson.instance import InstanceRoute, MemberName, EntryKeys
from yangson.datamodel import DataModel from yangson.datamodel import DataModel
...@@ -14,7 +14,7 @@ class CertHelpers: ...@@ -14,7 +14,7 @@ class CertHelpers:
class DataHelpers: class DataHelpers:
# Create parent data nodes to JSON subtree up to top level # Create parent data nodes to JSON subtree up to top level
@staticmethod @staticmethod
def node2doc(id: InstanceIdentifier, val: Any) -> Dict[str, Any]: def node2doc(id: InstanceRoute, val: Any) -> Dict[str, Any]:
n = val n = val
for isel in reversed(id): for isel in reversed(id):
if isinstance(isel, MemberName): if isinstance(isel, MemberName):
......
...@@ -12,15 +12,16 @@ from yangson.instance import NonexistentInstance, InstanceTypeError, DuplicateMe ...@@ -12,15 +12,16 @@ from yangson.instance import NonexistentInstance, InstanceTypeError, DuplicateMe
from jetconf.knot_api import KnotError from jetconf.knot_api import KnotError
from .config import CONFIG_GLOBAL, CONFIG_HTTP, NACM_ADMINS, API_ROOT_data, API_ROOT_STAGING_data, API_ROOT_ops from .config import CONFIG_GLOBAL, CONFIG_HTTP, NACM_ADMINS, API_ROOT_data, API_ROOT_STAGING_data, API_ROOT_ops
from .helpers import CertHelpers, DataHelpers, DateTimeHelpers, ErrorHelpers from .helpers import CertHelpers, DataHelpers, DateTimeHelpers, ErrorHelpers
from .data import \ from .data import (
BaseDatastore, \ BaseDatastore,
RpcInfo, \ RpcInfo,
DataLockError, \ DataLockError,
NacmForbiddenError, \ NacmForbiddenError,
NoHandlerError, \ NoHandlerError,
NoHandlerForOpError, \ NoHandlerForOpError,
InstanceAlreadyPresent, \ InstanceAlreadyPresent,
ChangeType ChangeType
)
QueryStrT = Dict[str, List[str]] QueryStrT = Dict[str, List[str]]
epretty = ErrorHelpers.epretty epretty = ErrorHelpers.epretty
...@@ -52,7 +53,7 @@ def api_root_handler(prot: "H2Protocol", headers: OrderedDict, stream_id: int): ...@@ -52,7 +53,7 @@ def api_root_handler(prot: "H2Protocol", headers: OrderedDict, stream_id: int):
prot.conn.send_data(stream_id, response_bytes, end_stream=True) prot.conn.send_data(stream_id, response_bytes, end_stream=True)
def _get(prot: "H2Protocol", stream_id: int, ds: BaseDatastore, pth: str): def _get(prot: "H2Protocol", stream_id: int, ds: BaseDatastore, pth: str, yl_data: bool=False):
username = CertHelpers.get_field(prot.client_cert, "emailAddress") username = CertHelpers.get_field(prot.client_cert, "emailAddress")
url_split = pth.split("?") url_split = pth.split("?")
...@@ -69,7 +70,7 @@ def _get(prot: "H2Protocol", stream_id: int, ds: BaseDatastore, pth: str): ...@@ -69,7 +70,7 @@ def _get(prot: "H2Protocol", stream_id: int, ds: BaseDatastore, pth: str):
try: try:
ds.lock_data(username) ds.lock_data(username)
n = ds.get_node_rpc(rpc1) n = ds.get_node_rpc(rpc1, yl_data)
response = json.dumps(n.value, indent=4) + "\n" response = json.dumps(n.value, indent=4) + "\n"
response_bytes = response.encode() response_bytes = response.encode()
...@@ -124,6 +125,8 @@ def create_get_api(ds: BaseDatastore): ...@@ -124,6 +125,8 @@ def create_get_api(ds: BaseDatastore):
prot.send_empty(stream_id, "403", "Forbidden") prot.send_empty(stream_id, "403", "Forbidden")
else: else:
_get(prot, stream_id, ds.nacm.nacm_ds, api_pth) _get(prot, stream_id, ds.nacm.nacm_ds, api_pth)
elif ns == "ietf-yang-library":
_get(prot, stream_id, ds, api_pth, yl_data=True)
else: else:
_get(prot, stream_id, ds, api_pth) _get(prot, stream_id, ds, api_pth)
......
from enum import Enum from enum import Enum
from typing import List from typing import List, Union, Dict, Any
from threading import Lock from threading import Lock
from .libknot.control import KnotCtl, KnotCtlType from .libknot.control import KnotCtl, KnotCtlType
from .config import CONFIG
KNOT = None # type: KnotConfig KNOT = None # type: KnotConfig
JsonNodeT = Union[Dict[str, Any], List]
class KnotError(Exception): class KnotError(Exception):
...@@ -161,7 +163,18 @@ class KnotConfig(KnotCtl): ...@@ -161,7 +163,18 @@ class KnotConfig(KnotCtl):
else: else:
self.send_block("zone-unset", section=section, identifier=identifier, item=item, zone=zone, owner=owner, ttl=ttl, rtype=rtype, data=data) self.send_block("zone-unset", section=section, identifier=identifier, item=item, zone=zone, owner=owner, ttl=ttl, rtype=rtype, data=data)
def zone_new(self, domain_name: str) -> str: def zone_status(self, domain_name: str=None) -> JsonNodeT:
if not self.connected:
raise KnotApiError("Knot socket is closed")
try:
self.send_block("zone-status", zone=domain_name)
resp = self.receive_block()
except Exception as e:
raise KnotInternalError(str(e))
return resp
def zone_new(self, domain_name: str) -> JsonNodeT:
if not self.connected: if not self.connected:
raise KnotApiError("Knot socket is closed") raise KnotApiError("Knot socket is closed")
...@@ -172,7 +185,7 @@ class KnotConfig(KnotCtl): ...@@ -172,7 +185,7 @@ class KnotConfig(KnotCtl):
raise KnotInternalError(str(e)) raise KnotInternalError(str(e))
return resp return resp
def zone_add_record(self, domain_name: str, rr: RRecordBase) -> str: def zone_add_record(self, domain_name: str, rr: RRecordBase) -> JsonNodeT:
if not self.connected: if not self.connected:
raise KnotApiError("Knot socket is closed") raise KnotApiError("Knot socket is closed")
...@@ -184,3 +197,11 @@ class KnotConfig(KnotCtl): ...@@ -184,3 +197,11 @@ class KnotConfig(KnotCtl):
except Exception as e: except Exception as e:
raise KnotInternalError(str(e)) raise KnotInternalError(str(e))
return resp return resp
def knot_api_init():
global KNOT
if KNOT is None:
KNOT = KnotConfig(CONFIG["KNOT"]["SOCKET"])
else:
raise ValueError("Knot API already instantiated")
from datetime import datetime from datetime import datetime
from typing import Dict, Any from typing import Dict, Any, List, Union
from yangson.datamodel import DataModel from yangson.datamodel import DataModel
from yangson.instance import InstanceIdentifier, InstanceNode from yangson.instance import InstanceRoute, InstanceNode, Value, EntryKeys, NonexistentInstance
from jetconf.knot_api import KnotInternalError
from .libknot.control import KnotCtl from .libknot.control import KnotCtl
from .knot_api import KNOT, KnotConfig from . import knot_api
from .helpers import DataHelpers
from .handler_list import StateDataHandlerList
JsonNodeT = Dict[str, Any] JsonNodeT = Union[Dict[str, Any], List]
class StateNonexistentInstance(NonexistentInstance):
def __init__(self, ii: InstanceRoute, text: str) -> None:
self.ii = ii
self.text = text
def __str__(self):
return str(self.ii) + ": " + self.text
class StateNodeHandlerBase: class StateNodeHandlerBase:
...@@ -21,9 +33,12 @@ class StateNodeHandlerBase: ...@@ -21,9 +33,12 @@ class StateNodeHandlerBase:
def add_member_handler(self, member: str, handler: "StateNodeHandlerBase"): def add_member_handler(self, member: str, handler: "StateNodeHandlerBase"):
self.member_handlers[member] = handler self.member_handlers[member] = handler
def update_node(self, node_ii: InstanceIdentifier, data_root: InstanceNode) -> InstanceNode: def update_node(self, node_ii: InstanceRoute, data_root: InstanceNode, with_container: bool) -> InstanceNode:
pass pass
def gen_container(self, ii: InstanceRoute, data: JsonNodeT) -> JsonNodeT:
return DataHelpers.node2doc(ii, data)
class ZoneSigningStateHandler(StateNodeHandlerBase): class ZoneSigningStateHandler(StateNodeHandlerBase):
def __init__(self, data_model: DataModel, ctl: KnotCtl): def __init__(self, data_model: DataModel, ctl: KnotCtl):
...@@ -31,17 +46,18 @@ class ZoneSigningStateHandler(StateNodeHandlerBase): ...@@ -31,17 +46,18 @@ class ZoneSigningStateHandler(StateNodeHandlerBase):
self.sch_pth = "/dns-server:dns-server-state/zone/dnssec-signing:dnssec-signing" self.sch_pth = "/dns-server:dns-server-state/zone/dnssec-signing:dnssec-signing"
self.schema_node = data_model.get_data_node(self.sch_pth) self.schema_node = data_model.get_data_node(self.sch_pth)
def update_node(self, node_ii: InstanceIdentifier, data_root: InstanceNode) -> InstanceNode: def update_node(self, node_ii: InstanceRoute, data_root: InstanceNode, with_container: bool) -> InstanceNode:
print("zone_state_signing_handler, ii = {}".format(node_ii)) print("zone_state_signing_handler, ii = {}".format(node_ii))
zone_name = node_ii[2].keys.get("domain") zone_name = node_ii[2].keys.get("domain")
zone_signing = { zone_signing = {
"enabled": True,
"key": [ "key": [