data.py 32.2 KB
Newer Older
Pavel Spirek's avatar
Pavel Spirek committed
1
import json
2

3
from threading import Lock
4
from colorlog import error, warning as warn, info
5
from typing import List, Any, Dict, Callable, Optional, Tuple
6
from datetime import datetime
7

8
from yangson.datamodel import DataModel
9 10
from yangson.enumerations import ValidationScope
from yangson.schemanode import SchemaNode, ListNode, LeafListNode, InternalNode
Pavel Spirek's avatar
Pavel Spirek committed
11
from yangson.instvalue import ArrayValue, ObjectValue
12 13 14
from yangson.instance import (
    InstanceNode,
    NonexistentInstance,
15
    InstanceValueError,
16 17 18
    MemberName,
    EntryKeys,
    EntryIndex,
19
    InstanceRoute,
20
    ArrayEntry,
21
    RootNode
Pavel Spirek's avatar
Pavel Spirek committed
22
)
23

24
from . import config
25
from .helpers import PathFormat, ErrorHelpers, LogHelpers, DataHelpers, JsonNodeT
26
from .nacm import NacmConfig, Permission, Action
27 28
from .journal import ChangeType, UsrChangeJournal, RpcInfo, DataChange
from .handler_base import ConfDataObjectHandler, ConfDataListHandler, StateDataContainerHandler, StateDataListHandler
29
from .handler_list import ConfDataHandlerList, StateDataHandlerList, OpHandlerList, ActionHandlerList
30 31 32 33 34 35 36 37 38 39
from .errors import (
    StagingDataException,
    NoHandlerForStateDataError,
    NoHandlerForOpError,
    InstanceAlreadyPresent,
    OpHandlerFailedError,
    NoHandlerError,
    DataLockError,
    NacmForbiddenError
)
40 41 42

epretty = ErrorHelpers.epretty
debug_data = LogHelpers.create_module_dbg_logger(__name__)
Pavel Spirek's avatar
Pavel Spirek committed
43 44


45
class BackendHandlers:
46
    def __init__(self, dm: DataModel):
47 48 49
        self.conf = ConfDataHandlerList()
        self.state = StateDataHandlerList()
        self.op = OpHandlerList()
50
        self.action = ActionHandlerList(dm)
51

52 53
        def _blankfn(*args, **kwargs):
            pass
54

55 56
        self.commit_begin = _blankfn   # type: Callable[[], None]
        self.commit_end = _blankfn     # type: Callable[[bool], None]
57

58

Pavel Spirek's avatar
Pavel Spirek committed
59
class BaseDatastore:
60
    def __init__(self, dm: DataModel, with_nacm: bool=False):
61
        self._dm = dm       # type: DataModel
62
        self._data = None   # type: InstanceNode
63
        self._yang_lib_data = self._dm.from_raw(self._dm.yang_library)  # type: InstanceNode
64
        self._data_history = []     # type: List[InstanceNode]
Pavel Spirek's avatar
Pavel Spirek committed
65
        self._data_lock = Lock()
Pavel Spirek's avatar
Pavel Spirek committed
66
        self._lock_username = None  # type: str
67
        self._usr_journals = {}     # type: Dict[str, UsrChangeJournal]
68
        self.nacm = None    # type: NacmConfig
69
        self.handlers = BackendHandlers(self._dm)
70
        self.nacm = NacmConfig(self, self._dm) if with_nacm else None
71

72 73 74 75
    # Returns DataModel object
    def get_dm(self) -> DataModel:
        return self._dm

Pavel Spirek's avatar
Pavel Spirek committed
76
    # Returns the root node of data tree
77 78 79 80 81
    def get_data_root(self, previous_version: int=0) -> InstanceNode:
        if previous_version > 0:
            return self._data_history[-previous_version]
        else:
            return self._data
Pavel Spirek's avatar
Pavel Spirek committed
82

83
    # Returns the root node of YANG library data tree
84 85 86
    def get_yl_data_root(self) -> InstanceNode:
        return self._yang_lib_data

87
    # Journal manipulation methods
Pavel Spirek's avatar
Pavel Spirek committed
88
    def make_user_journal(self, username: str):
89 90
        usr_journal = self._usr_journals.get(username)
        if usr_journal is not None:
Pavel Spirek's avatar
Pavel Spirek committed
91
            raise StagingDataException("Transaction for user \"{}\" already opened".format(username))
92
        else:
93
            self._usr_journals[username] = UsrChangeJournal(self._data)
94 95 96 97 98 99

    def get_user_journal(self, username: str):
        usr_journal = self._usr_journals.get(username)
        if usr_journal is not None:
            return usr_journal
        else:
Pavel Spirek's avatar
Pavel Spirek committed
100
            raise StagingDataException("Transaction for user \"{}\" not opened".format(username))
101 102

    def drop_user_journal(self, username: str):
103 104
        usr_journal = self._usr_journals.get(username)
        if usr_journal is not None:
105
            del self._usr_journals[username]
106
        else:
Pavel Spirek's avatar
Pavel Spirek committed
107
            raise StagingDataException("Transaction for user \"{}\" not opened".format(username))
108

109
    # Returns the root node of staging data tree (starts a new transaction if nonexistent)
110
    def get_data_root_staging(self, username: str) -> InstanceNode:
111 112 113 114
        try:
            usr_journal = self.get_user_journal(username)
        except StagingDataException:
            info("Starting new transaction for user \"{}\"".format(username))
Pavel Spirek's avatar
Pavel Spirek committed
115
            self.make_user_journal(username)
116 117
            usr_journal = self.get_user_journal(username)

118 119 120
        root = usr_journal.get_root_head()
        return root

121
    # Set a new Instance node as data root, store old root to archive
Pavel Spirek's avatar
Pavel Spirek committed
122
    def set_data_root(self, new_root: InstanceNode):
123
        self._data_history.append(self._data)
Pavel Spirek's avatar
Pavel Spirek committed
124 125
        self._data = new_root

126 127 128 129 130 131
    def data_root_rollback(self, history_steps: int, store_current: bool):
        if store_current:
            self._data_history.append(self._data)

        self._data = self._data_history[-history_steps]

132 133 134 135 136 137 138 139
    def parse_ii(self, path: str, path_format: PathFormat) -> InstanceRoute:
        if path_format == PathFormat.URL:
            ii = self._dm.parse_resource_id(path)
        else:
            ii = self._dm.parse_instance_id(path)

        return ii

140 141
    # Get schema node with particular schema address
    def get_schema_node(self, sch_pth: str) -> SchemaNode:
142
        sn = self._dm.get_data_node(sch_pth)
143
        if sn is None:
144 145
            # raise NonexistentSchemaNode(sch_pth)
            debug_data("Cannot find schema node for " + sch_pth)
146 147
        return sn

148
    # Run configuration data handlers
149
    def run_conf_edit_handler(self, ii: InstanceRoute, ch: DataChange):
Pavel Spirek's avatar
Pavel Spirek committed
150 151 152 153 154 155
        sch_pth_list = list(filter(lambda n: isinstance(n, MemberName), ii))

        if ch.change_type == ChangeType.CREATE:
            # Get target member name
            input_member_name_fq = tuple(ch.input_data.keys())[0]
            input_member_name_ns, input_member_name = input_member_name_fq.split(":", maxsplit=1)
156

Pavel Spirek's avatar
Pavel Spirek committed
157
            # Append it to ii
158 159 160 161 162 163
            schpth_last_ns = None
            for schpth_seg in sch_pth_list:
                if schpth_seg.namespace is not None:
                    schpth_last_ns = schpth_seg.namespace

            if (len(sch_pth_list) == 0) or (schpth_last_ns != input_member_name_ns):
164 165 166
                sch_pth_list.append(MemberName(input_member_name, input_member_name_ns))
            else:
                sch_pth_list.append(MemberName(input_member_name, None))
Pavel Spirek's avatar
Pavel Spirek committed
167 168 169 170 171 172 173

        sch_pth = DataHelpers.ii2str(sch_pth_list)
        sn = self.get_schema_node(sch_pth)

        if sn is None:
            return

174
        h = self.handlers.conf.get_handler(id(sn))
Pavel Spirek's avatar
Pavel Spirek committed
175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
        if h is not None:
            info("handler for actual data node triggered")
            if isinstance(h, ConfDataObjectHandler):
                if ch.change_type == ChangeType.CREATE:
                    h.create(ii, ch)
                elif ch.change_type == ChangeType.REPLACE:
                    h.replace(ii, ch)
                elif ch.change_type == ChangeType.DELETE:
                    h.delete(ii, ch)
            if isinstance(h, ConfDataListHandler):
                if ch.change_type == ChangeType.CREATE:
                    h.create_item(ii, ch)
                elif ch.change_type == ChangeType.REPLACE:
                    h.replace_item(ii, ch)
                elif ch.change_type == ChangeType.DELETE:
                    h.delete_item(ii, ch)
        else:
            sn = sn.parent
            while sn is not None:
194
                h = self.handlers.conf.get_handler(id(sn))
Pavel Spirek's avatar
Pavel Spirek committed
195 196 197 198 199 200 201 202
                if h is not None and isinstance(h, ConfDataObjectHandler):
                    info("handler for superior data node triggered, replace")
                    # print(h.schema_path)
                    # print(h.__class__.__name__)
                    h.replace(ii, ch)
                if h is not None and isinstance(h, ConfDataListHandler):
                    info("handler for superior data node triggered, replace_item")
                    h.replace_item(ii, ch)
203
                sn = sn.parent
204

Pavel Spirek's avatar
Pavel Spirek committed
205
    # Get data node, evaluate NACM if required
Pavel Spirek's avatar
Pavel Spirek committed
206 207
    def get_node_rpc(self, rpc: RpcInfo, staging=False) -> InstanceNode:
        ii = self.parse_ii(rpc.path, rpc.path_format)
208

Pavel Spirek's avatar
Pavel Spirek committed
209
        if staging:
210
            root = self.get_data_root_staging(rpc.username)
211 212
        else:
            root = self._data
213

Pavel Spirek's avatar
Pavel Spirek committed
214
        yl_data_request = False
Pavel Spirek's avatar
Pavel Spirek committed
215 216 217
        if (len(ii) > 0) and (isinstance(ii[0], MemberName)):
            # Not getting root
            ns_first = ii[0].namespace
218
            if (ns_first == "ietf-netconf-acm") and (rpc.username not in config.CFG.nacm["ALLOWED_USERS"]):
Pavel Spirek's avatar
Pavel Spirek committed
219 220 221
                raise NacmForbiddenError(rpc.username + " not allowed to access NACM data")
            elif ns_first == "ietf-yang-library":
                root = self._yang_lib_data
Pavel Spirek's avatar
Pavel Spirek committed
222
                yl_data_request = True
Pavel Spirek's avatar
Pavel Spirek committed
223 224 225
        else:
            # Root node requested
            # Remove NACM data if user is not NACM privieged
226
            if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
Pavel Spirek's avatar
Pavel Spirek committed
227 228 229 230 231 232 233
                try:
                    root = root.delete_item("ietf-netconf-acm:nacm")
                except NonexistentInstance:
                    pass

            # Append YANG library data
            for member_name, member_val in self._yang_lib_data.value.items():
234
                root = root.put_member(member_name, member_val).top()
Pavel Spirek's avatar
Pavel Spirek committed
235

236
        # Resolve schema node of the desired data node
237
        sch_pth_list = filter(lambda isel: isinstance(isel, MemberName), ii)
238
        sch_pth = DataHelpers.ii2str(sch_pth_list)
239
        sn = self.get_schema_node(sch_pth)
240

241 242
        state_roots = sn.state_roots()

Pavel Spirek's avatar
Pavel Spirek committed
243
        # Check if URL points to state data or node that contains state data
Pavel Spirek's avatar
Pavel Spirek committed
244
        if state_roots and not yl_data_request:
245
            debug_data("State roots: {}".format(state_roots))
246
            n = None
247

248 249 250
            for state_root_sch_pth in state_roots:
                state_root_sn = self._dm.get_data_node(state_root_sch_pth)

Pavel Spirek's avatar
Pavel Spirek committed
251
                # Check if the desired node is child of the state root
252 253 254
                sni = sn
                is_child = False
                while sni:
255
                    if sni is state_root_sn:
256 257 258 259 260 261
                        is_child = True
                        break
                    sni = sni.parent

                if is_child:
                    # Direct request for the state data
262
                    sdh = self.handlers.state.get_handler(state_root_sch_pth)
263
                    if sdh is not None:
264
                        if isinstance(sdh, StateDataContainerHandler):
265
                            state_handler_val = sdh.generate_node(ii, rpc.username, staging)
266
                            state_root_n = sdh.schema_node.orphan_instance(state_handler_val)
267
                        elif isinstance(sdh, StateDataListHandler):
268 269 270 271 272 273
                            if (sn is sdh.schema_node) and isinstance(ii[-1], MemberName):
                                state_handler_val = sdh.generate_list(ii, rpc.username, staging)
                                state_root_n = sdh.schema_node.orphan_instance(state_handler_val)
                            else:
                                state_handler_val = sdh.generate_item(ii, rpc.username, staging)
                                state_root_n = sdh.schema_node.orphan_entry(state_handler_val)
274 275
                        else:
                            state_root_n = None
276 277 278 279

                        # Select desired subnode from handler-generated content
                        ii_prefix, ii_rel = sdh.schema_node.split_instance_route(ii)
                        n = state_root_n.goto(ii_rel)
280 281 282 283 284

                        # There should be only one state root, no need to continue
                        if len(state_roots) != 1:
                            warn("URI points to directly to state data, but more state roots found")
                        break
285 286
                    else:
                        raise NoHandlerForStateDataError(rpc.path)
287
                else:
288
                    # Request for config data containing state data
289 290 291 292 293 294
                    n = root.goto(ii)

                    def _fill_state_roots(node: InstanceNode) -> InstanceNode:
                        if isinstance(node.value, ObjectValue):
                            if node.schema_node is state_root_sn.parent:
                                ii_gen = DataHelpers.node_get_ii(node)
295
                                _sdh = self.handlers.state.get_handler(state_root_sch_pth)
296
                                if _sdh is not None:
297
                                    try:
298 299 300 301 302 303
                                        if isinstance(_sdh, StateDataContainerHandler):
                                            _state_handler_val = _sdh.generate_node(ii_gen, rpc.username, staging)
                                        elif isinstance(_sdh, StateDataListHandler):
                                            _state_handler_val = _sdh.generate_list(ii_gen, rpc.username, staging)
                                        else:
                                            _state_handler_val = None
304 305 306 307 308 309 310 311 312 313 314
                                    except Exception as e:
                                        error("Error occured in state data generator (sn: {})".format(state_root_sch_pth))
                                        error(epretty(e))
                                        error("This state node will be omitted.")
                                    else:
                                        if state_root_sn.ns == state_root_sn.parent.ns:
                                            nm_name = state_root_sn.qual_name[0]
                                        else:
                                            nm_name = state_root_sn.qual_name[1] + ":" + state_root_sn.qual_name[0]

                                        # print("nm={}".format(nm_name))
315
                                        node = node.put_member(nm_name, _state_handler_val, raw=True).up()
316 317 318 319 320 321 322 323 324 325 326 327 328 329
                            else:
                                for key in node:
                                    member = node[key]
                                    node = _fill_state_roots(member).up()
                        elif isinstance(node.value, ArrayValue):
                            i = 0
                            arr_len = len(node.value)
                            while i < arr_len:
                                node = _fill_state_roots(node[i]).up()
                                i += 1

                        return node

                    n = _fill_state_roots(n)
330
                    root = n.top()
331
        else:
Pavel Spirek's avatar
Pavel Spirek committed
332
            # No state data in requested node
333
            n = root.goto(ii)
Pavel Spirek's avatar
Pavel Spirek committed
334

Pavel Spirek's avatar
Pavel Spirek committed
335
        # Process "with-defaults" query parameter
336 337
        try:
            with_defs = rpc.qs["with-defaults"][0]
338
        except (IndexError, KeyError):
339 340 341 342 343
            with_defs = None

        if with_defs == "report-all":
            n = n.add_defaults()

Pavel Spirek's avatar
Pavel Spirek committed
344
        # Evaluate NACM if required
345
        if self.nacm and not rpc.skip_nacm_check:
Pavel Spirek's avatar
Pavel Spirek committed
346
            nrpc = self.nacm.get_user_rules(rpc.username)
347
            if nrpc.check_data_node_permission(root, ii, Permission.NACM_ACCESS_READ) == Action.DENY:
Pavel Spirek's avatar
Pavel Spirek committed
348 349
                raise NacmForbiddenError()
            else:
350 351
                # Prune nodes that should not be accessible to user
                n = nrpc.prune_data_tree(n, root, ii, Permission.NACM_ACCESS_READ)
Pavel Spirek's avatar
Pavel Spirek committed
352

Pavel Spirek's avatar
Pavel Spirek committed
353
        # Process "depth" query parameter
354
        try:
355 356 357 358 359 360 361
            max_depth_str = rpc.qs["depth"][0]
            if max_depth_str == "unbounded":
                max_depth = None
            else:
                max_depth = int(max_depth_str) - 1
                if (max_depth < 0) or (max_depth > 65535):
                    raise ValueError()
362
        except (IndexError, KeyError):
363 364 365 366 367 368 369 370 371 372 373
            max_depth = None
        except ValueError:
            raise ValueError("Invalid value of query param \"depth\"")

        if max_depth is not None:
            def _tree_limit_depth(node: InstanceNode, depth: int) -> InstanceNode:
                if isinstance(node.value, ObjectValue):
                    if depth > max_depth:
                        node.value = ObjectValue({})
                    else:
                        for child_key in sorted(node.value.keys()):
374
                            m = node[child_key]
375 376
                            node = _tree_limit_depth(m, depth + 1).up()
                elif isinstance(node.value, ArrayValue):
377 378 379 380
                    depth -= 1
                    for i in range(len(node.value)):
                        e = node[i]
                        node = _tree_limit_depth(e, depth + 1).up()
381 382 383 384

                return node
            n = _tree_limit_depth(n, 1)

Pavel Spirek's avatar
Pavel Spirek committed
385
        # Return result
Pavel Spirek's avatar
Pavel Spirek committed
386 387
        return n

388
    # Create new data node
389
    def create_node_rpc(self, root: InstanceNode, rpc: RpcInfo, value: Any) -> Tuple[InstanceNode, bool]:
390
        ii = self.parse_ii(rpc.path, rpc.path_format)
391

392
        # Get target member name
393 394
        input_member_keys = tuple(value.keys())
        if len(input_member_keys) != 1:
395
            raise ValueError("Received json object must contain exactly one member")
Pavel Spirek's avatar
Pavel Spirek committed
396

397 398 399 400 401
        input_member_name_fq = input_member_keys[0]
        try:
            input_member_ns, input_member_name = input_member_name_fq.split(":", maxsplit=1)
        except ValueError:
            raise ValueError("Input object name must me in fully-qualified format")
402
        input_member_value = value[input_member_name_fq]
Pavel Spirek's avatar
Pavel Spirek committed
403

404
        # Deny any changes of NACM data for non-privileged users
405
        nacm_changed = False
406 407 408
        if (len(ii) > 0) and (isinstance(ii[0], MemberName)):
            # Not getting root
            ns_first = ii[0].namespace
409 410
            if ns_first == "ietf-netconf-acm":
                nacm_changed = True
411
                if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
412
                    raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
413 414
        else:
            # Editing root node
415 416
            if input_member_ns == "ietf-netconf-acm":
                nacm_changed = True
417
                if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
418
                    raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
419 420

        # Evaluate NACM
421
        if self.nacm and not rpc.skip_nacm_check:
422 423 424
            nrpc = self.nacm.get_user_rules(rpc.username)
            if nrpc.check_data_node_permission(root, ii, Permission.NACM_ACCESS_CREATE) == Action.DENY:
                raise NacmForbiddenError()
425

426 427
        n = root.goto(ii)

428
        # Get target schema node
429
        sn = n.schema_node  # type: InternalNode
430
        member_sn = sn.get_child(input_member_name, input_member_ns)
431

432 433 434
        if member_sn is None:
            raise ValueError("Received json object contains unknown member")

435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450
        # Check if target member already exists
        if sn.ns == member_sn.ns:
            try:
                existing_member = n[input_member_name]
            except NonexistentInstance:
                existing_member = None
        else:
            try:
                existing_member = n[input_member_name_fq]
            except NonexistentInstance:
                existing_member = None

        # Get query parameters
        insert = rpc.qs.get("insert", [None])[0]
        point = rpc.qs.get("point", [None])[0]

451
        if isinstance(member_sn, ListNode):
Pavel Spirek's avatar
Pavel Spirek committed
452
            # Append received node to list
453

454 455
            # Create list if necessary
            if existing_member is None:
456 457
                new_member_name = input_member_name if n.namespace == input_member_ns else input_member_name_fq
                existing_member = n.put_member(new_member_name, ArrayValue([]))
458

459 460
            # Get ListNode key names
            list_node_keys = member_sn.keys     # Key names in the form [(key, ns), ]
461 462

            if insert == "first":
463
                # Optimization
464
                if len(existing_member.value) > 0:
465
                    list_entry_first = existing_member[0]   # type: ArrayEntry
466
                    new_member = list_entry_first.insert_before(input_member_value, raw=True).up()
467
                else:
468
                    new_member = existing_member.update([input_member_value], raw=True)
469
            elif (insert == "last") or (insert is None):
470
                # Optimization
471
                if len(existing_member.value) > 0:
472
                    list_entry_last = existing_member[-1]   # type: ArrayEntry
473
                    new_member = list_entry_last.insert_after(input_member_value, raw=True).up()
474
                else:
475
                    new_member = existing_member.update([input_member_value], raw=True)
476 477 478 479 480 481 482 483 484 485 486
            elif (insert == "before") and (point is not None):
                point_keys_val = point.split(",")  # List key values passed in the "point" query argument
                if len(list_node_keys) != len(point_keys_val):
                    raise ValueError(
                        "Invalid number of keys passed in 'point' query: {} ({} expected)".format(
                            len(point_keys_val), len(list_node_keys)
                        )
                    )
                entry_keys = dict(map(lambda i: (list_node_keys[i], point_keys_val[i]), range(len(list_node_keys))))
                entry_sel = EntryKeys(entry_keys)
                point_list_entry = entry_sel.goto_step(existing_member)   # type: ArrayEntry
487
                new_member = point_list_entry.insert_before(input_member_value, raw=True).up()
488 489 490 491 492 493 494 495 496 497 498
            elif (insert == "after") and (point is not None):
                point_keys_val = point.split(",")  # List key values passed in the "point" query argument
                if len(list_node_keys) != len(point_keys_val):
                    raise ValueError(
                        "Invalid number of keys passed in 'point' query: {} ({} expected)".format(
                            len(point_keys_val), len(list_node_keys)
                        )
                    )
                entry_keys = dict(map(lambda i: (list_node_keys[i], point_keys_val[i]), range(len(list_node_keys))))
                entry_sel = EntryKeys(entry_keys)
                point_list_entry = entry_sel.goto_step(existing_member)   # type: ArrayEntry
499
                new_member = point_list_entry.insert_after(input_member_value, raw=True).up()
500
            else:
501
                raise ValueError("Invalid 'insert'/'point' query values")
502 503 504 505 506
        elif isinstance(member_sn, LeafListNode):
            # Append received node to leaf list

            # Create leaf list if necessary
            if existing_member is None:
507 508
                new_member_name = input_member_name if n.namespace == input_member_ns else input_member_name_fq
                existing_member = n.put_member(new_member_name, ArrayValue([]))
509

510
            # Convert input data from List/Dict to ArrayValue/ObjectValue
511
            new_value_item = member_sn.entry_from_raw(input_member_value)
512 513

            if insert == "first":
514
                new_member = existing_member.update(ArrayValue([new_value_item] + existing_member.value))
515
            elif (insert == "last") or (insert is None):
516
                new_member = existing_member.update(ArrayValue(existing_member.value + [new_value_item]))
517
            else:
518
                raise ValueError("Invalid 'insert' query value")
Pavel Spirek's avatar
Pavel Spirek committed
519
        else:
520
            # Create new container member
521

522
            if existing_member is None:
523
                # Create new node (object member)
524 525
                new_member_name = input_member_name if n.namespace == input_member_ns else input_member_name_fq
                new_member = n.put_member(new_member_name, input_member_value, raw=True)
526 527 528
            else:
                # Data node already exists
                raise InstanceAlreadyPresent("Member \"{}\" already present in \"{}\"".format(input_member_name, ii))
Pavel Spirek's avatar
Pavel Spirek committed
529

530
        return new_member.top(), nacm_changed
531

532
    # PUT data node
533
    def update_node_rpc(self, root: InstanceNode, rpc: RpcInfo, value: Any) -> Tuple[InstanceNode, bool]:
534
        ii = self.parse_ii(rpc.path, rpc.path_format)
535 536 537 538 539 540 541 542 543 544 545 546 547

        # Get target member name
        input_member_keys = tuple(value.keys())
        if len(input_member_keys) != 1:
            raise ValueError("Received json object must contain exactly one member")

        input_member_name_fq = input_member_keys[0]
        try:
            input_member_ns, input_member_name = input_member_name_fq.split(":", maxsplit=1)
        except ValueError:
            raise ValueError("Input object name must me in fully-qualified format")
        input_member_value = value[input_member_name_fq]

548
        n = root.goto(ii)
549

550
        # Deny any changes of NACM data for non-privileged users
551
        nacm_changed = False
552 553 554
        if (len(ii) > 0) and (isinstance(ii[0], MemberName)):
            # Not getting root
            ns_first = ii[0].namespace
555 556
            if ns_first == "ietf-netconf-acm":
                nacm_changed = True
557
                if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
558
                    raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
559 560 561 562
        else:
            # Replacing root node
            # Check if NACM data are present in the datastore
            nacm_val = n.value.get("ietf-netconf-acm:nacm")
563 564
            if nacm_val is not None:
                nacm_changed = True
565
                if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
566
                    raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
567 568

        # Evaluate NACM
569
        if self.nacm and not rpc.skip_nacm_check:
Pavel Spirek's avatar
Pavel Spirek committed
570
            nrpc = self.nacm.get_user_rules(rpc.username)
571
            if nrpc.check_data_node_permission(root, ii, Permission.NACM_ACCESS_UPDATE) == Action.DENY:
572
                raise NacmForbiddenError()
Pavel Spirek's avatar
Pavel Spirek committed
573

574
        new_n = n.update(input_member_value, raw=True)
575
        new_n.validate(ValidationScope.syntax)
576

577
        return new_n.top(), nacm_changed
578

579
    # Delete data node
580
    def delete_node_rpc(self, root: InstanceNode, rpc: RpcInfo) -> Tuple[InstanceNode, bool]:
581
        ii = self.parse_ii(rpc.path, rpc.path_format)
582
        n = root.goto(ii)
Pavel Spirek's avatar
Pavel Spirek committed
583

584
        # Deny any changes of NACM data for non-privileged users
585
        nacm_changed = False
586 587 588
        if (len(ii) > 0) and (isinstance(ii[0], MemberName)):
            # Not getting root
            ns_first = ii[0].namespace
589 590
            if ns_first == "ietf-netconf-acm":
                nacm_changed = True
591
                if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
592
                    raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
593 594 595 596
        else:
            # Deleting root node
            # Check if NACM data are present in the datastore
            nacm_val = n.value.get("ietf-netconf-acm:nacm")
597 598
            if nacm_val is not None:
                nacm_changed = True
599
                if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
600
                    raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
601 602

        # Evaluate NACM
603
        if self.nacm and not rpc.skip_nacm_check:
Pavel Spirek's avatar
Pavel Spirek committed
604
            nrpc = self.nacm.get_user_rules(rpc.username)
605
            if nrpc.check_data_node_permission(root, ii, Permission.NACM_ACCESS_DELETE) == Action.DENY:
606
                raise NacmForbiddenError()
Pavel Spirek's avatar
Pavel Spirek committed
607

608 609 610
        if len(ii) == 0:
            # Deleting entire datastore
            new_n = RootNode(ObjectValue({}), root.schema_node, datetime.now())
611
        else:
612 613 614 615 616 617 618 619 620 621 622 623 624
            n_parent = n.up()
            last_isel = ii[-1]
            if isinstance(n_parent.value, ArrayValue):
                if isinstance(last_isel, EntryIndex):
                    new_n = n_parent.delete_item(last_isel.index)
                elif isinstance(last_isel, EntryKeys):
                    new_n = n_parent.delete_item(n.index)
                else:
                    raise ValueError("Unknown node selector")
            elif isinstance(n_parent.value, ObjectValue):
                new_n = n_parent.delete_item(last_isel.namespace + ":" + last_isel.name if last_isel.namespace else last_isel.name)
            else:
                raise InstanceValueError(rpc.path, "Invalid target node type")
Pavel Spirek's avatar
Pavel Spirek committed
625

626
        return new_n.top(), nacm_changed
Pavel Spirek's avatar
Pavel Spirek committed
627

628
    # Invoke an operation
629
    def invoke_op_rpc(self, rpc: RpcInfo) -> JsonNodeT:
630 631
        if rpc.op_name.startswith("jetconf:"):
            # Jetconf internal operation
632
            op_handler = self.handlers.op.get_handler(rpc.op_name)
633 634
            if op_handler is None:
                raise NoHandlerForOpError(rpc.op_name)
635

636
            ret_data = op_handler(rpc)
637
        else:
638
            # External operation defined in data model
639
            if self.nacm and not rpc.skip_nacm_check:
Pavel Spirek's avatar
Pavel Spirek committed
640
                nrpc = self.nacm.get_user_rules(rpc.username)
641 642
                if nrpc.check_rpc_name(rpc.op_name) == Action.DENY:
                    raise NacmForbiddenError(
643
                        "Invocation of \"{}\" operation denied for user \"{}\"".format(rpc.op_name, rpc.username)
644 645
                    )

646
            op_handler = self.handlers.op.get_handler(rpc.op_name)
647
            if op_handler is None:
648
                raise NoHandlerForOpError(rpc.op_name)
649

650
            # Get operation input schema
651 652
            sn = self._dm.get_schema_node(rpc.path)
            sn_input = sn.get_child("input")
653

654 655
            # Input arguments are expected, this will validate them
            op_input_args = sn_input.from_raw(rpc.op_input_args) if sn_input.children else None
656

657 658 659 660
            try:
                ret_data = op_handler(op_input_args, rpc.username)
            except Exception as e:
                raise OpHandlerFailedError(epretty(e))
661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695

        return ret_data

    # Invoke a node action
    def invoke_action_rpc(self, root: InstanceNode, rpc: RpcInfo) -> JsonNodeT:
        ii = self.parse_ii(rpc.path, rpc.path_format)
        node_ii = ii[0:-1]
        n = root.goto(node_ii)

        # Evaluate NACM
        if self.nacm and not rpc.skip_nacm_check:
            nrpc = self.nacm.get_user_rules(rpc.username)
            if nrpc.check_data_node_permission(root, node_ii, Permission.NACM_ACCESS_EXEC) == Action.DENY:
                raise NacmForbiddenError(
                    "Invocation of \"{}\" operation denied for user \"{}\"".format(rpc.op_name, rpc.username)
                )

        ii_an = ii[-1]
        node_sn = n.schema_node
        sn = node_sn.get_child(ii_an.name, ii_an.namespace)

        action_handler = self.handlers.action.get_handler(id(sn))
        if action_handler is None:
            raise NoHandlerForOpError(rpc.path)

        # Get operation input schema
        sn_input = sn.get_child("input")

        # Input arguments are expected, this will validate them
        op_input_args = sn_input.from_raw(rpc.op_input_args) if sn_input.children else None

        try:
            ret_data = action_handler(ii, op_input_args, rpc.username)
        except Exception as e:
            raise OpHandlerFailedError(epretty(e))
696 697 698

        return ret_data

699
    def add_to_journal_rpc(self, ch_type: ChangeType, rpc: RpcInfo, value: Optional[JsonNodeT], new_root: InstanceNode, nacm_modified: bool):
Pavel Spirek's avatar
Pavel Spirek committed
700 701
        usr_journal = self._usr_journals.get(rpc.username)
        if usr_journal is not None:
702
            usr_journal.add(DataChange(ch_type, rpc, value, new_root, nacm_modified))
703 704 705
        else:
            raise NoHandlerError("No active changelist for user \"{}\"".format(rpc.username))

706
    # Lock datastore data
707 708
    def lock_data(self, username: str = None, blocking: bool=True):
        ret = self._data_lock.acquire(blocking=blocking, timeout=1)
Pavel Spirek's avatar
Pavel Spirek committed
709
        if ret:
Pavel Spirek's avatar
Pavel Spirek committed
710
            self._lock_username = username or "(unknown)"
711
            debug_data("Acquired datastore lock for user \"{}\"".format(username))
Pavel Spirek's avatar
Pavel Spirek committed
712
        else:
Pavel Spirek's avatar
Pavel Spirek committed
713
            raise DataLockError(
714
                "Failed to acquire datastore lock for user \"{}\", already locked by \"{}\"".format(
715 716 717
                    username,
                    self._lock_username
                )
Pavel Spirek's avatar
Pavel Spirek committed
718 719
            )

720
    # Unlock datastore data
Pavel Spirek's avatar
Pavel Spirek committed
721 722
    def unlock_data(self):
        self._data_lock.release()
723
        debug_data("Released datastore lock for user \"{}\"".format(self._lock_username))
Pavel Spirek's avatar
Pavel Spirek committed
724 725
        self._lock_username = None

726
    # Load data from persistent storage
727
    def load(self):
Pavel Spirek's avatar
Pavel Spirek committed
728 729
        raise NotImplementedError("Not implemented in base class")

730
    # Save data to persistent storage
731
    def save(self):
Pavel Spirek's avatar
Pavel Spirek committed
732 733
        raise NotImplementedError("Not implemented in base class")

Pavel Spirek's avatar
Pavel Spirek committed
734 735

class JsonDatastore(BaseDatastore):
736 737
    def __init__(self, dm: DataModel, json_file: str, with_nacm: bool=False):
        super().__init__(dm, with_nacm)
738 739 740
        self.json_file = json_file

    def load(self):
Pavel Spirek's avatar
Pavel Spirek committed
741
        self._data = None
742
        with open(self.json_file, "rt") as fp:
Pavel Spirek's avatar
Pavel Spirek committed
743
            self._data = self._dm.from_raw(json.load(fp))
Pavel Spirek's avatar
Pavel Spirek committed
744

745 746 747
        if self.nacm is not None:
            self.nacm.update()

748 749 750
    def save(self):
        with open(self.json_file, "w") as jfd:
            json.dump(self._data.raw_value(), jfd, indent=4)