data.py 30.6 KB
Newer Older
Pavel Spirek's avatar
Pavel Spirek committed
1
import json
2

3
from threading import Lock
4
from colorlog import error, warning as warn, info
5
from typing import List, Any, Dict, Callable, Optional, Tuple
6
from datetime import datetime
7

8
from yangson.datamodel import DataModel
9 10
from yangson.enumerations import ValidationScope
from yangson.schemanode import SchemaNode, ListNode, LeafListNode, InternalNode
Pavel Spirek's avatar
Pavel Spirek committed
11
from yangson.instvalue import ArrayValue, ObjectValue
12 13 14
from yangson.instance import (
    InstanceNode,
    NonexistentInstance,
15
    InstanceValueError,
16 17 18
    MemberName,
    EntryKeys,
    EntryIndex,
19
    InstanceRoute,
20
    ArrayEntry,
21
    RootNode
Pavel Spirek's avatar
Pavel Spirek committed
22
)
23

24
from . import config
25
from .helpers import PathFormat, ErrorHelpers, LogHelpers, DataHelpers, JsonNodeT
26
from .nacm import NacmConfig, Permission, Action
27 28 29
from .journal import ChangeType, UsrChangeJournal, RpcInfo, DataChange
from .handler_base import ConfDataObjectHandler, ConfDataListHandler, StateDataContainerHandler, StateDataListHandler
from .handler_list import ConfDataHandlerList, StateDataHandlerList, OpHandlerList
30 31 32 33 34 35 36 37 38 39
from .errors import (
    StagingDataException,
    NoHandlerForStateDataError,
    NoHandlerForOpError,
    InstanceAlreadyPresent,
    OpHandlerFailedError,
    NoHandlerError,
    DataLockError,
    NacmForbiddenError
)
40 41 42

epretty = ErrorHelpers.epretty
debug_data = LogHelpers.create_module_dbg_logger(__name__)
Pavel Spirek's avatar
Pavel Spirek committed
43 44


45
class BackendHandlers:
Pavel Spirek's avatar
Pavel Spirek committed
46
    def __init__(self):
47 48 49
        self.conf = ConfDataHandlerList()
        self.state = StateDataHandlerList()
        self.op = OpHandlerList()
50

51 52
        def _blankfn(*args, **kwargs):
            pass
53

54 55
        self.commit_begin = _blankfn   # type: Callable[[], None]
        self.commit_end = _blankfn     # type: Callable[[bool], None]
56

57

Pavel Spirek's avatar
Pavel Spirek committed
58
class BaseDatastore:
59
    def __init__(self, dm: DataModel, with_nacm: bool=False):
60
        self._dm = dm       # type: DataModel
61
        self._data = None   # type: InstanceNode
62
        self._yang_lib_data = self._dm.from_raw(self._dm.yang_library)  # type: InstanceNode
63
        self._data_history = []     # type: List[InstanceNode]
Pavel Spirek's avatar
Pavel Spirek committed
64
        self._data_lock = Lock()
Pavel Spirek's avatar
Pavel Spirek committed
65
        self._lock_username = None  # type: str
66
        self._usr_journals = {}     # type: Dict[str, UsrChangeJournal]
67 68 69
        self.nacm = None    # type: NacmConfig
        self.handlers = BackendHandlers()
        self.nacm = NacmConfig(self, self._dm) if with_nacm else None
70

71 72 73 74
    # Returns DataModel object
    def get_dm(self) -> DataModel:
        return self._dm

Pavel Spirek's avatar
Pavel Spirek committed
75
    # Returns the root node of data tree
76 77 78 79 80
    def get_data_root(self, previous_version: int=0) -> InstanceNode:
        if previous_version > 0:
            return self._data_history[-previous_version]
        else:
            return self._data
Pavel Spirek's avatar
Pavel Spirek committed
81

82
    # Returns the root node of YANG library data tree
83 84 85
    def get_yl_data_root(self) -> InstanceNode:
        return self._yang_lib_data

86
    # Journal manipulation methods
Pavel Spirek's avatar
Pavel Spirek committed
87
    def make_user_journal(self, username: str):
88 89
        usr_journal = self._usr_journals.get(username)
        if usr_journal is not None:
Pavel Spirek's avatar
Pavel Spirek committed
90
            raise StagingDataException("Transaction for user \"{}\" already opened".format(username))
91
        else:
92
            self._usr_journals[username] = UsrChangeJournal(self._data)
93 94 95 96 97 98

    def get_user_journal(self, username: str):
        usr_journal = self._usr_journals.get(username)
        if usr_journal is not None:
            return usr_journal
        else:
Pavel Spirek's avatar
Pavel Spirek committed
99
            raise StagingDataException("Transaction for user \"{}\" not opened".format(username))
100 101

    def drop_user_journal(self, username: str):
102 103
        usr_journal = self._usr_journals.get(username)
        if usr_journal is not None:
104
            del self._usr_journals[username]
105
        else:
Pavel Spirek's avatar
Pavel Spirek committed
106
            raise StagingDataException("Transaction for user \"{}\" not opened".format(username))
107

108
    # Returns the root node of staging data tree (starts a new transaction if nonexistent)
109
    def get_data_root_staging(self, username: str) -> InstanceNode:
110 111 112 113
        try:
            usr_journal = self.get_user_journal(username)
        except StagingDataException:
            info("Starting new transaction for user \"{}\"".format(username))
Pavel Spirek's avatar
Pavel Spirek committed
114
            self.make_user_journal(username)
115 116
            usr_journal = self.get_user_journal(username)

117 118 119
        root = usr_journal.get_root_head()
        return root

120
    # Set a new Instance node as data root, store old root to archive
Pavel Spirek's avatar
Pavel Spirek committed
121
    def set_data_root(self, new_root: InstanceNode):
122
        self._data_history.append(self._data)
Pavel Spirek's avatar
Pavel Spirek committed
123 124
        self._data = new_root

125 126 127 128 129 130
    def data_root_rollback(self, history_steps: int, store_current: bool):
        if store_current:
            self._data_history.append(self._data)

        self._data = self._data_history[-history_steps]

131 132 133 134 135 136 137 138
    def parse_ii(self, path: str, path_format: PathFormat) -> InstanceRoute:
        if path_format == PathFormat.URL:
            ii = self._dm.parse_resource_id(path)
        else:
            ii = self._dm.parse_instance_id(path)

        return ii

139 140
    # Get schema node with particular schema address
    def get_schema_node(self, sch_pth: str) -> SchemaNode:
141
        sn = self._dm.get_data_node(sch_pth)
142
        if sn is None:
143 144
            # raise NonexistentSchemaNode(sch_pth)
            debug_data("Cannot find schema node for " + sch_pth)
145 146
        return sn

147
    # Run configuration data handlers
148
    def run_conf_edit_handler(self, ii: InstanceRoute, ch: DataChange):
Pavel Spirek's avatar
Pavel Spirek committed
149 150 151 152 153 154
        sch_pth_list = list(filter(lambda n: isinstance(n, MemberName), ii))

        if ch.change_type == ChangeType.CREATE:
            # Get target member name
            input_member_name_fq = tuple(ch.input_data.keys())[0]
            input_member_name_ns, input_member_name = input_member_name_fq.split(":", maxsplit=1)
155

Pavel Spirek's avatar
Pavel Spirek committed
156
            # Append it to ii
157 158 159 160
            if (len(sch_pth_list) == 0) or (sch_pth_list[-1].namespace != input_member_name_ns):
                sch_pth_list.append(MemberName(input_member_name, input_member_name_ns))
            else:
                sch_pth_list.append(MemberName(input_member_name, None))
Pavel Spirek's avatar
Pavel Spirek committed
161 162 163 164 165 166 167

        sch_pth = DataHelpers.ii2str(sch_pth_list)
        sn = self.get_schema_node(sch_pth)

        if sn is None:
            return

168
        h = self.handlers.conf.get_handler(id(sn))
Pavel Spirek's avatar
Pavel Spirek committed
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
        if h is not None:
            info("handler for actual data node triggered")
            if isinstance(h, ConfDataObjectHandler):
                if ch.change_type == ChangeType.CREATE:
                    h.create(ii, ch)
                elif ch.change_type == ChangeType.REPLACE:
                    h.replace(ii, ch)
                elif ch.change_type == ChangeType.DELETE:
                    h.delete(ii, ch)
            if isinstance(h, ConfDataListHandler):
                if ch.change_type == ChangeType.CREATE:
                    h.create_item(ii, ch)
                elif ch.change_type == ChangeType.REPLACE:
                    h.replace_item(ii, ch)
                elif ch.change_type == ChangeType.DELETE:
                    h.delete_item(ii, ch)
        else:
            sn = sn.parent
            while sn is not None:
188
                h = self.handlers.conf.get_handler(id(sn))
Pavel Spirek's avatar
Pavel Spirek committed
189 190 191 192 193 194 195 196
                if h is not None and isinstance(h, ConfDataObjectHandler):
                    info("handler for superior data node triggered, replace")
                    # print(h.schema_path)
                    # print(h.__class__.__name__)
                    h.replace(ii, ch)
                if h is not None and isinstance(h, ConfDataListHandler):
                    info("handler for superior data node triggered, replace_item")
                    h.replace_item(ii, ch)
197
                sn = sn.parent
198

Pavel Spirek's avatar
Pavel Spirek committed
199
    # Get data node, evaluate NACM if required
Pavel Spirek's avatar
Pavel Spirek committed
200 201
    def get_node_rpc(self, rpc: RpcInfo, staging=False) -> InstanceNode:
        ii = self.parse_ii(rpc.path, rpc.path_format)
202

Pavel Spirek's avatar
Pavel Spirek committed
203
        if staging:
204
            root = self.get_data_root_staging(rpc.username)
205 206
        else:
            root = self._data
207

Pavel Spirek's avatar
Pavel Spirek committed
208
        yl_data_request = False
Pavel Spirek's avatar
Pavel Spirek committed
209 210 211
        if (len(ii) > 0) and (isinstance(ii[0], MemberName)):
            # Not getting root
            ns_first = ii[0].namespace
212
            if (ns_first == "ietf-netconf-acm") and (rpc.username not in config.CFG.nacm["ALLOWED_USERS"]):
Pavel Spirek's avatar
Pavel Spirek committed
213 214 215
                raise NacmForbiddenError(rpc.username + " not allowed to access NACM data")
            elif ns_first == "ietf-yang-library":
                root = self._yang_lib_data
Pavel Spirek's avatar
Pavel Spirek committed
216
                yl_data_request = True
Pavel Spirek's avatar
Pavel Spirek committed
217 218 219
        else:
            # Root node requested
            # Remove NACM data if user is not NACM privieged
220
            if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
Pavel Spirek's avatar
Pavel Spirek committed
221 222 223 224 225 226 227
                try:
                    root = root.delete_item("ietf-netconf-acm:nacm")
                except NonexistentInstance:
                    pass

            # Append YANG library data
            for member_name, member_val in self._yang_lib_data.value.items():
228
                root = root.put_member(member_name, member_val).top()
Pavel Spirek's avatar
Pavel Spirek committed
229

230
        # Resolve schema node of the desired data node
231
        sch_pth_list = filter(lambda isel: isinstance(isel, MemberName), ii)
232
        sch_pth = DataHelpers.ii2str(sch_pth_list)
233
        sn = self.get_schema_node(sch_pth)
234

235 236
        state_roots = sn.state_roots()

Pavel Spirek's avatar
Pavel Spirek committed
237
        # Check if URL points to state data or node that contains state data
Pavel Spirek's avatar
Pavel Spirek committed
238
        if state_roots and not yl_data_request:
239
            debug_data("State roots: {}".format(state_roots))
240
            n = None
241

242 243 244
            for state_root_sch_pth in state_roots:
                state_root_sn = self._dm.get_data_node(state_root_sch_pth)

Pavel Spirek's avatar
Pavel Spirek committed
245
                # Check if the desired node is child of the state root
246 247 248
                sni = sn
                is_child = False
                while sni:
249
                    if sni is state_root_sn:
250 251 252 253 254 255
                        is_child = True
                        break
                    sni = sni.parent

                if is_child:
                    # Direct request for the state data
256
                    sdh = self.handlers.state.get_handler(state_root_sch_pth)
257
                    if sdh is not None:
258
                        if isinstance(sdh, StateDataContainerHandler):
259
                            state_handler_val = sdh.generate_node(ii, rpc.username, staging)
260
                            state_root_n = sdh.schema_node.orphan_instance(state_handler_val)
261
                        elif isinstance(sdh, StateDataListHandler):
262 263 264 265 266 267
                            if (sn is sdh.schema_node) and isinstance(ii[-1], MemberName):
                                state_handler_val = sdh.generate_list(ii, rpc.username, staging)
                                state_root_n = sdh.schema_node.orphan_instance(state_handler_val)
                            else:
                                state_handler_val = sdh.generate_item(ii, rpc.username, staging)
                                state_root_n = sdh.schema_node.orphan_entry(state_handler_val)
268 269
                        else:
                            state_root_n = None
270 271 272 273

                        # Select desired subnode from handler-generated content
                        ii_prefix, ii_rel = sdh.schema_node.split_instance_route(ii)
                        n = state_root_n.goto(ii_rel)
274 275 276 277 278

                        # There should be only one state root, no need to continue
                        if len(state_roots) != 1:
                            warn("URI points to directly to state data, but more state roots found")
                        break
279 280
                    else:
                        raise NoHandlerForStateDataError(rpc.path)
281
                else:
282
                    # Request for config data containing state data
283 284 285 286 287 288
                    n = root.goto(ii)

                    def _fill_state_roots(node: InstanceNode) -> InstanceNode:
                        if isinstance(node.value, ObjectValue):
                            if node.schema_node is state_root_sn.parent:
                                ii_gen = DataHelpers.node_get_ii(node)
289
                                _sdh = self.handlers.state.get_handler(state_root_sch_pth)
290
                                if _sdh is not None:
291
                                    try:
292 293 294 295 296 297
                                        if isinstance(_sdh, StateDataContainerHandler):
                                            _state_handler_val = _sdh.generate_node(ii_gen, rpc.username, staging)
                                        elif isinstance(_sdh, StateDataListHandler):
                                            _state_handler_val = _sdh.generate_list(ii_gen, rpc.username, staging)
                                        else:
                                            _state_handler_val = None
298 299 300 301 302 303 304 305 306 307 308
                                    except Exception as e:
                                        error("Error occured in state data generator (sn: {})".format(state_root_sch_pth))
                                        error(epretty(e))
                                        error("This state node will be omitted.")
                                    else:
                                        if state_root_sn.ns == state_root_sn.parent.ns:
                                            nm_name = state_root_sn.qual_name[0]
                                        else:
                                            nm_name = state_root_sn.qual_name[1] + ":" + state_root_sn.qual_name[0]

                                        # print("nm={}".format(nm_name))
309
                                        node = node.put_member(nm_name, _state_handler_val, raw=True).up()
310 311 312 313 314 315 316 317 318 319 320 321 322 323
                            else:
                                for key in node:
                                    member = node[key]
                                    node = _fill_state_roots(member).up()
                        elif isinstance(node.value, ArrayValue):
                            i = 0
                            arr_len = len(node.value)
                            while i < arr_len:
                                node = _fill_state_roots(node[i]).up()
                                i += 1

                        return node

                    n = _fill_state_roots(n)
324
                    root = n.top()
325
        else:
Pavel Spirek's avatar
Pavel Spirek committed
326
            # No state data in requested node
327
            n = root.goto(ii)
Pavel Spirek's avatar
Pavel Spirek committed
328

Pavel Spirek's avatar
Pavel Spirek committed
329
        # Process "with-defaults" query parameter
330 331
        try:
            with_defs = rpc.qs["with-defaults"][0]
332
        except (IndexError, KeyError):
333 334 335 336 337
            with_defs = None

        if with_defs == "report-all":
            n = n.add_defaults()

Pavel Spirek's avatar
Pavel Spirek committed
338
        # Evaluate NACM if required
339
        if self.nacm and not rpc.skip_nacm_check:
Pavel Spirek's avatar
Pavel Spirek committed
340
            nrpc = self.nacm.get_user_rules(rpc.username)
341
            if nrpc.check_data_node_permission(root, ii, Permission.NACM_ACCESS_READ) == Action.DENY:
Pavel Spirek's avatar
Pavel Spirek committed
342 343
                raise NacmForbiddenError()
            else:
344 345
                # Prune nodes that should not be accessible to user
                n = nrpc.prune_data_tree(n, root, ii, Permission.NACM_ACCESS_READ)
Pavel Spirek's avatar
Pavel Spirek committed
346

Pavel Spirek's avatar
Pavel Spirek committed
347
        # Process "depth" query parameter
348
        try:
349 350 351 352 353 354 355
            max_depth_str = rpc.qs["depth"][0]
            if max_depth_str == "unbounded":
                max_depth = None
            else:
                max_depth = int(max_depth_str) - 1
                if (max_depth < 0) or (max_depth > 65535):
                    raise ValueError()
356
        except (IndexError, KeyError):
357 358 359 360 361 362 363 364 365 366 367
            max_depth = None
        except ValueError:
            raise ValueError("Invalid value of query param \"depth\"")

        if max_depth is not None:
            def _tree_limit_depth(node: InstanceNode, depth: int) -> InstanceNode:
                if isinstance(node.value, ObjectValue):
                    if depth > max_depth:
                        node.value = ObjectValue({})
                    else:
                        for child_key in sorted(node.value.keys()):
368
                            m = node[child_key]
369 370
                            node = _tree_limit_depth(m, depth + 1).up()
                elif isinstance(node.value, ArrayValue):
371 372 373 374
                    depth -= 1
                    for i in range(len(node.value)):
                        e = node[i]
                        node = _tree_limit_depth(e, depth + 1).up()
375 376 377 378

                return node
            n = _tree_limit_depth(n, 1)

Pavel Spirek's avatar
Pavel Spirek committed
379
        # Return result
Pavel Spirek's avatar
Pavel Spirek committed
380 381
        return n

382
    # Create new data node
383
    def create_node_rpc(self, root: InstanceNode, rpc: RpcInfo, value: Any) -> Tuple[InstanceNode, bool]:
384
        ii = self.parse_ii(rpc.path, rpc.path_format)
385

386
        # Get target member name
387 388
        input_member_keys = tuple(value.keys())
        if len(input_member_keys) != 1:
389
            raise ValueError("Received json object must contain exactly one member")
Pavel Spirek's avatar
Pavel Spirek committed
390

391 392 393 394 395
        input_member_name_fq = input_member_keys[0]
        try:
            input_member_ns, input_member_name = input_member_name_fq.split(":", maxsplit=1)
        except ValueError:
            raise ValueError("Input object name must me in fully-qualified format")
396
        input_member_value = value[input_member_name_fq]
Pavel Spirek's avatar
Pavel Spirek committed
397

398
        # Deny any changes of NACM data for non-privileged users
399
        nacm_changed = False
400 401 402
        if (len(ii) > 0) and (isinstance(ii[0], MemberName)):
            # Not getting root
            ns_first = ii[0].namespace
403 404
            if ns_first == "ietf-netconf-acm":
                nacm_changed = True
405
                if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
406
                    raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
407 408
        else:
            # Editing root node
409 410
            if input_member_ns == "ietf-netconf-acm":
                nacm_changed = True
411
                if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
412
                    raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
413 414

        # Evaluate NACM
415
        if self.nacm and not rpc.skip_nacm_check:
416 417 418
            nrpc = self.nacm.get_user_rules(rpc.username)
            if nrpc.check_data_node_permission(root, ii, Permission.NACM_ACCESS_CREATE) == Action.DENY:
                raise NacmForbiddenError()
419

420 421
        n = root.goto(ii)

422
        # Get target schema node
423
        sn = n.schema_node  # type: InternalNode
424
        member_sn = sn.get_child(input_member_name, input_member_ns)
425

426 427 428
        if member_sn is None:
            raise ValueError("Received json object contains unknown member")

429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444
        # Check if target member already exists
        if sn.ns == member_sn.ns:
            try:
                existing_member = n[input_member_name]
            except NonexistentInstance:
                existing_member = None
        else:
            try:
                existing_member = n[input_member_name_fq]
            except NonexistentInstance:
                existing_member = None

        # Get query parameters
        insert = rpc.qs.get("insert", [None])[0]
        point = rpc.qs.get("point", [None])[0]

445
        if isinstance(member_sn, ListNode):
Pavel Spirek's avatar
Pavel Spirek committed
446
            # Append received node to list
447

448 449
            # Create list if necessary
            if existing_member is None:
450 451
                new_member_name = input_member_name if n.namespace == input_member_ns else input_member_name_fq
                existing_member = n.put_member(new_member_name, ArrayValue([]))
452

453 454
            # Get ListNode key names
            list_node_keys = member_sn.keys     # Key names in the form [(key, ns), ]
455 456

            if insert == "first":
457
                # Optimization
458
                if len(existing_member.value) > 0:
459
                    list_entry_first = existing_member[0]   # type: ArrayEntry
460
                    new_member = list_entry_first.insert_before(input_member_value, raw=True).up()
461
                else:
462
                    new_member = existing_member.update([input_member_value], raw=True)
463
            elif (insert == "last") or (insert is None):
464
                # Optimization
465
                if len(existing_member.value) > 0:
466
                    list_entry_last = existing_member[-1]   # type: ArrayEntry
467
                    new_member = list_entry_last.insert_after(input_member_value, raw=True).up()
468
                else:
469
                    new_member = existing_member.update([input_member_value], raw=True)
470 471 472 473 474 475 476 477 478 479 480
            elif (insert == "before") and (point is not None):
                point_keys_val = point.split(",")  # List key values passed in the "point" query argument
                if len(list_node_keys) != len(point_keys_val):
                    raise ValueError(
                        "Invalid number of keys passed in 'point' query: {} ({} expected)".format(
                            len(point_keys_val), len(list_node_keys)
                        )
                    )
                entry_keys = dict(map(lambda i: (list_node_keys[i], point_keys_val[i]), range(len(list_node_keys))))
                entry_sel = EntryKeys(entry_keys)
                point_list_entry = entry_sel.goto_step(existing_member)   # type: ArrayEntry
481
                new_member = point_list_entry.insert_before(input_member_value, raw=True).up()
482 483 484 485 486 487 488 489 490 491 492
            elif (insert == "after") and (point is not None):
                point_keys_val = point.split(",")  # List key values passed in the "point" query argument
                if len(list_node_keys) != len(point_keys_val):
                    raise ValueError(
                        "Invalid number of keys passed in 'point' query: {} ({} expected)".format(
                            len(point_keys_val), len(list_node_keys)
                        )
                    )
                entry_keys = dict(map(lambda i: (list_node_keys[i], point_keys_val[i]), range(len(list_node_keys))))
                entry_sel = EntryKeys(entry_keys)
                point_list_entry = entry_sel.goto_step(existing_member)   # type: ArrayEntry
493
                new_member = point_list_entry.insert_after(input_member_value, raw=True).up()
494
            else:
495
                raise ValueError("Invalid 'insert'/'point' query values")
496 497 498 499 500
        elif isinstance(member_sn, LeafListNode):
            # Append received node to leaf list

            # Create leaf list if necessary
            if existing_member is None:
501 502
                new_member_name = input_member_name if n.namespace == input_member_ns else input_member_name_fq
                existing_member = n.put_member(new_member_name, ArrayValue([]))
503

504
            # Convert input data from List/Dict to ArrayValue/ObjectValue
505
            new_value_item = member_sn.entry_from_raw(input_member_value)
506 507

            if insert == "first":
508
                new_member = existing_member.update(ArrayValue([new_value_item] + existing_member.value))
509
            elif (insert == "last") or (insert is None):
510
                new_member = existing_member.update(ArrayValue(existing_member.value + [new_value_item]))
511
            else:
512
                raise ValueError("Invalid 'insert' query value")
Pavel Spirek's avatar
Pavel Spirek committed
513
        else:
514
            # Create new container member
515

516
            if existing_member is None:
517
                # Create new node (object member)
518 519
                new_member_name = input_member_name if n.namespace == input_member_ns else input_member_name_fq
                new_member = n.put_member(new_member_name, input_member_value, raw=True)
520 521 522
            else:
                # Data node already exists
                raise InstanceAlreadyPresent("Member \"{}\" already present in \"{}\"".format(input_member_name, ii))
Pavel Spirek's avatar
Pavel Spirek committed
523

524
        return new_member.top(), nacm_changed
525

526
    # PUT data node
527
    def update_node_rpc(self, root: InstanceNode, rpc: RpcInfo, value: Any) -> Tuple[InstanceNode, bool]:
528
        ii = self.parse_ii(rpc.path, rpc.path_format)
529 530 531 532 533 534 535 536 537 538 539 540 541

        # Get target member name
        input_member_keys = tuple(value.keys())
        if len(input_member_keys) != 1:
            raise ValueError("Received json object must contain exactly one member")

        input_member_name_fq = input_member_keys[0]
        try:
            input_member_ns, input_member_name = input_member_name_fq.split(":", maxsplit=1)
        except ValueError:
            raise ValueError("Input object name must me in fully-qualified format")
        input_member_value = value[input_member_name_fq]

542
        n = root.goto(ii)
543

544
        # Deny any changes of NACM data for non-privileged users
545
        nacm_changed = False
546 547 548
        if (len(ii) > 0) and (isinstance(ii[0], MemberName)):
            # Not getting root
            ns_first = ii[0].namespace
549 550
            if ns_first == "ietf-netconf-acm":
                nacm_changed = True
551
                if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
552
                    raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
553 554 555 556
        else:
            # Replacing root node
            # Check if NACM data are present in the datastore
            nacm_val = n.value.get("ietf-netconf-acm:nacm")
557 558
            if nacm_val is not None:
                nacm_changed = True
559
                if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
560
                    raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
561 562

        # Evaluate NACM
563
        if self.nacm and not rpc.skip_nacm_check:
Pavel Spirek's avatar
Pavel Spirek committed
564
            nrpc = self.nacm.get_user_rules(rpc.username)
565
            if nrpc.check_data_node_permission(root, ii, Permission.NACM_ACCESS_UPDATE) == Action.DENY:
566
                raise NacmForbiddenError()
Pavel Spirek's avatar
Pavel Spirek committed
567

568
        new_n = n.update(input_member_value, raw=True)
569
        new_n.validate(ValidationScope.syntax)
570

571
        return new_n.top(), nacm_changed
572

573
    # Delete data node
574
    def delete_node_rpc(self, root: InstanceNode, rpc: RpcInfo) -> Tuple[InstanceNode, bool]:
575
        ii = self.parse_ii(rpc.path, rpc.path_format)
576
        n = root.goto(ii)
Pavel Spirek's avatar
Pavel Spirek committed
577

578
        # Deny any changes of NACM data for non-privileged users
579
        nacm_changed = False
580 581 582
        if (len(ii) > 0) and (isinstance(ii[0], MemberName)):
            # Not getting root
            ns_first = ii[0].namespace
583 584
            if ns_first == "ietf-netconf-acm":
                nacm_changed = True
585
                if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
586
                    raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
587 588 589 590
        else:
            # Deleting root node
            # Check if NACM data are present in the datastore
            nacm_val = n.value.get("ietf-netconf-acm:nacm")
591 592
            if nacm_val is not None:
                nacm_changed = True
593
                if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
594
                    raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
595 596

        # Evaluate NACM
597
        if self.nacm and not rpc.skip_nacm_check:
Pavel Spirek's avatar
Pavel Spirek committed
598
            nrpc = self.nacm.get_user_rules(rpc.username)
599
            if nrpc.check_data_node_permission(root, ii, Permission.NACM_ACCESS_DELETE) == Action.DENY:
600
                raise NacmForbiddenError()
Pavel Spirek's avatar
Pavel Spirek committed
601

602 603 604
        if len(ii) == 0:
            # Deleting entire datastore
            new_n = RootNode(ObjectValue({}), root.schema_node, datetime.now())
605
        else:
606 607 608 609 610 611 612 613 614 615 616 617 618
            n_parent = n.up()
            last_isel = ii[-1]
            if isinstance(n_parent.value, ArrayValue):
                if isinstance(last_isel, EntryIndex):
                    new_n = n_parent.delete_item(last_isel.index)
                elif isinstance(last_isel, EntryKeys):
                    new_n = n_parent.delete_item(n.index)
                else:
                    raise ValueError("Unknown node selector")
            elif isinstance(n_parent.value, ObjectValue):
                new_n = n_parent.delete_item(last_isel.namespace + ":" + last_isel.name if last_isel.namespace else last_isel.name)
            else:
                raise InstanceValueError(rpc.path, "Invalid target node type")
Pavel Spirek's avatar
Pavel Spirek committed
619

620
        return new_n.top(), nacm_changed
Pavel Spirek's avatar
Pavel Spirek committed
621

622
    # Invoke an operation
623
    def invoke_op_rpc(self, rpc: RpcInfo) -> JsonNodeT:
624 625
        if rpc.op_name.startswith("jetconf:"):
            # Jetconf internal operation
626
            op_handler = self.handlers.op.get_handler(rpc.op_name)
627 628
            if op_handler is None:
                raise NoHandlerForOpError(rpc.op_name)
629

630
            ret_data = op_handler(rpc)
631
        else:
632
            # External operation defined in data model
633
            if self.nacm and not rpc.skip_nacm_check:
Pavel Spirek's avatar
Pavel Spirek committed
634
                nrpc = self.nacm.get_user_rules(rpc.username)
635 636
                if nrpc.check_rpc_name(rpc.op_name) == Action.DENY:
                    raise NacmForbiddenError(
637
                        "Invocation of \"{}\" operation denied for user \"{}\"".format(rpc.op_name, rpc.username)
638 639
                    )

640
            op_handler = self.handlers.op.get_handler(rpc.op_name)
641
            if op_handler is None:
642
                raise NoHandlerForOpError(rpc.op_name)
643

644
            # Get operation input schema
645 646
            sn = self._dm.get_schema_node(rpc.path)
            sn_input = sn.get_child("input")
647

648 649
            # Input arguments are expected, this will validate them
            op_input_args = sn_input.from_raw(rpc.op_input_args) if sn_input.children else None
650

651 652 653 654
            try:
                ret_data = op_handler(op_input_args, rpc.username)
            except Exception as e:
                raise OpHandlerFailedError(epretty(e))
655 656 657

        return ret_data

658
    def add_to_journal_rpc(self, ch_type: ChangeType, rpc: RpcInfo, value: Optional[JsonNodeT], new_root: InstanceNode, nacm_modified: bool):
Pavel Spirek's avatar
Pavel Spirek committed
659 660
        usr_journal = self._usr_journals.get(rpc.username)
        if usr_journal is not None:
661
            usr_journal.add(DataChange(ch_type, rpc, value, new_root, nacm_modified))
662 663 664
        else:
            raise NoHandlerError("No active changelist for user \"{}\"".format(rpc.username))

665
    # Lock datastore data
666 667
    def lock_data(self, username: str = None, blocking: bool=True):
        ret = self._data_lock.acquire(blocking=blocking, timeout=1)
Pavel Spirek's avatar
Pavel Spirek committed
668
        if ret:
Pavel Spirek's avatar
Pavel Spirek committed
669
            self._lock_username = username or "(unknown)"
670
            debug_data("Acquired datastore lock for user \"{}\"".format(username))
Pavel Spirek's avatar
Pavel Spirek committed
671
        else:
Pavel Spirek's avatar
Pavel Spirek committed
672
            raise DataLockError(
673
                "Failed to acquire datastore lock for user \"{}\", already locked by \"{}\"".format(
674 675 676
                    username,
                    self._lock_username
                )
Pavel Spirek's avatar
Pavel Spirek committed
677 678
            )

679
    # Unlock datastore data
Pavel Spirek's avatar
Pavel Spirek committed
680 681
    def unlock_data(self):
        self._data_lock.release()
682
        debug_data("Released datastore lock for user \"{}\"".format(self._lock_username))
Pavel Spirek's avatar
Pavel Spirek committed
683 684
        self._lock_username = None

685
    # Load data from persistent storage
686
    def load(self):
Pavel Spirek's avatar
Pavel Spirek committed
687 688
        raise NotImplementedError("Not implemented in base class")

689
    # Save data to persistent storage
690
    def save(self):
Pavel Spirek's avatar
Pavel Spirek committed
691 692
        raise NotImplementedError("Not implemented in base class")

Pavel Spirek's avatar
Pavel Spirek committed
693 694

class JsonDatastore(BaseDatastore):
695 696
    def __init__(self, dm: DataModel, json_file: str, with_nacm: bool=False):
        super().__init__(dm, with_nacm)
697 698 699
        self.json_file = json_file

    def load(self):
Pavel Spirek's avatar
Pavel Spirek committed
700
        self._data = None
701
        with open(self.json_file, "rt") as fp:
Pavel Spirek's avatar
Pavel Spirek committed
702
            self._data = self._dm.from_raw(json.load(fp))
Pavel Spirek's avatar
Pavel Spirek committed
703

704 705 706
        if self.nacm is not None:
            self.nacm.update()

707 708 709
    def save(self):
        with open(self.json_file, "w") as jfd:
            json.dump(self._data.raw_value(), jfd, indent=4)