Commit 42980f58 authored by Pavel Spirek's avatar Pavel Spirek

Code refactor and cleanup, version bump

parent bac0ddff
PROJECT = jetconf
VERSION = 0.2.6
VERSION = 0.3.0
.PHONY = tags deps install-deps test
tags:
......
......@@ -3,22 +3,12 @@ import colorlog
import getopt
import logging
import sys
import signal
from importlib import import_module
from pkg_resources import resource_string, get_distribution
from pkg_resources import get_distribution
from colorlog import error, info
from yaml.parser import ParserError
from yangson.enumerations import ContentType, ValidationScope
from yangson.exceptions import YangsonException
from yangson.schemanode import SchemaError, SemanticError
from yangson.datamodel import DataModel
from . import op_internal
from .rest_server import RestServer
from .config import CONFIG_GLOBAL, CONFIG_NACM, load_config, validate_config, print_config
from .helpers import ErrorHelpers
from . import config, jetconf
from .errors import JetconfInitError
def print_help():
......@@ -29,8 +19,6 @@ def print_help():
def main():
config_file = "config.yaml"
# Check for Python version
if sys.version_info < (3, 5):
print("Jetconf requires Python version 3.5 or higher")
......@@ -40,6 +28,8 @@ def main():
jetconf_version = get_distribution("jetconf").version
# Parse command line arguments
config_file = "config.yaml"
try:
opts, args = getopt.getopt(sys.argv[1:], "c:vh")
except getopt.GetoptError:
......@@ -58,18 +48,21 @@ def main():
sys.exit(0)
# Load configuration
jc_config = config.JcConfig()
config.CFG = jc_config
try:
load_config(config_file)
jc_config.load_file(config_file)
except FileNotFoundError:
print("Configuration file does not exist")
sys.exit(1)
except ParserError as e:
except ValueError as e:
print("Configuration syntax error: " + str(e))
sys.exit(1)
# Validate configuration
try:
validate_config()
jc_config.validate()
except ValueError as e:
print("Error: " + str(e))
sys.exit(1)
......@@ -80,16 +73,16 @@ def main():
"warning": logging.WARNING,
"info": logging.INFO,
"debug": logging.INFO
}.get(CONFIG_GLOBAL["LOG_LEVEL"], logging.INFO)
}.get(jc_config.glob["LOG_LEVEL"], logging.INFO)
logging.root.handlers.clear()
# Daemonize
if CONFIG_GLOBAL["LOGFILE"] not in ("-", "stdout"):
if jc_config.glob["LOGFILE"] not in ("-", "stdout"):
# Setup basic logging
logging.basicConfig(
format="%(asctime)s %(levelname)-8s %(message)s",
level=log_level,
filename=CONFIG_GLOBAL["LOGFILE"]
filename=jc_config.glob["LOGFILE"]
)
# Go to background
......@@ -138,92 +131,30 @@ def main():
info("Jetconf version {}".format(jetconf_version))
# Print configuration
print_config()
# Create pidfile
fl = os.open(CONFIG_GLOBAL["PIDFILE"], os.O_WRONLY + os.O_CREAT, 0o666)
try:
os.lockf(fl, os.F_TLOCK, 0)
os.write(fl, str(os.getpid()).encode())
os.fsync(fl)
except BlockingIOError:
error("Jetconf daemon already running (pidfile exists). Exiting.")
sys.exit(1)
# Set signal handlers
def sig_exit_handler(signum, frame):
os.close(fl)
os.unlink(CONFIG_GLOBAL["PIDFILE"])
info("Exiting.")
sys.exit(0)
jc_config.print()
signal.signal(signal.SIGTERM, sig_exit_handler)
signal.signal(signal.SIGINT, sig_exit_handler)
# Instantiate Jetconf main class
jc = jetconf.Jetconf(jc_config)
jetconf.JC = jc
# Import backend modules
backend_package = CONFIG_GLOBAL["BACKEND_PACKAGE"]
try:
usr_state_data_handlers = import_module(backend_package + ".usr_state_data_handlers")
usr_conf_data_handlers = import_module(backend_package + ".usr_conf_data_handlers")
usr_op_handlers = import_module(backend_package + ".usr_op_handlers")
usr_datastore = import_module(backend_package + ".usr_datastore")
except ImportError as e:
error(ErrorHelpers.epretty(e))
error("Cannot import backend package \"{}\". Exiting.".format(backend_package))
sys.exit(1)
# Load data model
yang_mod_dir = CONFIG_GLOBAL["YANG_LIB_DIR"]
yang_lib_str = resource_string(backend_package, "yang-library-data.json").decode("utf-8")
datamodel = DataModel(yang_lib_str, [yang_mod_dir])
# Datastore init
datastore = usr_datastore.UserDatastore(datamodel, CONFIG_GLOBAL["DATA_JSON_FILE"], with_nacm=CONFIG_NACM["ENABLED"])
try:
datastore.load()
except (FileNotFoundError, YangsonException) as e:
error("Cannot load JSON datastore " + CONFIG_GLOBAL["DATA_JSON_FILE"])
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
# Validate datastore on startup
try:
datastore.get_data_root().validate(ValidationScope.all, ContentType.config)
except (SchemaError, SemanticError) as e:
error("Initial validation of datastore failed")
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
jc.init()
except JetconfInitError as e:
error(str(e))
jc.cleanup()
# Register handlers for configuration data
usr_conf_data_handlers.register_conf_handlers(datastore)
# Exit
info("Exiting (error)")
sys.exit(1)
# Register handlers for state data
usr_state_data_handlers.register_state_handlers(datastore)
# Run Jetconf (this will block until shutdown)
jc.run()
# Register handlers for operations
op_internal.register_op_handlers(datastore)
usr_op_handlers.register_op_handlers(datastore)
jc.cleanup()
# Write datastore content to the backend application (if required)
try:
confh_ros = usr_conf_data_handlers.run_on_startup
except AttributeError:
pass
else:
try:
confh_ros()
except Exception as e:
error("Writing configuration to backend failed")
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
# Create HTTP server
rest_srv = RestServer()
rest_srv.register_api_handlers(datastore)
rest_srv.register_static_handlers()
# Run HTTP server
rest_srv.run()
# Exit
info("Exiting")
sys.exit(0)
if __name__ == "__main__":
......
......@@ -2,23 +2,35 @@ import os
import yaml
from colorlog import info
from yaml.parser import ParserError
_yang_mod_dir_env = os.environ.get("YANG_MODPATH")
# For backward compatibility
CONFIG_GLOBAL = {}
CONFIG_HTTP = {}
CONFIG_NACM = {}
CONFIG = {}
CONFIG_GLOBAL = {
CFG = None # type: JcConfig
class JcConfig:
def __init__(self):
yang_mod_dir_env = os.environ.get("YANG_MODPATH")
glob_def = {
"TIMEZONE": "GMT",
"LOGFILE": "-",
"PIDFILE": "/tmp/jetconf.pid",
"PERSISTENT_CHANGES": True,
"LOG_LEVEL": "info",
"LOG_DBG_MODULES": ["*"],
"YANG_LIB_DIR": _yang_mod_dir_env,
"YANG_LIB_DIR": yang_mod_dir_env,
"DATA_JSON_FILE": "data.json",
"VALIDATE_TRANSACTIONS": True,
"BACKEND_PACKAGE": "jetconf_jukebox"
}
}
CONFIG_HTTP = {
http_def = {
"DOC_ROOT": "doc-root",
"DOC_DEFAULT_NAME": "index.html",
"API_ROOT": "/restconf",
......@@ -32,55 +44,68 @@ CONFIG_HTTP = {
"SERVER_SSL_PRIVKEY": "server.key",
"CA_CERT": "ca.pem",
"DBG_DISABLE_CERTS": False
}
}
CONFIG_NACM = {
nacm_def = {
"ENABLED": True,
"ALLOWED_USERS": []
}
CONFIG_KNOT = {
"SOCKET": "/tmp/knot.sock"
}
CONFIG = {
"GLOBAL": CONFIG_GLOBAL,
"HTTP_SERVER": CONFIG_HTTP,
"NACM": CONFIG_NACM,
"KNOT": CONFIG_KNOT
}
API_ROOT_data = os.path.join(CONFIG_HTTP["API_ROOT"], "data")
API_ROOT_RUNNING_data = os.path.join(CONFIG_HTTP["API_ROOT_RUNNING"], "data")
API_ROOT_ops = os.path.join(CONFIG_HTTP["API_ROOT"], "operations")
API_ROOT_ylv = os.path.join(CONFIG_HTTP["API_ROOT"], "yang-library-version")
}
root_def = {
"GLOBAL": glob_def,
"HTTP_SERVER": http_def,
"NACM": nacm_def
}
def load_config(filename: str) -> bool:
global API_ROOT_data
global API_ROOT_RUNNING_data
global API_ROOT_ops
global API_ROOT_ylv
self.glob = glob_def
self.http = http_def
self.nacm = nacm_def
self.root = root_def
with open(filename) as conf_fd:
# Shortcuts
self.api_root_data = None
self.api_root_running_data = None
self.api_root_ops = None
self.api_root_ylv = None
self._gen_shortcuts()
def _gen_shortcuts(self):
global CONFIG_GLOBAL
global CONFIG_HTTP
global CONFIG_NACM
global CONFIG
CONFIG_GLOBAL.update(self.glob)
CONFIG_HTTP.update(self.http)
CONFIG_NACM.update(self.nacm)
CONFIG.update(self.root)
api_root = self.http["API_ROOT"]
api_root_running = self.http["API_ROOT_RUNNING"]
self.api_root_data = os.path.join(api_root, "data")
self.api_root_running_data = os.path.join(api_root_running, "data")
self.api_root_ops = os.path.join(api_root, "operations")
self.api_root_ylv = os.path.join(api_root, "yang-library-version")
def load_file(self, file_path: str) -> bool:
with open(file_path) as conf_fd:
try:
conf_yaml = yaml.load(conf_fd)
for conf_key in CONFIG.keys():
except ParserError as e:
raise ValueError(str(e))
for conf_key in self.root.keys():
try:
CONFIG[conf_key].update(conf_yaml[conf_key])
self.root[conf_key].update(conf_yaml[conf_key])
except KeyError:
pass
# Shortcuts
API_ROOT_data = os.path.join(CONFIG_HTTP["API_ROOT"], "data")
API_ROOT_RUNNING_data = os.path.join(CONFIG_HTTP["API_ROOT_RUNNING"], "data")
API_ROOT_ops = os.path.join(CONFIG_HTTP["API_ROOT"], "operations")
API_ROOT_ylv = os.path.join(CONFIG_HTTP["API_ROOT"], "yang-library-version")
self._gen_shortcuts()
def validate_config():
if CONFIG_GLOBAL["YANG_LIB_DIR"] is None:
def validate(self):
if self.glob["YANG_LIB_DIR"] is None:
raise ValueError("YANG module directory must be specified (in config file or YANG_MODPATH env variable)")
def print_config():
info("Using config:\n" + yaml.dump(CONFIG, default_flow_style=False))
def print(self):
info("Using config:\n" + yaml.dump(self.root, default_flow_style=False))
......@@ -31,9 +31,9 @@ from yangson.instance import (
ObjectMember
)
from . import config
from .helpers import PathFormat, ErrorHelpers, LogHelpers, DataHelpers, JsonNodeT
from .config import CONFIG, CONFIG_NACM
from .nacm import NacmConfig, Permission, Action, NacmForbiddenError
from .nacm import NacmConfig, Permission, Action
from .handler_list import (
OP_HANDLERS,
STATE_DATA_HANDLES,
......@@ -43,7 +43,17 @@ from .handler_list import (
StateDataContainerHandler,
StateDataListHandler
)
from .errors import JetconfError
from .errors import (
ConfHandlerFailedError,
StagingDataException,
NoHandlerForStateDataError,
NoHandlerForOpError,
InstanceAlreadyPresent,
OpHandlerFailedError,
NoHandlerError,
DataLockError,
NacmForbiddenError
)
epretty = ErrorHelpers.epretty
debug_data = LogHelpers.create_module_dbg_logger(__name__)
......@@ -55,46 +65,6 @@ class ChangeType(Enum):
DELETE = 2
class DataLockError(JetconfError):
pass
class StagingDataException(JetconfError):
pass
class InstanceAlreadyPresent(JetconfError):
pass
class HandlerError(JetconfError):
pass
class NoHandlerError(HandlerError):
pass
class ConfHandlerFailedError(HandlerError):
pass
class OpHandlerFailedError(HandlerError):
pass
class NoHandlerForOpError(NoHandlerError):
def __init__(self, op_name: str):
self.op_name = op_name
def __str__(self):
return "Nonexistent handler for operation \"{}\"".format(self.op_name)
class NoHandlerForStateDataError(NoHandlerError):
pass
class RpcInfo:
def __init__(self):
self.username = None # type: str
......@@ -169,7 +139,7 @@ class UsrChangeJournal:
try:
# Validate syntax and semantics of new data
if CONFIG["GLOBAL"]["VALIDATE_TRANSACTIONS"] is True:
if config.CFG.glob["VALIDATE_TRANSACTIONS"] is True:
nr.validate(ValidationScope.all, ContentType.config)
except (SchemaError, SemanticError) as e:
error("Data validation error:")
......@@ -394,7 +364,7 @@ class BaseDatastore:
if (len(ii) > 0) and (isinstance(ii[0], MemberName)):
# Not getting root
ns_first = ii[0].namespace
if (ns_first == "ietf-netconf-acm") and (rpc.username not in CONFIG_NACM["ALLOWED_USERS"]):
if (ns_first == "ietf-netconf-acm") and (rpc.username not in config.CFG.nacm["ALLOWED_USERS"]):
raise NacmForbiddenError(rpc.username + " not allowed to access NACM data")
elif ns_first == "ietf-yang-library":
root = self._yang_lib_data
......@@ -402,7 +372,7 @@ class BaseDatastore:
else:
# Root node requested
# Remove NACM data if user is not NACM privieged
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]:
if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
try:
root = root.delete_item("ietf-netconf-acm:nacm")
except NonexistentInstance:
......@@ -583,13 +553,13 @@ class BaseDatastore:
ns_first = ii[0].namespace
if ns_first == "ietf-netconf-acm":
nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]:
if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
else:
# Editing root node
if input_member_ns == "ietf-netconf-acm":
nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]:
if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
# Evaluate NACM
......@@ -729,7 +699,7 @@ class BaseDatastore:
ns_first = ii[0].namespace
if ns_first == "ietf-netconf-acm":
nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]:
if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
else:
# Replacing root node
......@@ -737,7 +707,7 @@ class BaseDatastore:
nacm_val = n.value.get("ietf-netconf-acm:nacm")
if nacm_val is not None:
nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]:
if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
# Evaluate NACM
......@@ -763,7 +733,7 @@ class BaseDatastore:
ns_first = ii[0].namespace
if ns_first == "ietf-netconf-acm":
nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]:
if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
else:
# Deleting root node
......@@ -771,7 +741,7 @@ class BaseDatastore:
nacm_val = n.value.get("ietf-netconf-acm:nacm")
if nacm_val is not None:
nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]:
if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
# Evaluate NACM
......
......@@ -3,17 +3,59 @@ from yangson.instance import InstanceRoute, NonexistentInstance
# Base class for all exceptions defined in jetconf
class JetconfError(Exception):
def __init__(self, msg=""):
self.msg = msg
pass
def __str__(self):
return self.msg
# Jetconf errors
class JetconfInitError(JetconfError):
pass
class BackendError(JetconfError):
pass
class DataLockError(JetconfError):
pass
class StagingDataException(JetconfError):
pass
class InstanceAlreadyPresent(JetconfError):
pass
# Handler errors
class HandlerError(JetconfError):
pass
class NoHandlerError(HandlerError):
pass
class ConfHandlerFailedError(HandlerError):
pass
class OpHandlerFailedError(HandlerError):
pass
class NoHandlerForOpError(NoHandlerError):
def __init__(self, op_name: str):
self.op_name = op_name
def __str__(self):
return "Nonexistent handler for operation \"{}\"".format(self.op_name)
class NoHandlerForStateDataError(NoHandlerError):
pass
class StateNonexistentInstance(NonexistentInstance):
def __init__(self, ii: InstanceRoute, text: str) -> None:
self.ii = ii
......@@ -21,3 +63,21 @@ class StateNonexistentInstance(NonexistentInstance):
def __str__(self):
return str(self.ii) + ": " + self.text
# NACM errors
class NacmError(JetconfError):
pass
class NonexistentUserError(NacmError):
pass
class NacmForbiddenError(NacmError):
def __init__(self, msg="Access to data node rejected by NACM", rule=None):
self.msg = msg
self.rule = rule
def __str__(self):
return "{} (rule: {})".format(self.msg, str(self.rule))
from typing import List, Tuple, Callable, Any
from typing import Dict, Callable
from yangson.schemanode import SchemaNode
from yangson.schemadata import SchemaData
......@@ -7,17 +7,15 @@ from yangson.instance import InstanceRoute
from .helpers import JsonNodeT
HandlerSelectorT = Any
# ---------- Conf data base objects ----------
class ConfDataObjectHandler:
# ---------- Base classes for conf data handlers ----------
class ConfDataHandlerBase:
def __init__(self, ds: "BaseDatastore", sch_pth: str):
self.ds = ds
self.schema_path = sch_pth # type: str
self.schema_node = ds.get_schema_node(sch_pth) # type: SchemaNode
class ConfDataObjectHandler(ConfDataHandlerBase):
def create(self, ii: InstanceRoute, ch: "DataChange"):
pass
......@@ -31,12 +29,7 @@ class ConfDataObjectHandler:
return self.__class__.__name__ + ": listening at " + self.schema_path
class ConfDataListHandler:
def __init__(self, ds: "BaseDatastore", sch_pth: str):
self.ds = ds
self.schema_path = sch_pth # type: str
self.schema_node = ds.get_schema_node(sch_pth) # type: SchemaNode
class ConfDataListHandler(ConfDataHandlerBase):
def create_item(self, ii: InstanceRoute, ch: "DataChange"):
pass
......@@ -59,8 +52,7 @@ class ConfDataListHandler:
return self.__class__.__name__ + ": listening at " + self.schema_path
# ---------- State data base objects ----------
# ---------- Base classes for state data handlers ----------
class StateDataHandlerBase:
def __init__(self, datastore: "BaseDatastore", schema_path: str):
self.ds = datastore
......@@ -82,56 +74,29 @@ class StateDataListHandler(StateDataHandlerBase):
pass
# ---------- Handles lists ----------
class BaseHandlerList:
def __init__(self):
self.handlers = [] # type: List[Tuple[HandlerSelectorT, Callable]]
self.default_handler = None # type: Callable
def register(self, identifier: str, handler: Callable):
raise NotImplementedError("Not implemented in base class")
def register_default(self, handler: Callable):
self.default_handler = handler
def get_handler(self, identifier: str) -> Any:
raise NotImplementedError("Not implemented in base class")
class OpHandlerList(BaseHandlerList):
def register(self, handler: Callable, op_name: str):
self.handlers.append((op_name, handler))
def get_handler(self, op_name: str) -> Callable:
for h in self.handlers:
if h[0] == op_name:
return h[1]
return self.default_handler
# ---------- Handler lists ----------
class ConfDataHandlerList:
def __init__(self):
self.handlers = [] # type: List[Tuple[HandlerSelectorT, BaseDataListener]]
self.handlers = {} # type: Dict[int, ConfDataHandlerBase]
self.handlers_pth = {} # type: Dict[str, ConfDataHandlerBase]
def register(self, handler: "BaseDataListener"):
def register(self, handler: ConfDataHandlerBase):
sch_node_id = id(handler.schema_node)
self.handlers.append((sch_node_id, handler))
self.handlers[sch_node_id] = handler
self.handlers_pth[handler.schema_path] = handler
def get_handler(self, sch_node_id: str) -> "BaseDataListener":
for h in self.handlers:
if h[0] == sch_node_id:
return h[1]
def get_handler(self, sch_node_id: int) -> ConfDataHandlerBase:
return self.handlers.get(sch_node_id)
return None
def get_handler_by_pth(self, sch_pth: str) -> ConfDataHandlerBase:
return self.handlers_pth.get(sch_pth)
class StateDataHandlerList:
def __init__(self):
self.handlers = []
def register(self, handler: "StateNodeHandlerBase"):
def register(self, handler: "StateDataHandlerBase"):
saddr = SchemaData.path2route(handler.sch_pth)
self.handlers.append((saddr, handler))
......@@ -151,8 +116,22 @@ class StateDataHandlerList:
return None
# ---------- Handler list globals ----------
class OpHandlerList:
def __init__(self):
self.handlers = {} # type: Dict[str, Callable]
self.default_handler = None # type: Callable
def register(self, handler: Callable, op_name: str):
self.handlers[op_name] = handler
def register_default(self, handler: Callable):
self.default_handler = handler
def get_handler(self, op_name: str) -> Callable:
return self.handlers.get(op_name, self.default_handler)
# ---------- Handler list globals ----------
OP_HANDLERS = OpHandlerList()
STATE_DATA_HANDLES = StateDataHandlerList()
CONF_DATA_HANDLES = ConfDataHandlerList()
......@@ -5,10 +5,11 @@ from enum import Enum
from typing import List, Dict, Union, Any, Iterable
from datetime import datetime
from pytz import timezone
from yangson.instance import InstanceRoute, MemberName, EntryKeys, InstanceNode, ArrayValue, NonexistentInstance
from yangson.schemanode import ListNode, ContainerNode
from .config import CONFIG_GLOBAL, CONFIG_HTTP
from . import config
JsonNodeT = Union[Dict[str, Any], List]
SSLCertT = Dict[str, Any]
...