Commit 42980f58 authored by Pavel Spirek's avatar Pavel Spirek

Code refactor and cleanup, version bump

parent bac0ddff
PROJECT = jetconf
VERSION = 0.2.6
VERSION = 0.3.0
.PHONY = tags deps install-deps test
tags:
......
......@@ -3,22 +3,12 @@ import colorlog
import getopt
import logging
import sys
import signal
from importlib import import_module
from pkg_resources import resource_string, get_distribution
from pkg_resources import get_distribution
from colorlog import error, info
from yaml.parser import ParserError
from yangson.enumerations import ContentType, ValidationScope
from yangson.exceptions import YangsonException
from yangson.schemanode import SchemaError, SemanticError
from yangson.datamodel import DataModel
from . import op_internal
from .rest_server import RestServer
from .config import CONFIG_GLOBAL, CONFIG_NACM, load_config, validate_config, print_config
from .helpers import ErrorHelpers
from . import config, jetconf
from .errors import JetconfInitError
def print_help():
......@@ -29,8 +19,6 @@ def print_help():
def main():
config_file = "config.yaml"
# Check for Python version
if sys.version_info < (3, 5):
print("Jetconf requires Python version 3.5 or higher")
......@@ -40,6 +28,8 @@ def main():
jetconf_version = get_distribution("jetconf").version
# Parse command line arguments
config_file = "config.yaml"
try:
opts, args = getopt.getopt(sys.argv[1:], "c:vh")
except getopt.GetoptError:
......@@ -58,38 +48,41 @@ def main():
sys.exit(0)
# Load configuration
jc_config = config.JcConfig()
config.CFG = jc_config
try:
load_config(config_file)
jc_config.load_file(config_file)
except FileNotFoundError:
print("Configuration file does not exist")
sys.exit(1)
except ParserError as e:
except ValueError as e:
print("Configuration syntax error: " + str(e))
sys.exit(1)
# Validate configuration
try:
validate_config()
jc_config.validate()
except ValueError as e:
print("Error: " + str(e))
sys.exit(1)
# Set logging level
log_level = {
"error": logging.ERROR,
"warning": logging.WARNING,
"info": logging.INFO,
"debug": logging.INFO
}.get(CONFIG_GLOBAL["LOG_LEVEL"], logging.INFO)
}.get(jc_config.glob["LOG_LEVEL"], logging.INFO)
logging.root.handlers.clear()
# Daemonize
if CONFIG_GLOBAL["LOGFILE"] not in ("-", "stdout"):
if jc_config.glob["LOGFILE"] not in ("-", "stdout"):
# Setup basic logging
logging.basicConfig(
format="%(asctime)s %(levelname)-8s %(message)s",
level=log_level,
filename=CONFIG_GLOBAL["LOGFILE"]
filename=jc_config.glob["LOGFILE"]
)
# Go to background
......@@ -138,92 +131,30 @@ def main():
info("Jetconf version {}".format(jetconf_version))
# Print configuration
print_config()
# Create pidfile
fl = os.open(CONFIG_GLOBAL["PIDFILE"], os.O_WRONLY + os.O_CREAT, 0o666)
try:
os.lockf(fl, os.F_TLOCK, 0)
os.write(fl, str(os.getpid()).encode())
os.fsync(fl)
except BlockingIOError:
error("Jetconf daemon already running (pidfile exists). Exiting.")
sys.exit(1)
jc_config.print()
# Set signal handlers
def sig_exit_handler(signum, frame):
os.close(fl)
os.unlink(CONFIG_GLOBAL["PIDFILE"])
info("Exiting.")
sys.exit(0)
# Instantiate Jetconf main class
jc = jetconf.Jetconf(jc_config)
jetconf.JC = jc
signal.signal(signal.SIGTERM, sig_exit_handler)
signal.signal(signal.SIGINT, sig_exit_handler)
# Import backend modules
backend_package = CONFIG_GLOBAL["BACKEND_PACKAGE"]
try:
usr_state_data_handlers = import_module(backend_package + ".usr_state_data_handlers")
usr_conf_data_handlers = import_module(backend_package + ".usr_conf_data_handlers")
usr_op_handlers = import_module(backend_package + ".usr_op_handlers")
usr_datastore = import_module(backend_package + ".usr_datastore")
except ImportError as e:
error(ErrorHelpers.epretty(e))
error("Cannot import backend package \"{}\". Exiting.".format(backend_package))
sys.exit(1)
# Load data model
yang_mod_dir = CONFIG_GLOBAL["YANG_LIB_DIR"]
yang_lib_str = resource_string(backend_package, "yang-library-data.json").decode("utf-8")
datamodel = DataModel(yang_lib_str, [yang_mod_dir])
# Datastore init
datastore = usr_datastore.UserDatastore(datamodel, CONFIG_GLOBAL["DATA_JSON_FILE"], with_nacm=CONFIG_NACM["ENABLED"])
try:
datastore.load()
except (FileNotFoundError, YangsonException) as e:
error("Cannot load JSON datastore " + CONFIG_GLOBAL["DATA_JSON_FILE"])
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
jc.init()
except JetconfInitError as e:
error(str(e))
jc.cleanup()
# Validate datastore on startup
try:
datastore.get_data_root().validate(ValidationScope.all, ContentType.config)
except (SchemaError, SemanticError) as e:
error("Initial validation of datastore failed")
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
# Register handlers for configuration data
usr_conf_data_handlers.register_conf_handlers(datastore)
# Exit
info("Exiting (error)")
sys.exit(1)
# Register handlers for state data
usr_state_data_handlers.register_state_handlers(datastore)
# Run Jetconf (this will block until shutdown)
jc.run()
# Register handlers for operations
op_internal.register_op_handlers(datastore)
usr_op_handlers.register_op_handlers(datastore)
jc.cleanup()
# Write datastore content to the backend application (if required)
try:
confh_ros = usr_conf_data_handlers.run_on_startup
except AttributeError:
pass
else:
try:
confh_ros()
except Exception as e:
error("Writing configuration to backend failed")
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
# Create HTTP server
rest_srv = RestServer()
rest_srv.register_api_handlers(datastore)
rest_srv.register_static_handlers()
# Run HTTP server
rest_srv.run()
# Exit
info("Exiting")
sys.exit(0)
if __name__ == "__main__":
......
......@@ -2,85 +2,110 @@ import os
import yaml
from colorlog import info
_yang_mod_dir_env = os.environ.get("YANG_MODPATH")
CONFIG_GLOBAL = {
"TIMEZONE": "GMT",
"LOGFILE": "-",
"PIDFILE": "/tmp/jetconf.pid",
"PERSISTENT_CHANGES": True,
"LOG_LEVEL": "info",
"LOG_DBG_MODULES": ["*"],
"YANG_LIB_DIR": _yang_mod_dir_env,
"DATA_JSON_FILE": "data.json",
"VALIDATE_TRANSACTIONS": True,
"BACKEND_PACKAGE": "jetconf_jukebox"
}
CONFIG_HTTP = {
"DOC_ROOT": "doc-root",
"DOC_DEFAULT_NAME": "index.html",
"API_ROOT": "/restconf",
"API_ROOT_RUNNING": "/restconf_running",
"SERVER_NAME": "jetconf-h2",
"UPLOAD_SIZE_LIMIT": 1,
"LISTEN_LOCALHOST_ONLY": False,
"PORT": 8443,
"SERVER_SSL_CERT": "server.crt",
"SERVER_SSL_PRIVKEY": "server.key",
"CA_CERT": "ca.pem",
"DBG_DISABLE_CERTS": False
}
CONFIG_NACM = {
"ENABLED": True,
"ALLOWED_USERS": []
}
CONFIG_KNOT = {
"SOCKET": "/tmp/knot.sock"
}
CONFIG = {
"GLOBAL": CONFIG_GLOBAL,
"HTTP_SERVER": CONFIG_HTTP,
"NACM": CONFIG_NACM,
"KNOT": CONFIG_KNOT
}
API_ROOT_data = os.path.join(CONFIG_HTTP["API_ROOT"], "data")
API_ROOT_RUNNING_data = os.path.join(CONFIG_HTTP["API_ROOT_RUNNING"], "data")
API_ROOT_ops = os.path.join(CONFIG_HTTP["API_ROOT"], "operations")
API_ROOT_ylv = os.path.join(CONFIG_HTTP["API_ROOT"], "yang-library-version")
def load_config(filename: str) -> bool:
global API_ROOT_data
global API_ROOT_RUNNING_data
global API_ROOT_ops
global API_ROOT_ylv
with open(filename) as conf_fd:
conf_yaml = yaml.load(conf_fd)
for conf_key in CONFIG.keys():
from yaml.parser import ParserError
# For backward compatibility
CONFIG_GLOBAL = {}
CONFIG_HTTP = {}
CONFIG_NACM = {}
CONFIG = {}
CFG = None # type: JcConfig
class JcConfig:
def __init__(self):
yang_mod_dir_env = os.environ.get("YANG_MODPATH")
glob_def = {
"TIMEZONE": "GMT",
"LOGFILE": "-",
"PIDFILE": "/tmp/jetconf.pid",
"PERSISTENT_CHANGES": True,
"LOG_LEVEL": "info",
"LOG_DBG_MODULES": ["*"],
"YANG_LIB_DIR": yang_mod_dir_env,
"DATA_JSON_FILE": "data.json",
"VALIDATE_TRANSACTIONS": True,
"BACKEND_PACKAGE": "jetconf_jukebox"
}
http_def = {
"DOC_ROOT": "doc-root",
"DOC_DEFAULT_NAME": "index.html",
"API_ROOT": "/restconf",
"API_ROOT_RUNNING": "/restconf_running",
"SERVER_NAME": "jetconf-h2",
"UPLOAD_SIZE_LIMIT": 1,
"LISTEN_LOCALHOST_ONLY": False,
"PORT": 8443,
"SERVER_SSL_CERT": "server.crt",
"SERVER_SSL_PRIVKEY": "server.key",
"CA_CERT": "ca.pem",
"DBG_DISABLE_CERTS": False
}
nacm_def = {
"ENABLED": True,
"ALLOWED_USERS": []
}
root_def = {
"GLOBAL": glob_def,
"HTTP_SERVER": http_def,
"NACM": nacm_def
}
self.glob = glob_def
self.http = http_def
self.nacm = nacm_def
self.root = root_def
# Shortcuts
self.api_root_data = None
self.api_root_running_data = None
self.api_root_ops = None
self.api_root_ylv = None
self._gen_shortcuts()
def _gen_shortcuts(self):
global CONFIG_GLOBAL
global CONFIG_HTTP
global CONFIG_NACM
global CONFIG
CONFIG_GLOBAL.update(self.glob)
CONFIG_HTTP.update(self.http)
CONFIG_NACM.update(self.nacm)
CONFIG.update(self.root)
api_root = self.http["API_ROOT"]
api_root_running = self.http["API_ROOT_RUNNING"]
self.api_root_data = os.path.join(api_root, "data")
self.api_root_running_data = os.path.join(api_root_running, "data")
self.api_root_ops = os.path.join(api_root, "operations")
self.api_root_ylv = os.path.join(api_root, "yang-library-version")
def load_file(self, file_path: str) -> bool:
with open(file_path) as conf_fd:
try:
CONFIG[conf_key].update(conf_yaml[conf_key])
except KeyError:
pass
# Shortcuts
API_ROOT_data = os.path.join(CONFIG_HTTP["API_ROOT"], "data")
API_ROOT_RUNNING_data = os.path.join(CONFIG_HTTP["API_ROOT_RUNNING"], "data")
API_ROOT_ops = os.path.join(CONFIG_HTTP["API_ROOT"], "operations")
API_ROOT_ylv = os.path.join(CONFIG_HTTP["API_ROOT"], "yang-library-version")
conf_yaml = yaml.load(conf_fd)
except ParserError as e:
raise ValueError(str(e))
for conf_key in self.root.keys():
try:
self.root[conf_key].update(conf_yaml[conf_key])
except KeyError:
pass
def validate_config():
if CONFIG_GLOBAL["YANG_LIB_DIR"] is None:
raise ValueError("YANG module directory must be specified (in config file or YANG_MODPATH env variable)")
self._gen_shortcuts()
def validate(self):
if self.glob["YANG_LIB_DIR"] is None:
raise ValueError("YANG module directory must be specified (in config file or YANG_MODPATH env variable)")
def print_config():
info("Using config:\n" + yaml.dump(CONFIG, default_flow_style=False))
def print(self):
info("Using config:\n" + yaml.dump(self.root, default_flow_style=False))
......@@ -31,9 +31,9 @@ from yangson.instance import (
ObjectMember
)
from . import config
from .helpers import PathFormat, ErrorHelpers, LogHelpers, DataHelpers, JsonNodeT
from .config import CONFIG, CONFIG_NACM
from .nacm import NacmConfig, Permission, Action, NacmForbiddenError
from .nacm import NacmConfig, Permission, Action
from .handler_list import (
OP_HANDLERS,
STATE_DATA_HANDLES,
......@@ -43,7 +43,17 @@ from .handler_list import (
StateDataContainerHandler,
StateDataListHandler
)
from .errors import JetconfError
from .errors import (
ConfHandlerFailedError,
StagingDataException,
NoHandlerForStateDataError,
NoHandlerForOpError,
InstanceAlreadyPresent,
OpHandlerFailedError,
NoHandlerError,
DataLockError,
NacmForbiddenError
)
epretty = ErrorHelpers.epretty
debug_data = LogHelpers.create_module_dbg_logger(__name__)
......@@ -55,46 +65,6 @@ class ChangeType(Enum):
DELETE = 2
class DataLockError(JetconfError):
pass
class StagingDataException(JetconfError):
pass
class InstanceAlreadyPresent(JetconfError):
pass
class HandlerError(JetconfError):
pass
class NoHandlerError(HandlerError):
pass
class ConfHandlerFailedError(HandlerError):
pass
class OpHandlerFailedError(HandlerError):
pass
class NoHandlerForOpError(NoHandlerError):
def __init__(self, op_name: str):
self.op_name = op_name
def __str__(self):
return "Nonexistent handler for operation \"{}\"".format(self.op_name)
class NoHandlerForStateDataError(NoHandlerError):
pass
class RpcInfo:
def __init__(self):
self.username = None # type: str
......@@ -169,7 +139,7 @@ class UsrChangeJournal:
try:
# Validate syntax and semantics of new data
if CONFIG["GLOBAL"]["VALIDATE_TRANSACTIONS"] is True:
if config.CFG.glob["VALIDATE_TRANSACTIONS"] is True:
nr.validate(ValidationScope.all, ContentType.config)
except (SchemaError, SemanticError) as e:
error("Data validation error:")
......@@ -394,7 +364,7 @@ class BaseDatastore:
if (len(ii) > 0) and (isinstance(ii[0], MemberName)):
# Not getting root
ns_first = ii[0].namespace
if (ns_first == "ietf-netconf-acm") and (rpc.username not in CONFIG_NACM["ALLOWED_USERS"]):
if (ns_first == "ietf-netconf-acm") and (rpc.username not in config.CFG.nacm["ALLOWED_USERS"]):
raise NacmForbiddenError(rpc.username + " not allowed to access NACM data")
elif ns_first == "ietf-yang-library":
root = self._yang_lib_data
......@@ -402,7 +372,7 @@ class BaseDatastore:
else:
# Root node requested
# Remove NACM data if user is not NACM privieged
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]:
if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
try:
root = root.delete_item("ietf-netconf-acm:nacm")
except NonexistentInstance:
......@@ -583,13 +553,13 @@ class BaseDatastore:
ns_first = ii[0].namespace
if ns_first == "ietf-netconf-acm":
nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]:
if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
else:
# Editing root node
if input_member_ns == "ietf-netconf-acm":
nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]:
if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
# Evaluate NACM
......@@ -729,7 +699,7 @@ class BaseDatastore:
ns_first = ii[0].namespace
if ns_first == "ietf-netconf-acm":
nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]:
if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
else:
# Replacing root node
......@@ -737,7 +707,7 @@ class BaseDatastore:
nacm_val = n.value.get("ietf-netconf-acm:nacm")
if nacm_val is not None:
nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]:
if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
# Evaluate NACM
......@@ -763,7 +733,7 @@ class BaseDatastore:
ns_first = ii[0].namespace
if ns_first == "ietf-netconf-acm":
nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]:
if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
else:
# Deleting root node
......@@ -771,7 +741,7 @@ class BaseDatastore:
nacm_val = n.value.get("ietf-netconf-acm:nacm")
if nacm_val is not None:
nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]:
if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
# Evaluate NACM
......
......@@ -3,17 +3,59 @@ from yangson.instance import InstanceRoute, NonexistentInstance
# Base class for all exceptions defined in jetconf
class JetconfError(Exception):
def __init__(self, msg=""):
self.msg = msg
pass
def __str__(self):
return self.msg
# Jetconf errors
class JetconfInitError(JetconfError):
pass
class BackendError(JetconfError):
pass
class DataLockError(JetconfError):
pass
class StagingDataException(JetconfError):
pass
class InstanceAlreadyPresent(JetconfError):
pass
# Handler errors
class HandlerError(JetconfError):
pass
class NoHandlerError(HandlerError):
pass
class ConfHandlerFailedError(HandlerError):
pass
class OpHandlerFailedError(HandlerError):
pass
class NoHandlerForOpError(NoHandlerError):
def __init__(self, op_name: str):
self.op_name = op_name
def __str__(self):
return "Nonexistent handler for operation \"{}\"".format(self.op_name)
class NoHandlerForStateDataError(NoHandlerError):
pass
class StateNonexistentInstance(NonexistentInstance):
def __init__(self, ii: InstanceRoute, text: str) -> None:
self.ii = ii
......@@ -21,3 +63,21 @@ class StateNonexistentInstance(NonexistentInstance):
def __str__(self):
return str(self.ii) + ": " + self.text
# NACM errors
class NacmError(JetconfError):
pass
class NonexistentUserError(NacmError):
pass
class NacmForbiddenError(NacmError):
def __init__(self, msg="Access to data node rejected by NACM", rule=None):
self.msg = msg
self.rule = rule
def __str__(self):
return "{} (rule: {})".format(self.msg, str(self.rule))
from typing import List, Tuple, Callable, Any
from typing import Dict, Callable
from yangson.schemanode import SchemaNode
from yangson.schemadata import SchemaData
......@@ -7,17 +7,15 @@ from yangson.instance import InstanceRoute
from .helpers import JsonNodeT
HandlerSelectorT = Any
# ---------- Conf data base objects ----------
class ConfDataObjectHandler:
# ---------- Base classes for conf data handlers ----------
class ConfDataHandlerBase:
def __init__(self, ds: "BaseDatastore", sch_pth: str):
self.ds = ds
self.schema_path = sch_pth # type: str
self.schema_node = ds.get_schema_node(sch_pth) # type: SchemaNode
class ConfDataObjectHandler(ConfDataHandlerBase):
def create(self, ii: InstanceRoute, ch: "DataChange"):
pass
......@@ -31,12 +29,7 @@ class ConfDataObjectHandler:
return self.__class__.__name__ + ": listening at " + self.schema_path
class ConfDataListHandler:
def __init__(self, ds: "BaseDatastore", sch_pth: str):
self.ds = ds
self.schema_path = sch_pth # type: str
self.schema_node = ds.get_schema_node(sch_pth) # type: SchemaNode
class ConfDataListHandler(ConfDataHandlerBase):
def create_item(self, ii: InstanceRoute, ch: "DataChange"):
pass
......@@ -59,8 +52,7 @@ class ConfDataListHandler:
return self.__class__.__name__ + ": listening at " + self.schema_path
# ---------- State data base objects ----------
# ---------- Base classes for state data handlers ----------
class StateDataHandlerBase:
def __init__(self, datastore: "BaseDatastore", schema_path: str):
self.ds = datastore
......@@ -82,56 +74,29 @@ class StateDataListHandler(StateDataHandlerBase):
pass
# ---------- Handles lists ----------
class BaseHandlerList:
def __init__(self):
self.handlers = [] # type: List[Tuple[HandlerSelectorT, Callable]]
self.default_handler = None # type: Callable
def register(self, identifier: str, handler: Callable):
raise NotImplementedError("Not implemented in base class")
def register_default(self, handler: Callable):
self.default_handler = handler
def get_handler(self, identifier: str) -> Any:
raise NotImplementedError("Not implemented in base class")
class OpHandlerList(BaseHandlerList):
def register(self, handler: Callable, op_name: str):
self.handlers.append((op_name, handler))
def get_handler(self, op_name: str) -> Callable:
for h in self.handlers:
if h[0] == op_name:
return h[1]
return self.default_handler
# ---------- Handler lists ----------
class ConfDataHandlerList:
def __init__(self):
self.handlers = [] # type: List[Tuple[HandlerSelectorT, BaseDataListener]]
self.handlers = {} # type: Dict[int, ConfDataHandlerBase]
self.handlers_pth = {} # type: Dict[str, ConfDataHandlerBase]
def register(self, handler: "BaseDataListener"):
def register(self, handler: ConfDataHandlerBase):
sch_node_id = id(handler.schema_node)
self.handlers.append((sch_node_id, handler))
self.handlers[sch_node_id] = handler
self.handlers_pth[handler.schema_path] = handler
def get_handler(self, sch_node_id: str) -> "BaseDataListener":
for h in self.handlers:
if h[0] == sch_node_id:
return h[1]
def get_handler(self, sch_node_id: int) -> ConfDataHandlerBase:
return self.handlers.get(sch_node_id)
return None
def get_handler_by_pth(self, sch_pth: str) -> ConfDataHandlerBase:
return self.handlers_pth.get(sch_pth)
class StateDataHandlerList: