Commit 42980f58 authored by Pavel Spirek's avatar Pavel Spirek

Code refactor and cleanup, version bump

parent bac0ddff
PROJECT = jetconf PROJECT = jetconf
VERSION = 0.2.6 VERSION = 0.3.0
.PHONY = tags deps install-deps test .PHONY = tags deps install-deps test
tags: tags:
......
...@@ -3,22 +3,12 @@ import colorlog ...@@ -3,22 +3,12 @@ import colorlog
import getopt import getopt
import logging import logging
import sys import sys
import signal
from importlib import import_module from pkg_resources import get_distribution
from pkg_resources import resource_string, get_distribution
from colorlog import error, info from colorlog import error, info
from yaml.parser import ParserError
from yangson.enumerations import ContentType, ValidationScope from . import config, jetconf
from yangson.exceptions import YangsonException from .errors import JetconfInitError
from yangson.schemanode import SchemaError, SemanticError
from yangson.datamodel import DataModel
from . import op_internal
from .rest_server import RestServer
from .config import CONFIG_GLOBAL, CONFIG_NACM, load_config, validate_config, print_config
from .helpers import ErrorHelpers
def print_help(): def print_help():
...@@ -29,8 +19,6 @@ def print_help(): ...@@ -29,8 +19,6 @@ def print_help():
def main(): def main():
config_file = "config.yaml"
# Check for Python version # Check for Python version
if sys.version_info < (3, 5): if sys.version_info < (3, 5):
print("Jetconf requires Python version 3.5 or higher") print("Jetconf requires Python version 3.5 or higher")
...@@ -40,6 +28,8 @@ def main(): ...@@ -40,6 +28,8 @@ def main():
jetconf_version = get_distribution("jetconf").version jetconf_version = get_distribution("jetconf").version
# Parse command line arguments # Parse command line arguments
config_file = "config.yaml"
try: try:
opts, args = getopt.getopt(sys.argv[1:], "c:vh") opts, args = getopt.getopt(sys.argv[1:], "c:vh")
except getopt.GetoptError: except getopt.GetoptError:
...@@ -58,38 +48,41 @@ def main(): ...@@ -58,38 +48,41 @@ def main():
sys.exit(0) sys.exit(0)
# Load configuration # Load configuration
jc_config = config.JcConfig()
config.CFG = jc_config
try: try:
load_config(config_file) jc_config.load_file(config_file)
except FileNotFoundError: except FileNotFoundError:
print("Configuration file does not exist") print("Configuration file does not exist")
sys.exit(1) sys.exit(1)
except ParserError as e: except ValueError as e:
print("Configuration syntax error: " + str(e)) print("Configuration syntax error: " + str(e))
sys.exit(1) sys.exit(1)
# Validate configuration # Validate configuration
try: try:
validate_config() jc_config.validate()
except ValueError as e: except ValueError as e:
print("Error: " + str(e)) print("Error: " + str(e))
sys.exit(1) sys.exit(1)
# Set logging level # Set logging level
log_level = { log_level = {
"error": logging.ERROR, "error": logging.ERROR,
"warning": logging.WARNING, "warning": logging.WARNING,
"info": logging.INFO, "info": logging.INFO,
"debug": logging.INFO "debug": logging.INFO
}.get(CONFIG_GLOBAL["LOG_LEVEL"], logging.INFO) }.get(jc_config.glob["LOG_LEVEL"], logging.INFO)
logging.root.handlers.clear() logging.root.handlers.clear()
# Daemonize # Daemonize
if CONFIG_GLOBAL["LOGFILE"] not in ("-", "stdout"): if jc_config.glob["LOGFILE"] not in ("-", "stdout"):
# Setup basic logging # Setup basic logging
logging.basicConfig( logging.basicConfig(
format="%(asctime)s %(levelname)-8s %(message)s", format="%(asctime)s %(levelname)-8s %(message)s",
level=log_level, level=log_level,
filename=CONFIG_GLOBAL["LOGFILE"] filename=jc_config.glob["LOGFILE"]
) )
# Go to background # Go to background
...@@ -138,92 +131,30 @@ def main(): ...@@ -138,92 +131,30 @@ def main():
info("Jetconf version {}".format(jetconf_version)) info("Jetconf version {}".format(jetconf_version))
# Print configuration # Print configuration
print_config() jc_config.print()
# Create pidfile
fl = os.open(CONFIG_GLOBAL["PIDFILE"], os.O_WRONLY + os.O_CREAT, 0o666)
try:
os.lockf(fl, os.F_TLOCK, 0)
os.write(fl, str(os.getpid()).encode())
os.fsync(fl)
except BlockingIOError:
error("Jetconf daemon already running (pidfile exists). Exiting.")
sys.exit(1)
# Set signal handlers # Instantiate Jetconf main class
def sig_exit_handler(signum, frame): jc = jetconf.Jetconf(jc_config)
os.close(fl) jetconf.JC = jc
os.unlink(CONFIG_GLOBAL["PIDFILE"])
info("Exiting.")
sys.exit(0)
signal.signal(signal.SIGTERM, sig_exit_handler)
signal.signal(signal.SIGINT, sig_exit_handler)
# Import backend modules
backend_package = CONFIG_GLOBAL["BACKEND_PACKAGE"]
try:
usr_state_data_handlers = import_module(backend_package + ".usr_state_data_handlers")
usr_conf_data_handlers = import_module(backend_package + ".usr_conf_data_handlers")
usr_op_handlers = import_module(backend_package + ".usr_op_handlers")
usr_datastore = import_module(backend_package + ".usr_datastore")
except ImportError as e:
error(ErrorHelpers.epretty(e))
error("Cannot import backend package \"{}\". Exiting.".format(backend_package))
sys.exit(1)
# Load data model
yang_mod_dir = CONFIG_GLOBAL["YANG_LIB_DIR"]
yang_lib_str = resource_string(backend_package, "yang-library-data.json").decode("utf-8")
datamodel = DataModel(yang_lib_str, [yang_mod_dir])
# Datastore init
datastore = usr_datastore.UserDatastore(datamodel, CONFIG_GLOBAL["DATA_JSON_FILE"], with_nacm=CONFIG_NACM["ENABLED"])
try: try:
datastore.load() jc.init()
except (FileNotFoundError, YangsonException) as e: except JetconfInitError as e:
error("Cannot load JSON datastore " + CONFIG_GLOBAL["DATA_JSON_FILE"]) error(str(e))
error(ErrorHelpers.epretty(e)) jc.cleanup()
sig_exit_handler(0, None)
# Validate datastore on startup # Exit
try: info("Exiting (error)")
datastore.get_data_root().validate(ValidationScope.all, ContentType.config) sys.exit(1)
except (SchemaError, SemanticError) as e:
error("Initial validation of datastore failed")
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
# Register handlers for configuration data
usr_conf_data_handlers.register_conf_handlers(datastore)
# Register handlers for state data # Run Jetconf (this will block until shutdown)
usr_state_data_handlers.register_state_handlers(datastore) jc.run()
# Register handlers for operations jc.cleanup()
op_internal.register_op_handlers(datastore)
usr_op_handlers.register_op_handlers(datastore)
# Write datastore content to the backend application (if required) # Exit
try: info("Exiting")
confh_ros = usr_conf_data_handlers.run_on_startup sys.exit(0)
except AttributeError:
pass
else:
try:
confh_ros()
except Exception as e:
error("Writing configuration to backend failed")
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
# Create HTTP server
rest_srv = RestServer()
rest_srv.register_api_handlers(datastore)
rest_srv.register_static_handlers()
# Run HTTP server
rest_srv.run()
if __name__ == "__main__": if __name__ == "__main__":
......
...@@ -2,85 +2,110 @@ import os ...@@ -2,85 +2,110 @@ import os
import yaml import yaml
from colorlog import info from colorlog import info
from yaml.parser import ParserError
_yang_mod_dir_env = os.environ.get("YANG_MODPATH")
# For backward compatibility
CONFIG_GLOBAL = { CONFIG_GLOBAL = {}
"TIMEZONE": "GMT", CONFIG_HTTP = {}
"LOGFILE": "-", CONFIG_NACM = {}
"PIDFILE": "/tmp/jetconf.pid", CONFIG = {}
"PERSISTENT_CHANGES": True,
"LOG_LEVEL": "info", CFG = None # type: JcConfig
"LOG_DBG_MODULES": ["*"],
"YANG_LIB_DIR": _yang_mod_dir_env,
"DATA_JSON_FILE": "data.json", class JcConfig:
"VALIDATE_TRANSACTIONS": True, def __init__(self):
"BACKEND_PACKAGE": "jetconf_jukebox" yang_mod_dir_env = os.environ.get("YANG_MODPATH")
}
glob_def = {
CONFIG_HTTP = { "TIMEZONE": "GMT",
"DOC_ROOT": "doc-root", "LOGFILE": "-",
"DOC_DEFAULT_NAME": "index.html", "PIDFILE": "/tmp/jetconf.pid",
"API_ROOT": "/restconf", "PERSISTENT_CHANGES": True,
"API_ROOT_RUNNING": "/restconf_running", "LOG_LEVEL": "info",
"SERVER_NAME": "jetconf-h2", "LOG_DBG_MODULES": ["*"],
"UPLOAD_SIZE_LIMIT": 1, "YANG_LIB_DIR": yang_mod_dir_env,
"LISTEN_LOCALHOST_ONLY": False, "DATA_JSON_FILE": "data.json",
"PORT": 8443, "VALIDATE_TRANSACTIONS": True,
"BACKEND_PACKAGE": "jetconf_jukebox"
"SERVER_SSL_CERT": "server.crt", }
"SERVER_SSL_PRIVKEY": "server.key",
"CA_CERT": "ca.pem", http_def = {
"DBG_DISABLE_CERTS": False "DOC_ROOT": "doc-root",
} "DOC_DEFAULT_NAME": "index.html",
"API_ROOT": "/restconf",
CONFIG_NACM = { "API_ROOT_RUNNING": "/restconf_running",
"ENABLED": True, "SERVER_NAME": "jetconf-h2",
"ALLOWED_USERS": [] "UPLOAD_SIZE_LIMIT": 1,
} "LISTEN_LOCALHOST_ONLY": False,
"PORT": 8443,
CONFIG_KNOT = {
"SOCKET": "/tmp/knot.sock" "SERVER_SSL_CERT": "server.crt",
} "SERVER_SSL_PRIVKEY": "server.key",
"CA_CERT": "ca.pem",
CONFIG = { "DBG_DISABLE_CERTS": False
"GLOBAL": CONFIG_GLOBAL, }
"HTTP_SERVER": CONFIG_HTTP,
"NACM": CONFIG_NACM, nacm_def = {
"KNOT": CONFIG_KNOT "ENABLED": True,
} "ALLOWED_USERS": []
}
API_ROOT_data = os.path.join(CONFIG_HTTP["API_ROOT"], "data")
API_ROOT_RUNNING_data = os.path.join(CONFIG_HTTP["API_ROOT_RUNNING"], "data") root_def = {
API_ROOT_ops = os.path.join(CONFIG_HTTP["API_ROOT"], "operations") "GLOBAL": glob_def,
API_ROOT_ylv = os.path.join(CONFIG_HTTP["API_ROOT"], "yang-library-version") "HTTP_SERVER": http_def,
"NACM": nacm_def
}
def load_config(filename: str) -> bool:
global API_ROOT_data self.glob = glob_def
global API_ROOT_RUNNING_data self.http = http_def
global API_ROOT_ops self.nacm = nacm_def
global API_ROOT_ylv self.root = root_def
with open(filename) as conf_fd: # Shortcuts
conf_yaml = yaml.load(conf_fd) self.api_root_data = None
for conf_key in CONFIG.keys(): self.api_root_running_data = None
self.api_root_ops = None
self.api_root_ylv = None
self._gen_shortcuts()
def _gen_shortcuts(self):
global CONFIG_GLOBAL
global CONFIG_HTTP
global CONFIG_NACM
global CONFIG
CONFIG_GLOBAL.update(self.glob)
CONFIG_HTTP.update(self.http)
CONFIG_NACM.update(self.nacm)
CONFIG.update(self.root)
api_root = self.http["API_ROOT"]
api_root_running = self.http["API_ROOT_RUNNING"]
self.api_root_data = os.path.join(api_root, "data")
self.api_root_running_data = os.path.join(api_root_running, "data")
self.api_root_ops = os.path.join(api_root, "operations")
self.api_root_ylv = os.path.join(api_root, "yang-library-version")
def load_file(self, file_path: str) -> bool:
with open(file_path) as conf_fd:
try: try:
CONFIG[conf_key].update(conf_yaml[conf_key]) conf_yaml = yaml.load(conf_fd)
except KeyError: except ParserError as e:
pass raise ValueError(str(e))
# Shortcuts
API_ROOT_data = os.path.join(CONFIG_HTTP["API_ROOT"], "data")
API_ROOT_RUNNING_data = os.path.join(CONFIG_HTTP["API_ROOT_RUNNING"], "data")
API_ROOT_ops = os.path.join(CONFIG_HTTP["API_ROOT"], "operations")
API_ROOT_ylv = os.path.join(CONFIG_HTTP["API_ROOT"], "yang-library-version")
for conf_key in self.root.keys():
try:
self.root[conf_key].update(conf_yaml[conf_key])
except KeyError:
pass
def validate_config(): self._gen_shortcuts()
if CONFIG_GLOBAL["YANG_LIB_DIR"] is None:
raise ValueError("YANG module directory must be specified (in config file or YANG_MODPATH env variable)")
def validate(self):
if self.glob["YANG_LIB_DIR"] is None:
raise ValueError("YANG module directory must be specified (in config file or YANG_MODPATH env variable)")
def print_config(): def print(self):
info("Using config:\n" + yaml.dump(CONFIG, default_flow_style=False)) info("Using config:\n" + yaml.dump(self.root, default_flow_style=False))
...@@ -31,9 +31,9 @@ from yangson.instance import ( ...@@ -31,9 +31,9 @@ from yangson.instance import (
ObjectMember ObjectMember
) )
from . import config
from .helpers import PathFormat, ErrorHelpers, LogHelpers, DataHelpers, JsonNodeT from .helpers import PathFormat, ErrorHelpers, LogHelpers, DataHelpers, JsonNodeT
from .config import CONFIG, CONFIG_NACM from .nacm import NacmConfig, Permission, Action
from .nacm import NacmConfig, Permission, Action, NacmForbiddenError
from .handler_list import ( from .handler_list import (
OP_HANDLERS, OP_HANDLERS,
STATE_DATA_HANDLES, STATE_DATA_HANDLES,
...@@ -43,7 +43,17 @@ from .handler_list import ( ...@@ -43,7 +43,17 @@ from .handler_list import (
StateDataContainerHandler, StateDataContainerHandler,
StateDataListHandler StateDataListHandler
) )
from .errors import JetconfError from .errors import (
ConfHandlerFailedError,
StagingDataException,
NoHandlerForStateDataError,
NoHandlerForOpError,
InstanceAlreadyPresent,
OpHandlerFailedError,
NoHandlerError,
DataLockError,
NacmForbiddenError
)
epretty = ErrorHelpers.epretty epretty = ErrorHelpers.epretty
debug_data = LogHelpers.create_module_dbg_logger(__name__) debug_data = LogHelpers.create_module_dbg_logger(__name__)
...@@ -55,46 +65,6 @@ class ChangeType(Enum): ...@@ -55,46 +65,6 @@ class ChangeType(Enum):
DELETE = 2 DELETE = 2
class DataLockError(JetconfError):
pass
class StagingDataException(JetconfError):
pass
class InstanceAlreadyPresent(JetconfError):
pass
class HandlerError(JetconfError):
pass
class NoHandlerError(HandlerError):
pass
class ConfHandlerFailedError(HandlerError):
pass
class OpHandlerFailedError(HandlerError):
pass
class NoHandlerForOpError(NoHandlerError):
def __init__(self, op_name: str):
self.op_name = op_name
def __str__(self):
return "Nonexistent handler for operation \"{}\"".format(self.op_name)
class NoHandlerForStateDataError(NoHandlerError):
pass
class RpcInfo: class RpcInfo:
def __init__(self): def __init__(self):
self.username = None # type: str self.username = None # type: str
...@@ -169,7 +139,7 @@ class UsrChangeJournal: ...@@ -169,7 +139,7 @@ class UsrChangeJournal:
try: try:
# Validate syntax and semantics of new data # Validate syntax and semantics of new data
if CONFIG["GLOBAL"]["VALIDATE_TRANSACTIONS"] is True: if config.CFG.glob["VALIDATE_TRANSACTIONS"] is True:
nr.validate(ValidationScope.all, ContentType.config) nr.validate(ValidationScope.all, ContentType.config)
except (SchemaError, SemanticError) as e: except (SchemaError, SemanticError) as e:
error("Data validation error:") error("Data validation error:")
...@@ -394,7 +364,7 @@ class BaseDatastore: ...@@ -394,7 +364,7 @@ class BaseDatastore:
if (len(ii) > 0) and (isinstance(ii[0], MemberName)): if (len(ii) > 0) and (isinstance(ii[0], MemberName)):
# Not getting root # Not getting root
ns_first = ii[0].namespace ns_first = ii[0].namespace
if (ns_first == "ietf-netconf-acm") and (rpc.username not in CONFIG_NACM["ALLOWED_USERS"]): if (ns_first == "ietf-netconf-acm") and (rpc.username not in config.CFG.nacm["ALLOWED_USERS"]):
raise NacmForbiddenError(rpc.username + " not allowed to access NACM data") raise NacmForbiddenError(rpc.username + " not allowed to access NACM data")
elif ns_first == "ietf-yang-library": elif ns_first == "ietf-yang-library":
root = self._yang_lib_data root = self._yang_lib_data
...@@ -402,7 +372,7 @@ class BaseDatastore: ...@@ -402,7 +372,7 @@ class BaseDatastore:
else: else:
# Root node requested # Root node requested
# Remove NACM data if user is not NACM privieged # Remove NACM data if user is not NACM privieged
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]: if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
try: try:
root = root.delete_item("ietf-netconf-acm:nacm") root = root.delete_item("ietf-netconf-acm:nacm")
except NonexistentInstance: except NonexistentInstance:
...@@ -583,13 +553,13 @@ class BaseDatastore: ...@@ -583,13 +553,13 @@ class BaseDatastore:
ns_first = ii[0].namespace ns_first = ii[0].namespace
if ns_first == "ietf-netconf-acm": if ns_first == "ietf-netconf-acm":
nacm_changed = True nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]: if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data") raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
else: else:
# Editing root node # Editing root node
if input_member_ns == "ietf-netconf-acm": if input_member_ns == "ietf-netconf-acm":
nacm_changed = True nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]: if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data") raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
# Evaluate NACM # Evaluate NACM
...@@ -729,7 +699,7 @@ class BaseDatastore: ...@@ -729,7 +699,7 @@ class BaseDatastore:
ns_first = ii[0].namespace ns_first = ii[0].namespace
if ns_first == "ietf-netconf-acm": if ns_first == "ietf-netconf-acm":
nacm_changed = True nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]: if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data") raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
else: else:
# Replacing root node # Replacing root node
...@@ -737,7 +707,7 @@ class BaseDatastore: ...@@ -737,7 +707,7 @@ class BaseDatastore:
nacm_val = n.value.get("ietf-netconf-acm:nacm") nacm_val = n.value.get("ietf-netconf-acm:nacm")
if nacm_val is not None: if nacm_val is not None:
nacm_changed = True nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]: if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data") raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
# Evaluate NACM # Evaluate NACM
...@@ -763,7 +733,7 @@ class BaseDatastore: ...@@ -763,7 +733,7 @@ class BaseDatastore:
ns_first = ii[0].namespace ns_first = ii[0].namespace
if ns_first == "ietf-netconf-acm": if ns_first == "ietf-netconf-acm":
nacm_changed = True nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]: if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data") raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
else: else:
# Deleting root node # Deleting root node
...@@ -771,7 +741,7 @@ class BaseDatastore: ...@@ -771,7 +741,7 @@ class BaseDatastore:
nacm_val = n.value.get("ietf-netconf-acm:nacm") nacm_val = n.value.get("ietf-netconf-acm:nacm")
if nacm_val is not None: if nacm_val is not None:
nacm_changed = True nacm_changed = True
if rpc.username not in CONFIG_NACM["ALLOWED_USERS"]: if rpc.username not in config.CFG.nacm["ALLOWED_USERS"]:
raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data") raise NacmForbiddenError(rpc.username + " not allowed to modify NACM data")
# Evaluate NACM # Evaluate NACM
......
...@@ -3,17 +3,59 @@ from yangson.instance import InstanceRoute, NonexistentInstance ...@@ -3,17 +3,59 @@ from yangson.instance import InstanceRoute, NonexistentInstance
# Base class for all exceptions defined in jetconf # Base class for all exceptions defined in jetconf
class JetconfError(Exception): class JetconfError(Exception):
def __init__(self, msg=""): pass
self.msg = msg
def __str__(self):
return self.msg # Jetconf errors
class JetconfInitError(JetconfError):
pass
class BackendError(JetconfError): class BackendError(JetconfError):
pass pass
class DataLockError(JetconfError):
pass
class StagingDataException(JetconfError):
pass
class InstanceAlreadyPresent(JetconfError):
pass
# Handler errors
class HandlerError(JetconfError):
pass
class NoHandlerError(HandlerError):
pass
class ConfHandlerFailedError(HandlerError):
pass
class OpHandlerFailedError(HandlerError):
pass
class NoHandlerForOpError(NoHandlerError):
def __init__(self, op_name: str):
self.op_name = op_name
def __str__(self):
return "Nonexistent handler for operation \"{}\"".format(self.op_name)
class NoHandlerForStateDataError(NoHandlerError):
pass
class StateNonexistentInstance(NonexistentInstance): class StateNonexistentInstance(NonexistentInstance):
def __init__(self, ii: InstanceRoute, text: str) -> None: def __init__(self, ii: InstanceRoute, text: str) -> None:
self.ii = ii self.ii = ii
...@@ -21,3 +63,21 @@ class StateNonexistentInstance(NonexistentInstance): ...@@ -21,3 +63,21 @@ class StateNonexistentInstance(NonexistentInstance):
def __str__(self): def __str__(self):
return str(self.ii) + ": " + self.text return str(self.ii) + ": " + self.text
# NACM errors
class NacmError(JetconfError):
pass
class NonexistentUserError(NacmError):
pass
class NacmForbiddenError(NacmError):
def __init__(self, msg="Access to data node rejected by NACM", rule=None):
self.msg = msg
self.rule = rule
def __str__(self):