jwutils: Move to jwutils -> jw.util

Move all implementation source code from the jwutils module to jw.util. For compatibility with existing Python modules, keep a thin, autogenerated compatibility shim under jwutils.
Signed-off-by: Jan Lindemann <jan@janware.com>

src/python/jw/util/ArgsContainer.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+
+import argparse
+
+from collections import OrderedDict
+
+from .log import slog
+
+class ArgsContainer: # export
+
+    __args: OrderedDict[str, str] = OrderedDict()
+    __kwargs: OrderedDict[str, str] = OrderedDict()
+    __values: dict[str, str] = {}
+    __specified_args: list[str] = list()
+
+    def __getattr__(self, name):
+        values = self.__values
+        if name in self.__values:
+            return self.__values[name]
+        if name in self.__kwargs.keys():
+            d = self.__kwargs[name]
+            if 'default' in d:
+                return d['default']
+            raise Exception(f'No value for argument "{name}"')
+        raise Exception(f'No argument "{name}" defined')
+
+    def __setattr__(self, name, value):
+        if not name in self.__kwargs.keys():
+            raise Exception(f'No argument "{name}" defined')
+        self.__values[name] = value
+        self.__specified_args.append(name)
+
+    def add_argument(self, *args, **kwargs):
+        for arg in args:
+            if arg[0] != '-':
+                name = arg
+                break
+            if arg[:2] == '--':
+                name = arg[2:]
+                break
+        else:
+            raise Exception('Missing argument name')
+        name = name.replace('-', '_')
+        self.__args[name] = args
+        self.__kwargs[name] = kwargs
+
+    def keys(self):
+        return self.__args.keys()
+
+    def args(self, name) -> str:
+        return self.__args[name]
+
+    def kwargs(self, name) -> str:
+        return self.__kwargs[name]
+
+    def dump(self, prio, *args, **kwargs):
+        caller = get_caller_pos(**kwargs)
+        for name in self.__kwargs.keys():
+            val = None
+            try:
+                val = self.__getattr__(name)
+            except:
+                pass
+            slog(prio, f'{name}: {val}', caller=caller)
+
+    @property
+    def specified_args(self):
+        return self.__specified_args
+
+def add_argument(p: argparse.ArgumentParser|ArgsContainer, name: str, *args, **kwargs): # export
+
+    key = name.strip('--').replace('-', '_')
+    if isinstance(p, ArgsContainer):
+        if key in p.keys():
+            return
+    elif isinstance(p, argparse.ArgumentParser):
+        for action in p._actions:
+            if key == action.dest:
+                return
+    else:
+        raise Exception('Unknown type {type(p)} of {p} passed to add_argument()')
+
+    p.add_argument(name, *args, **kwargs)

src/python/jw/util/Bunch.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+from typing import Any
+
+class Bunch: # export
+
+    def __init__(self, **kwargs):
+        self.__dict__.update(kwargs)
+
+    def __getitem__(self, key: str) -> Any:
+        return self.__dict__[key]

src/python/jw/util/Cmd.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+import inspect, sys, re, abc, argparse
+from argparse import ArgumentParser, _SubParsersAction
+
+from . import log
+
+# full blown example of one level of nested subcommands
+# git -C project remote -v show -n myremote
+
+class Cmd(abc.ABC): # export
+
+    @abc.abstractmethod
+    async def run(self, args):
+        pass
+
+    def __init__(self, name: str, help: str) -> None:
+        from . import Cmds
+        self.name = name
+        self.help = help
+        self.parent = None
+        self.children: list[Cmd] = []
+        self.child_classes: list[type[Cmd]] = []
+        self.app: Cmds|None = None
+
+    async def _run(self, args):
+        pass
+
+    def add_parser(self, parsers) -> ArgumentParser:
+        r = parsers.add_parser(self.name, help=self.help,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+        r.set_defaults(func=self.run)
+        return r
+
+    def add_subcommands(self, cmd: str|type[Cmd]|list[type[Cmd]]) -> None:
+        if isinstance(cmd, str):
+            sc = []
+            for name, obj in inspect.getmembers(sys.modules[self.__class__.__module__]):
+                if inspect.isclass(obj):
+                    if re.search(cmd, str(obj)):
+                        sc.append(obj)
+                        log.slog(log.DEBUG, f"Found subcommand {obj}")
+            self.add_subcommands(sc)
+            return
+        if isinstance(cmd, list):
+            for c in cmd:
+                self.add_subcommands(c)
+            return
+        self.child_classes.append(cmd)
+
+    # To be overridden by derived class in case the command does take arguments.
+    # Will be called from App base class constructor and set up the parser hierarchy
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        pass
+
+    def conf_value(self, path, default=None):
+        ret = None if self.app is None else self.app.conf_value(path, default)
+        if ret is None and default is not None:
+            return default
+        return ret

src/python/jw/util/Cmds.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+
+import os, sys, argcomplete, argparse, importlib, inspect, re, pickle, asyncio, cProfile
+from argparse import ArgumentParser
+from pathlib import Path, PurePath
+
+from .log import *
+from .stree import serdes
+
+class Cmds: # export
+
+    def __instantiate(self, cls):
+        try:
+            r = cls()
+        except Exception as e:
+            slog(ERR, f'Failed to instantiate command of type {cls}: {e}')
+            raise
+        r.cmds = self # TODO: Rename Cmds class to App, "Cmds" isn't very self-explanatory
+        r.app = self
+        return r
+
+    def __add_cmd_to_parser(self, cmd, parsers):
+        parser = cmd.add_parser(parsers)
+        cmd.add_arguments(parser)
+        if len(cmd.child_classes) > len(cmd.children):
+            for c in cmd.child_classes:
+                cmd.children.append(self.__instantiate(c))
+        if len(cmd.children) > 0:
+            subparsers = parser.add_subparsers(title='Available subcommands of ' + cmd.name, metavar='')
+            for sub_cmd in cmd.children:
+                self.__add_cmd_to_parser(sub_cmd, subparsers)
+
+    def __parse_config(self):
+        exe_stem = str(PurePath(sys.argv[0]).stem)
+        path = str(Path.home()) + '/.' + exe_stem + 'rc'
+        if not os.path.exists(path):
+            return None, []
+        slog(DEBUG, 'Reading configuration "{}"'.format(path))
+        return serdes.read(path, ''), [path]
+
+    def __init__(self, description: str = '', filter: str = '^Cmd.*', modules: None=None, eloop: None=None) -> None:
+        self.__description = description
+        self.__filter = filter
+        self.__modules = modules
+        self.__conf, self.__conf_paths = self.__parse_config()
+        self.__cmds = []
+        self.eloop = eloop
+        self.__own_eloop = False
+        if eloop is None:
+            self.eloop = asyncio.get_event_loop()
+            self.__own_eloop = True
+
+        log_level = "notice"
+        log_flags = 'stderr,position,prio,color'
+        log_file = None
+        # poor man's parsing in the absence of a complete command-line definition
+        for i in range(1, len(sys.argv)):
+            if i >= len(sys.argv) - 1:
+                break
+            arg = sys.argv[i]
+            if arg == '--log-level':
+                i += 1
+                log_level = sys.argv[i]
+                continue
+            if arg == '--log-flags':
+                log_flags = sys.argv[i]
+                continue
+        set_flags(log_flags)
+        set_level(log_level)
+        slog(DEBUG, "set log level to {}".format(log_level))
+        self.__parser = argparse.ArgumentParser(usage=os.path.basename(sys.argv[0]) + ' [options]',
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter, description=self.__description)
+        self.__parser.add_argument('--log-flags', help='Log flags', default=log_flags)
+        self.__parser.add_argument('--log-level', help='Log level', default=log_level)
+        self.__parser.add_argument('--backtrace', help='Show exception backtraces', action='store_true', default=False)
+        self.__parser.add_argument('--write-profile', help='Profile code and store output to file', default=None)
+        self.__parser.add_argument('--log-file', help='Log file', default=log_file)
+        if self.__modules == None:
+            self.__modules = [ '__main__' ]
+        subcmds = set()
+        slog(DEBUG, '-- searching for commands')
+        for m in self.__modules: # type: ignore
+            if m != '__main__':
+                importlib.import_module(m)
+            for name, c in inspect.getmembers(sys.modules[m], inspect.isclass):
+                if not re.match(self.__filter, name):
+                    slog(DEBUG, 'o "{}.{}" has wrong name'.format(m, name))
+                    continue
+                if inspect.isabstract(c):
+                    slog(DEBUG, 'o "{}.{}" is abstract'.format(m, name))
+                    continue
+                slog(DEBUG, 'o "{}.{}" is fine, instantiating'.format(m, name))
+                cmd = self.__instantiate(c)
+                #cmd.add_parser(subparsers)
+                self.__cmds.append(cmd)
+                subcmds.update(cmd.child_classes)
+
+        cmds = [cmd for cmd in self.__cmds if type(cmd) not in subcmds]
+        subparsers = self.__parser.add_subparsers(title='Available commands', metavar='')
+        for cmd in cmds:
+            slog(DEBUG, f'Adding top-level command {cmd} to parser')
+            self.__add_cmd_to_parser(cmd, subparsers)
+
+    # Run all sub-commands. Overwrite if you want to do anything before or after
+    async def _run(self, argv=None):
+        return await self.args.func(self.args)
+
+    async def __run(self, argv=None):
+        argcomplete.autocomplete(self.__parser)
+        self.args = self.__parser.parse_args(args=argv)
+        set_flags(self.args.log_flags)
+        set_level(self.args.log_level)
+        self.__back_trace = self.args.backtrace
+        exit_status = 0
+
+        # This is the toplevel parser, i.e. no func member has been added to the args via
+        #
+        #    Cmds.__init__()
+        #    Cmds.__add_cmd_to_parser(cmd, subparsers)
+        #    CmdXXX.add_parser(parsers)
+        #    super().add_parser(parsers)
+        #    Cmd.__parser.set_defaults(func=self.run)
+        #
+        if not hasattr(self.args, 'func'):
+            self.__parser.print_help()
+            return None
+
+        pr = None if self.args.write_profile is None else cProfile.Profile()
+        if pr is not None:
+            pr.enable()
+
+        if self.__conf:
+            self.__conf.dump(DEBUG, "Configuration")
+        if self.args.log_file is not None:
+            add_log_file(self.args.log_file)
+
+        try:
+            ret = await self._run(self.args)
+        except Exception as e:
+            if hasattr(e, 'message'):
+                slog(ERR, e.message)
+            else:
+                slog(ERR, f'Exception: {type(e)}: {e}')
+            exit_status = 1
+            if self.__back_trace:
+                raise
+        finally:
+            if pr is not None:
+                pr.disable()
+                slog(NOTICE, f'Writing profile statistics to {self.args.write_profile}')
+                pr.dump_stats(self.args.write_profile)
+
+        if exit_status:
+            sys.exit(exit_status)
+
+    def __del__(self):
+        if self.__own_eloop:
+            if self.eloop is not None:
+                self.eloop.close()
+                self.eloop = None
+            self.__own_eloop = False
+
+    def conf_value(self, path, default=None):
+        ret = None if self.__conf is None else self.__conf.value(path)
+        if ret is None and default is not None:
+            return default
+        return ret
+
+    def parser(self) -> ArgumentParser:
+        return self.__parser
+
+    def run(self, argv=None) -> None:
+        #return self.__run()
+        return self.eloop.run_until_complete(self.__run(argv)) # type: ignore
+
+def run_sub_commands(description = '', filter = '^Cmd.*', modules=None, argv=None): # export
+    cmds = Cmds(description, filter, modules)
+    return cmds.run(argv=argv)

src/python/jw/util/Config.py

@@ -0,0 +1,169 @@
+# -*- coding: utf-8 -*-
+
+from typing import Optional, Dict, cast
+import os, re, glob, sys
+from pathlib import Path, PosixPath
+
+from . import stree
+from .stree.StringTree import StringTree
+from .log import *
+
+class Config(): # export
+
+    def __load(self, search_dirs, glob_paths, refuse_mode_mask):
+
+        def __is_abs(path):
+            if path is None:
+                return False
+            if len(path) == 0:
+                return False
+            if path[0] != '/':
+                return False
+            return True
+
+        ret = StringTree("", "")
+        exe = Path(os.path.basename(sys.argv[0])).stem
+        if glob_paths is None:
+            glob_paths = [f'.{exe}', f'{exe}.conf']
+        elif isinstance(glob_paths, str):
+            glob_paths = [glob_paths]
+        if search_dirs is None:
+            env_key = re.sub('[-.]', '_', exe)
+            search_dirs = os.getenv(env_key)
+        for path in glob_paths:
+            dirs = search_dirs
+            if dirs is None:
+                dirs = [''] if __is_abs(path) else [ str(Path.home()), str(Path.cwd()) ]
+            for d in dirs:
+                g = d + '/' + path if len(d) else path
+                slog(DEBUG, 'Looking for config "{}"'.format(g))
+                for f in glob.glob(g):
+                    slog(DEBUG, 'Reading config "{}"'.format(f))
+                    paths_buf = []
+                    tree = stree.read(f, paths_buf=paths_buf)
+                    assert(len(paths_buf))
+                    if refuse_mode_mask is not None:
+                        for p in paths_buf:
+                            st = os.stat(p)
+                            if st.st_mode & refuse_mode_mask:
+                                for item in tree.child_list():
+                                    if item.content is None:
+                                        continue
+                                    if not re.search('password|secret', cast(str, item.content), flags=re.IGNORECASE):
+                                        continue
+                                    msg = "Config files define secret, but at least one has file permissions open for world"
+                                    slog(ERR, f'{msg}:')
+                                    for pp in paths_buf:
+                                        slog(ERR, f'  {((os.stat(pp).st_mode) & 0o7777):o} {pp}')
+                                    raise Exception(msg)
+                    tree.dump(DEBUG, f)
+                    ret.add("", tree)
+        return ret
+
+    def __init__(self,
+                 search_dirs: Optional[list[str]]=None,
+                 glob_paths: Optional[list[str]]=None,
+                 glob_paths_env_key: Optional[str]=None,
+                 defaults: Optional[Dict[str, str]]=None,
+                 tree: Optional[StringTree]=None,
+                 parent=None,
+                 root_section=None,
+                 refuse_mode_mask=0o0027
+            ) -> None:
+
+        self.__parent = parent
+
+        if tree is not None:
+            assert(search_dirs is None)
+            assert(glob_paths is None)
+            assert(glob_paths_env_key is None)
+            self.__conf = tree
+        else:
+            assert(tree is None)
+            if glob_paths_env_key is not None:
+                glob_paths_env = os.getenv(glob_paths_env_key)
+                if glob_paths_env is not None:
+                    if glob_paths is None:
+                        glob_paths = []
+                    glob_paths.extend(glob_paths_env.split(':'))
+
+            self.__conf = self.__load(search_dirs=search_dirs, glob_paths=glob_paths,
+                                      refuse_mode_mask=refuse_mode_mask)
+
+        if root_section is not None:
+            tmp = self.__conf.get(root_section)
+            if tmp is None:
+                tmp = StringTree("", "")
+            self.__conf = tmp
+
+        if defaults is not None:
+            for key, val in defaults.items():
+                if self.__conf.get(key) is None:
+                    self.__conf[key] = val
+
+        self.__conf.dump(DEBUG, "superposed configuration")
+
+    def __getitem__(self, key: str) -> str:
+        ret = self.get(key)
+        if ret is None:
+            raise KeyError(key)
+        return ret
+
+    def __setitem__(self, key: str, value: str):
+        return self.set(key, value)
+
+    @property
+    def parent(self):
+        return self.__parent
+
+    @property
+    def root(self):
+        if self.__parent is None:
+            return self
+        return self.__parent.root
+
+    def set(self, key: str, val):
+        self.__conf[key] = val
+
+    def get(self, key: str, default = None) -> Optional[str]:
+        item = self.__conf.get(key)
+        if item:
+            return item.value()
+        return default
+
+    def entries(self, key: str) -> list[str]:
+        item = self.__conf.get(key)
+        if item is None:
+            return []
+        return [name for name, child in item.children.items()]
+
+    # This is an alias for get()
+    def value(self, key: str, default = None) -> Optional[str]:
+        return self.get(key, default)
+
+    def branch(self, path: str, throw: bool=True): # type: ignore # Optional[Config]: FIXME: Don't know how to get hold of this type here
+        if self.__conf:
+            tree = self.__conf.get(path)
+            if tree is None:
+                msg = f'Tried to get non-existent branch "{path}" from config'
+                if not throw:
+                    slog(DEBUG, msg)
+                    return None
+                self.dump(ERR, msg)
+                raise Exception(msg)
+            return Config(tree=tree, parent=self) # type: ignore
+        return None
+
+    def dump(self, prio: int, *args, **kwargs) -> None:
+        caller = get_caller_pos(1, kwargs)
+        self.__conf.dump(prio, caller=caller, *args, **kwargs)
+
+    @property
+    def name(self):
+        return self.__conf.content
+
+    def find(self, key: str|None, val: str|None, match:StringTree.Match=StringTree.Match.Equal) -> list[str]:
+        return self.__conf.find(key, val, match=match)
+
+    #def __getattr__(self, name: str):
+    #    return getattr(self.__conf, name)

src/python/jw/util/CppState.py

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+
+class CppState: # export
+
+    def __init__(self):
+        self.__pair_square = ['[', ']']
+        self.__pair_curly = ['{', '}']
+        self.__pair_ext = ['<', '>']
+        self.__pair_paren = ['(', ')']
+        self.__pair_c_comment = ['/*', '*/']
+        self.__pair_special = ['?', '?']
+        self.reset()
+
+    def reset(self):
+        self.curly = 0
+        self.square = 0
+        self.ext = 0
+        self.paren = 0
+        self.in_c_comment = False
+        self.in_cpp_comment = False
+        self.in_special = False
+        self.rule = []
+        self.rules = []
+        self.things = []
+
+    def optional(self):
+        return self.square != 0 or self.curly != 0
+
+    def update(self, tok, line):
+        if not self.in_comment():
+            if tok == '[':
+                self.square += 1
+                self.things.append(self.__pair_square)
+            elif tok == ']':
+                self.square -= 1
+                assert(self.things.pop() == self.__pair_square)
+            elif tok == '{':
+                self.curly += 1
+                self.things.append(self.__pair_curly)
+            elif tok == '}':
+                self.curly -= 1
+                assert(self.things.pop() == self.__pair_curly)
+            elif tok == '(':
+                self.paren += 1
+                self.things.append(self.__pair_paren)
+            elif tok == ')':
+                self.paren -= 1
+                assert(self.things.pop() == self.__pair_paren)
+            elif tok == '<':
+                self.ext += 1
+                self.things.append(self.__pair_ext)
+            elif tok == '>':
+                self.ext -= 1
+                assert(self.things.pop() == self.__pair_ext)
+            elif tok == '?':
+                if not self.in_special:
+                    self.in_special = True
+                    self.things.append(self.__pair_special)
+                else:
+                    self.in_special = False
+                    assert(self.things.pop() == self.__pair_special)
+            elif tok == '/*':
+                self.in_c_comment = True
+                self.things.append(self.__pair_c_comment)
+            elif tok == '*/':
+                raise Exception("Unmatched closing C-style comment mark", tok, "in line", line)
+        else:
+            if self.in_cpp_comment:
+                if tok == '\n':
+                    self.in_cpp_comment = False
+                    return
+            if tok == '/*':
+                raise Exception("Nested C-style comment", tok, "in line", line)
+            elif tok == '*/':
+                assert(self.things.pop() == self.__pair_c_comment)
+                self.in_c_comment = False
+
+        if self.curly < 0 or self.square < 0 or self.ext < 0 or self.paren < 0:
+            raise Exception("Unbalanced BNF bracket", tok, "in line", line)
+        return self.optional()
+
+    def in_list(self):
+        return self.curly > 0
+
+    def in_option(self):
+        return self.square > 0
+
+    def in_paren(self):
+        return self.paren > 0
+
+    def in_ext(self):
+        return self.ext > 0
+
+    def in_comment(self):
+        return self.in_c_comment or self.in_cpp_comment
+
+    def in_something(self):
+        if len(self.things) == 0:
+            return None
+        return self.things[-1]
+
+    def is_optional(self):
+        return self.in_list() or self.in_option()
+

src/python/jw/util/Makefile

@@ -0,0 +1,6 @@
+TOPDIR = ../../../..
+
+PY_UPDATE_INIT_PY ?= false
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-mod.mk

src/python/jw/util/Object.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+
+from . import log
+
+class Object(object): # export
+
+    def __init__(self):
+        self.log_level = log.level
+
+    def log(self, prio, *args):
+        if self.log_level == log.level:
+            log.slog(prio, args)
+            return
+        if prio <= self.log_level:
+            msg = ""
+            for count, thing in enumerate(args):
+                msg += ' ' + str(*thing)
+            if len(msg):
+                print(msg[1:])
+
+    def debug(self, *args):
+        log.slog(log.DEBUG, args)

src/python/jw/util/Options.py

@@ -0,0 +1,168 @@
+import re
+import json
+from collections import OrderedDict
+from .log import *
+import shlex
+import traceback
+
+class Options: # export
+
+    class OrderedData:
+
+        def __init__(self, pairs = None):
+            self.__pairs = [] if pairs is None else pairs
+
+        def add(self, lhs, rhs):
+            self.__pairs.append((lhs, rhs))
+
+        def dump(self):
+            for p in self.__pairs:
+                print(p)
+
+        @property
+        def pairs(self):
+            return self.__pairs
+
+    def __parse_json(self, spec, cls):
+        spec = spec.strip()
+        if len(spec) < 3:
+            return None
+        if spec[0] != '{':
+            spec = '{' + spec + '}'
+        try:
+            return json.loads(spec, object_pairs_hook=cls)
+        except:
+            pass
+        return None
+
+    def __parse(self, opts_str, cls):
+        r = self.__parse_json(opts_str, cls)
+        if r is not None:
+            return r
+        r = cls()
+        try:
+            opt_strs = shlex.split(opts_str)
+        except Exception as e:
+            slog_m(ERR, traceback.format_exc())
+            slog(ERR, 'Failed to split options string >{}<'.format(opts_str))
+            raise
+        for opt_str in opt_strs:
+            opt_str = re.sub(r'\s*=\s*', '=', opt_str)
+            sides = opt_str.split('=')
+            lhs = sides[0].strip()
+            if not len(lhs):
+                continue
+            if self.__allowed_keys and not lhs in self.__allowed_keys:
+                raise Exception('Field "{}" not supported'.format(lhs))
+            rhs = ' '.join(sides[1:]).strip() if len(sides) > 1 else self.__true_val
+            if cls == OrderedDict:
+                r[lhs] = rhs
+            elif cls == self.OrderedData:
+                r.add(lhs, rhs)
+        return r
+
+    def __recache(self):
+        self.__list.clear()
+        self.__dict.clear()
+        for key, val in self.__data.pairs:
+            self.__list.append(key)
+            if key not in self.__dict.keys():
+                self.__dict[key] = val
+            else:
+                cur = self.__dict[key]
+                if isinstance(cur, str):
+                    cur = [cur, val]
+                elif isinstance(cur, set):
+                    cur.add(val)
+                elif isinstance(cur, list):
+                    cur.append(val)
+                else:
+                    cur = [cur, val]
+                self.__dict[key] = cur
+        self.__str = self.__str__()
+
+    def __getitem__(self, key):
+        if not key in self.__dict.keys():
+            return None
+        return self.__dict[key]
+
+    def __str__(self):
+        return ', '.join(str(p[0]) + ': ' + str(p[1]) for p in self.__data.pairs)
+
+    def __repr__(self):
+        return self.__str__()
+
+    def __format__(self, fmt):
+        return self.__str__()
+
+    def __len__(self):
+        return len(self.__data.pairs)
+
+    def __contains__(self, keys):
+        if not type(keys) in [list, set]:
+            return keys in self.__dict.keys()
+        for key in keys:
+            if not key in self.__dict.keys():
+                return False
+        return True
+
+    def __iter__(self):
+        return iter(self.__list)
+
+    def __next__(self):
+        return next(self.__list)
+
+    def __init__(self, spec=None, delimiter=',', allowed_keys=None, true_val=True):
+        self.__true_val = true_val
+        self.__allowed_keys = None
+        self.__delimiter = delimiter
+        self.__data = self.OrderedData() if spec is None else self.__parse(spec, self.OrderedData)
+        self.__dict = {}
+        #self.__dict = OrderedDict() if spec is None else self.__parse(spec, OrderedDict)
+        self.__list = []
+        self.__str = None
+        self.__recache()
+
+    def dump(self, prio, caller=None):
+        if caller is None:
+            caller = get_caller_pos()
+        for key, val in self.__data.pairs:
+            slog(prio, "{}=\"{}\"".format(key, val), caller=caller)
+
+    def keys(self):
+        return self.__dict.keys()
+
+    def items(self):
+        #return self.__dict.items()
+        return self.__data.pairs
+
+    def get(self, key, default=None, by_index=False):
+        if by_index:
+            if type(key) != int:
+                raise KeyError('Tried to get value from options string with ' +
+                        'index {} of type "{}": {}'.format(key, type(key), str(self)))
+            if key >= len(self.__data.pairs):
+                if default is not None:
+                    return default
+                raise KeyError('Tried to get value from options string with ' +
+                        'index {} of {}: {}'.format(key, len(self.__data.pairs), str(self)))
+            return self.__list[key]
+        if key in self.__dict.keys():
+            return self.__dict[key]
+        if default is not None:
+            return default
+        raise KeyError('Key "{}" is not present in options string: {}'.format(key, str(self)))
+
+    def update(self, rhs):
+        if hasattr(rhs, 'items'):
+            for key, val in rhs.items():
+                self.__dict[key] = val
+            return
+        if isinstance(rhs, str):
+            self.update(self.__parse(rhs))
+            return
+        raise Exception('Tried to update options with object of incompatible type {}'.format(type(rhs)))
+
+    def append_to(self, obj):
+        for opt in self.__list:
+            setattr(obj, opt[0], opt[1])

src/python/jw/util/Process.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from enum import Enum, Flag, auto
+from typing import List
+
+def _sigchld_handler(signum, process):
+    if not signum == signal.SIGCHLD:
+        return
+    Process.propagate_signal(signum)
+
+class Process(ABC): # export
+
+    __processes: List[Process]    = []
+
+    class State(Enum):
+        Running                   = auto()
+        Shutdown                  = auto()
+        Done                      = auto()
+
+    class Flags(Flag):
+        FailOnExitWithoutShutdown = auto()
+
+    def __init__(self):
+        self.__state = Running
+        self.__flags = Flags.FailOnExitWithoutShutdown
+        if len(self.__processes) == 0:
+            self._signals().add_handler(signals.SIGCHLD, _sigchld_handler)
+        self.__processes.add(self)
+
+    @classmethod
+    def propagate_signal(cls, signum):
+        for p in cls.__processes:
+            p.__signal(signum)
+
+    def signal(self, signum):
+        if signum == signals.SIGCHLD:
+            self.exited()
+
+    @abstractmethod
+    def _pid(self):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def signals(cls):
+        pass
+
+    # to be reimplemented
+    def _request_shutdown(self):
+        pass
+
+    # to be reimplemented
+    def name(self):
+        return str(self._pid())
+
+    def request_shutdown(self):
+        if not self.__state == Shutdown:
+            self.__state = Shutdown
+            self._request_shutdown()
+
+    def exited(self):
+        if self.__state == Process.State.Running:
+            slog(ERR, 'process "{}" exited unexpectedly'.format(process.name()))
+            if __flags & Process.Flags.FailOnExitWithoutShutdown:
+                slog(ERR, 'exiting')
+                exit(1)
+        self.__state = Process.State.Done
+        self.__processes.erase(self)
+        if len(self.__processes) == 0:
+            self._signals().remove_handler(signals.SIGCHLD) # FIXME: broken logic

src/python/jw/util/RedirectStdIO.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+
+import os, io, sys, traceback
+from fcntl import fcntl, F_GETFL, F_SETFL
+
+class RedirectStdIO: # export
+
+    def __init__(self, stderr='on', stdout='off'):
+        self.__stderr = stderr
+        self.__stdout = stdout
+        # TODO: arguments not fully implemented,
+        #       Add support for 'off', 'on', 'stdxxx' (redirection)
+        pass
+
+    def __enter__(self):
+        if self.__stdout == 'off':
+            rfd, wfd = os.pipe()
+            flags = fcntl(rfd, F_GETFL)
+            fcntl(rfd, F_SETFL, flags | os.O_NONBLOCK)
+            self.rfile = io.open(rfd, 'r')
+            self.fake_stdout_stream = io.open(wfd, 'w')
+            self.real_stdout_fd = os.dup(1)
+            #os.close(1)
+            os.dup2(wfd, 1)
+
+    def __exit__(self, type, value, traceback):
+        if self.__stdout == 'off':
+            sys.stdout.flush()
+            os.dup2(self.real_stdout_fd, 1)
+            if type is not None:
+                    #print("-------- Error while stdio was suppressed --------")
+                    #traceback.print_stack()
+                    #print(traceback)
+                    print("-------- Captured output --------")
+                    print(*self.rfile.readlines())
+                    self.rfile.close()
+                    #print('type = ' + str(type))
+                    #print('value = ' + str(value))
+                    raise type(value)

src/python/jw/util/Signals.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+
+from typing import Dict, Callable
+from abc import ABC, abstractmethod
+
+_handled_signals: Dict[int, Callable] = {}
+
+def _signal_handler(signal, frame):
+    if not signal in _handled_signals.keys():
+        return
+    for h in _handled_signals[signal]:
+        h.func(signal, *h.args)
+
+class Signals:
+
+    class Handler:
+        def __init__(self, func, args):
+            self.func = func
+            self.args = args
+
+    def __init(self):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def _add_handler(self, signal, handler):
+        raise Exception("_add_handler() is not reimplemented")
+
+    @classmethod
+    def add_handler(cls, signals, handler, *args):
+        for signal in signals:
+            h = Signals.Handler(handler, args)
+            if not signal in _handled_signals.keys():
+                _handled_signals[signal] = [h]
+                cls._add_signal_handler(signal, _signal_handler)
+            else:
+                _handled_signals[signal].add(h)

src/python/jw/util/StopWatch.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+
+from datetime import datetime
+
+from .log import *
+
+class StopWatch: # export
+
+    def __init__(self, name=''):
+        self.__start = datetime.now()
+        self.__last = self.__start
+        self.name = name
+
+    def reset(self):
+        self.__start = datetime.now()
+
+    def logDelta(self, prio, *args, **kwargs):
+        now = datetime.now()
+        if args is not None:
+            msg = ' '.join(args)
+        else:
+            msg = '------------------ '
+        caller = kwargs['caller'] if 'caller' in kwargs.keys() else get_caller_pos(1)
+        slog(prio, '{} {} {}'.format(self.name, str(now - self.__last), msg), caller=caller)
+        self.__last = now

src/python/jw/util/__init__.py

@@ -0,0 +1,3 @@
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

src/python/jw/util/algo/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-mod.mk

src/python/jw/util/algo/ShuntingYard.py

@@ -0,0 +1,341 @@
+# -*- coding: utf-8 -*-
+
+import re, shlex
+from collections import namedtuple
+
+from ..log import *
+
+L, R = 'Left Right'.split()
+ARG, KEYW, QUOTED, LPAREN, RPAREN = 'arg kw quoted ( )'.split()
+
+class Operator: # export
+
+    def __init__(self, func=None, nargs=2, precedence=3, assoc=L):
+        self.func = func
+        self.nargs = nargs
+        self.prec = precedence
+        self.assoc = assoc
+
+class Stack:
+
+    def __init__(self, itemlist=[]):
+        self.items = itemlist
+
+    def __repr__(self):
+        return str(self.items)
+
+    def isEmpty(self):
+        if self.items == []:
+            return True
+        return False
+
+    def peek(self):
+        return self.items[-1:][0]
+
+    def pop(self):
+        return self.items.pop()
+
+    def push(self, item):
+        self.items.append(item)
+        return 0
+
+class ShuntingYard(object): # export
+
+    def __init__(self, operators = None):
+        self.do_debug = prio_gets_logged(DEBUG)
+        self.__ops = {}
+        if operators is not None:
+            for k, v in operators.items():
+                self.add_operator(k, v.func, v.nargs, v.prec, v.assoc)
+
+    def debug(self, *args):
+        if self.do_debug:
+            msg = ""
+            for count, thing in enumerate(args):
+                msg += ' ' + str(thing)
+            if len(msg):
+                slog(DEBUG, msg[1:], caller=get_caller_pos())
+
+    def operator(self, key: str) -> Operator:
+        return self.__ops[key]
+
+    def token_string(self):
+        r = ""
+        for k in sorted(self.__ops):
+            v = self.__ops[k]
+            buf = ", \"" + k
+            if v.nargs == 1:
+                if k[len(k)-1].isalnum():
+                    buf = buf + ' '
+                buf = buf + "xxx"
+            buf = buf + "\""
+            r = r + buf
+
+        if len(r):
+            return r[2:]
+        return r
+
+    def tokenize(self, spec):
+
+        regex = ""
+        for k in self.__ops.keys():
+            regex = regex + "|" + re.escape(k)
+
+        regex = regex[1:]
+
+        scanner = re.Scanner([
+          (regex,       lambda scanner,token:(KEYW,   token)),
+          (r"\"[^\"]*\"|'[^']*'",  lambda scanner,token:(QUOTED, token[1:-1])),
+          (r"[^\s()]+", lambda scanner,token:(ARG,    token)),
+          (r"\s+",      None), # None == skip token.
+        ])
+
+        tokens, remainder = scanner.scan(spec)
+        if len(remainder)>0:
+            raise Exception("Failed to tokenize " + spec + ", remaining bit is ", remainder)
+
+        #self.debug(tokens)
+        return tokens
+        r = []
+        for e in tokens:
+            if e[0] == "quoted":
+                r.append(e[1][1:-1])
+            else:
+                r.append(e[1])
+
+        return r
+
+    def add_operator(self, name, func, nargs, precedence, assoc):
+        self.__ops[name] = Operator(func, nargs, precedence, assoc)
+
+    def infix_to_postfix(self, infix):
+        tokenized = self.tokenize(infix)
+        self.debug("tokenized = ", tokenized)
+        outq, stack = [], []
+        table = ['TOKEN,ACTION,RPN OUTPUT,OP STACK,NOTES'.split(',')]
+        for toktype, token in tokenized:
+            self.debug("Checking token", token)
+            note = action = ''
+            if toktype in [ ARG, QUOTED ]:
+                action = 'Add arg to output'
+                outq.append(token)
+                table.append( (token, action, outq, (s[0] for s in stack), note) )
+            elif toktype == KEYW:
+                val = self.__ops[token]
+                t1, op1 = token, val
+                v = t1
+                note = 'Pop ops from stack to output'
+                while stack:
+                    t2, op2 = stack[-1]
+                    if (op1.assoc == L and op1.prec <= op2.prec) or (op1.assoc == R and op1.prec < op2.prec):
+                        if t1 != RPAREN:
+                            if t2 != LPAREN:
+                                stack.pop()
+                                action = '(Pop op)'
+                                outq.append(t2)
+                            else:
+                                break
+                        else:
+                            if t2 != LPAREN:
+                                stack.pop()
+                                action = '(Pop op)'
+                                outq.append(t2)
+                            else:
+                                stack.pop()
+                                action = '(Pop & discard "(")'
+                                table.append( (v, action, outq, (s[0] for s in stack), note) )
+                                break
+                        table.append( (v, action, (outq), (s[0] for s in stack), note) )
+                        v = note = ''
+                    else:
+                        note = ''
+                        break
+                    note = ''
+                note = ''
+                if t1 != RPAREN:
+                    stack.append((token, val))
+                    action = 'Push op token to stack'
+                else:
+                    action = 'Discard ")"'
+                table.append( (v, action, (outq), (s[0] for s in stack), note) )
+        note = 'Drain stack to output'
+        while stack:
+            v = ''
+            t2, op2 = stack[-1]
+            action = '(Pop op)'
+            stack.pop()
+            outq.append(t2)
+            table.append( (v, action, outq, (s[0] for s in stack), note) )
+            v = note = ''
+        if self.do_debug:
+            maxcolwidths = [len(max(x, key=len)) for x in [zip(*table)]]
+            caller = get_caller_pos()
+            row = table[0]
+            slog(DEBUG, ' '.join('{cell:^{width}}'.format(width=width, cell=cell) for (width, cell) in zip(maxcolwidths, row)))
+            for row in table[1:]:
+                slog(DEBUG, ' '.join('{cell:<{width}}'.format(width=width, cell=cell) for (width, cell) in zip(maxcolwidths, row)))
+        return table[-1][2]
+
+    def infix_to_postfix_orig(self, infix):
+
+        s = Stack()
+        r = []
+        tokens = self.tokenize(infix)
+
+        for tokinfo in tokens:
+
+            self.debug(tokinfo)
+            toktype, token = tokinfo[0], tokinfo[1]
+
+            self.debug("Checking token ", token)
+
+            if token not in self.__ops.keys():
+                r.append(token)
+                continue
+
+            if token == '(':
+                s.push(token)
+                continue
+
+            if token == ')':
+                topToken = s.pop()
+                while topToken != '(':
+                    r.append(topToken)
+                    topToken = s.pop()
+                continue
+
+            while (not s.isEmpty()) and (self.__ops[s.peek()].prec >= self.__ops[token].prec):
+                #self.debug(token)
+                r.append(s.pop())
+                #self.debug(r)
+
+            s.push(token)
+            self.debug((s.peek()))
+
+        while not s.isEmpty():
+            opToken = s.pop()
+            r.append(opToken)
+            #self.debug(r)
+
+        return r
+        #return " ".join(r)
+
+    def eval_postfix(self, postfixexpr):
+
+        self.debug("postfix = ", postfixexpr)
+        vals = Stack()
+
+        for token in postfixexpr:
+
+            self.debug("Evaluating token %s" % (token))
+            if token not in self.__ops.keys():
+                vals.push(token)
+                continue
+
+            op = self.__ops[token]
+            args = []
+            self.debug(f'Adding {op.nargs} arguments: {vals}')
+            for i in range(0, op.nargs):
+                self.debug("Adding argument %d" % (i))
+                args.append(vals.pop())
+            #self.debug("running %s(%s)" % (token, ', '.join(reversed(args))))
+            val = op.func(*reversed(args))
+            self.debug("%s(%s) = %s" % (token, ', '.join(map(str, reversed(args))), val))
+            vals.push(val)
+
+        return vals.pop()
+
+    def eval(self, infix):
+        if not isinstance(infix, str):
+            return infix
+        postfix = self.infix_to_postfix(infix)
+        self.debug(f'"{infix}" --> {postfix}')
+        for token in postfix:
+            self.debug("Token is %s" % (token))
+        return self.eval_postfix(postfix)
+
+if __name__ == '__main__':
+
+    # ------------- testbed calculator
+
+    from locale import atof
+
+    class Calculator(ShuntingYard):
+
+        #def tokenize(self, string):
+        #    return string.split()
+
+        def f_mult(self, a, b):
+            return str(atof(a) * atof(b));
+
+        def f_div(self, a, b):
+            return str(atof(a) / atof(b));
+
+        def f_add(self, a, b):
+            return str(atof(a) + atof(b));
+
+        def f_sub(self, a, b):
+            return str(atof(a) - atof(b));
+
+        def __init__(self):
+            Op = Operator
+            operators = {
+                '^': Op(None,        2, 4, R),
+                '*': Op(self.f_mult, 2, 3, L),
+                '/': Op(self.f_div,  2, 3, L),
+                '+': Op(self.f_add,  2, 2, L),
+                '-': Op(self.f_sub,  2, 2, L),
+                '(': Op(None,        0, 9, L),
+                ')': Op(None,        0, 0, L),
+            }
+            super(Calculator, self).__init__(operators)
+
+    rr = Calculator().eval("( 2 * 3 + 4 * 5 ) / ( 5 - 3 )")
+    print("Result =", rr)
+
+    # ------------- testbed match object
+
+    Object = namedtuple("Object", [ "Name", "Label" ])
+
+    class Matcher(ShuntingYard):
+
+        def f_is_name(self, a):
+            if obj.Name == a:
+                return "True"
+            return "False"
+
+        def f_matches_label(self, a):
+            if re.match(a, obj.Label):
+                return "True"
+            return "False"
+
+        def f_is_not(self, a):
+            if a == "True":
+                return "False"
+            return "True"
+
+        def f_and(self, a_, b_):
+            a = a_ == "True"
+            b = b_ == "True"
+            if a and b:
+                return "True"
+            return "False"
+
+        def __init__(self, obj):
+            Op = Operator
+            operators = {
+                '(':       Op(None,                  2, 9, L),
+                ')':       Op(None,                  2, 0, L),
+                'name=':   Op(self.f_is_name,        1, 3, R),
+                'and':     Op(self.f_and,            2, 3, L),
+                'label~=': Op(self.f_matches_label,  1, 3, R),
+                'False':   Op(None,                  0, 3, L),
+                'True':    Op(None,                  0, 3, L),
+                'not':     Op(self.f_is_not,         1, 3, R),
+            }
+            super(Matcher, self).__init__(operators)
+
+    obj = Object("hans", "wurst")
+
+    r = Matcher(obj).eval("name=hans and (not label~=worst)")
+    print("Result =", r)

src/python/jw/util/asyncio/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-mod.mk

src/python/jw/util/asyncio/Process.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+
+from abc import abstractmethod
+
+from ..Process import Process as ProcessBase
+from .Signals import Signals
+
+class Process(ProcessBase): # export
+
+    __signals = Signals()
+
+    def __init__(self, aio):
+        super().__init()
+        self.aio = aio
+
+    @classmethod
+    @abstractmethod
+    def signals(cls):
+        return cls.__signals
+
+    def wait(self):
+        return self.aio.wait()

src/python/jw/util/asyncio/ShellCmd.py

@@ -0,0 +1,108 @@
+import asyncio
+from ..log import *
+
+# FIXME: Derive this from Process, or merge the classes entirely
+
+class ShellCmd: # export
+
+    class SubprocessProtocol(asyncio.SubprocessProtocol):
+
+        def __init__(self, process, name):
+            self.process = process
+            self.name = name
+            super().__init__()
+
+        def pipe_data_received(self, fd, data):
+            stream = "stdout" if fd == 1 else ("stderr" if fd == 2 else str(fd))
+            tag = stream + '@' + self.name
+            data = data.decode().rstrip('\n')
+            prio = WARNING if fd == 2 else INFO
+            for line in data.split('\n'):
+                slog(prio, "[%s] %s" % (tag, line.rstrip('\r\n')))
+
+        def process_exited(self):
+            slog(DEBUG, "[%s] process exited" % (self.name))
+            super().process_exited()
+            self.process.exited()
+
+    class ShutdownState:
+        Running     = 1
+        Triggered   = 2
+        Completed   = 3
+        Unnecessary = 4
+
+    def __init__(self, cmdline, eloop=None, name=None):
+        if eloop is None:
+            eloop = asyncio.get_running_loop()
+        self.__eloop = eloop
+        self.__cmdline = cmdline
+        self.__name = name if name is not None else cmdline[0]
+        self.__transport = None
+        self.__protocol = None
+        self.__proc = None
+        self.__rc = None
+        self.__task = None
+        self.__shutdown = self.ShutdownState.Unnecessary
+
+    async def __exec(self):
+
+        def format_cmdline(arr):
+            r = ''
+            for tok in arr:
+                if re.search(' ', tok):
+                    r += ' "%s"' % tok
+                    continue
+                r += ' ' + tok
+            return r[1:]
+
+        try:
+            slog(INFO, "Running shell command [{}]: {}".format(self.__name, format_cmdline(self.__cmdline)))
+            self.__transport, self.__protocol = await self.__eloop.subprocess_exec(
+                lambda: self.SubprocessProtocol(self, self.__name),
+                *self.__cmdline,
+            )
+            self.__proc = self.__transport.get_extra_info('subprocess') # Popen instance
+        except:
+            slog(ERR, "Failed to run process [{}]".format(self.__name))
+            raise
+
+    def __reap(self):
+        if self.__rc is None and self.__transport:
+            self.__transport = None
+            self.__rc = self.__proc.wait()
+
+    # to be called from SubprocessProtocol / SIGCHLD handler
+    def exited(self):
+        slog(DEBUG, "Process {} exited".format(self.__name))
+        self.__reap()
+
+    async def __cleanup(self):
+        pid = self.__reap()
+        sd_fine = self.__shutdown in [ self.ShutdownState.Unnecessary, self.ShutdownState.Completed ]
+        if self.__rc == 0 and sd_fine:
+            slog(INFO, "The shell command [{}], pid {}, has exited cleanly".format(self.__name, self.__proc.pid))
+            self.monitor = self.console = self.__protocol = self.__task = None
+            return 0
+        slog(ERR, "The process ([{}], pid {}) has exited {}with status code {}, aborting".format(
+            self.__name, pid, "" if sd_fine else "prematurely ", self.__rc))
+        exit(1)
+
+    async def init(self):
+        self.__task = await self.__eloop.create_task(self.__exec())
+
+    async def cleanup(self):
+        await self.__cleanup()
+
+    async def run(self):
+        await self.init()
+        await self.cleanup()
+
+if __name__ == '__main__':
+    from .. import log
+    log.set_level('info')
+    async def run():
+        sp = ShellCmd([ 'echo', 'hello world!' ])
+        await sp.run()
+
+    asyncio.run(run())
+

src/python/jw/util/asyncio/Signals.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+
+import asyncio
+
+from ..Signals import Signals as SignalsBase
+
+class Signals(SignalsBase):
+
+    # reimplemented from Signals
+    @classmethod
+    def _add_handler(cls, signal, handler):
+        loop = asyncio.get_running_loop()
+        loop.add_signal_handler(signal, handler, None) # None = *args

src/python/jw/util/auth/Auth.py

@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+
+from typing import Optional, Union, Self
+
+import abc
+
+from enum import Flag, Enum, auto
+
+from ..log import *
+from ..Config import Config
+from ..misc import load_object
+
+class Access(Enum): # export
+    Read   = auto()
+    Modify = auto()
+    Create = auto()
+    Delete = auto()
+
+class ProjectFlags(Flag): # export
+    NoFlags      = auto()
+    Contributing = auto()
+    Active       = auto()
+
+class Group: # export
+
+    def __repr__(self):
+        return f'Group({self.name})'
+
+    @abc.abstractmethod
+    def _name(self) -> str:
+        pass
+
+    @property
+    def name(self) -> str:
+        return self._name()
+
+class User: # export
+
+    def __repr__(self):
+        return f'User({self.name})'
+
+    @abc.abstractmethod
+    def _name(self) -> str:
+        raise NotImplementedError
+
+    @property
+    def name(self) -> str:
+        return self._name()
+
+    def _display_name(self) -> str:
+        return self._name()
+
+    @property
+    def display_name(self) -> str:
+        return self._display_name()
+
+    @abc.abstractmethod
+    def _groups(self) -> list[Group]:
+        raise NotImplementedError
+
+    @property
+    def groups(self) -> list[Group]:
+        return self._groups()
+
+    @abc.abstractmethod
+    def _email(self) -> str:
+        raise NotImplementedError
+
+    @property
+    def email(self) -> str:
+        return self._email()
+
+class Auth(abc.ABC): # export
+
+    @classmethod
+    def load(cls, conf: Config, tp: str='') -> Self:
+        if tp == '':
+            val = conf.get('type')
+            if val is None:
+                msg = f'No type specified in auth configuration'
+                conf.dump(ERR, msg)
+                raise Exception(msg)
+            tp = val
+        return load_object(f'jwutils.auth.{tp}.Auth', Auth, 'Auth', conf)
+
+    def __init__(self, conf: Config):
+        self.__conf = conf
+        self.__base_user_by_email: Optional[dict[str, User]] = None
+
+    @property
+    def conf(self):
+        return self.__conf
+
+    @abc.abstractmethod
+    def _access(self, what: str, access_type: Optional[Access], who: User|Group|None) -> bool:
+        raise NotImplementedError
+
+    def access(self, what: str, access_type: Optional[Access]=None, who: Optional[Union[User|Group]]=None) -> bool:
+        return self._access(what, access_type, who)
+
+    @abc.abstractmethod
+    def _current_user(self) -> User:
+        raise NotImplementedError
+
+    @property
+    def current_user(self) -> User:
+        return self._current_user()
+
+    @abc.abstractmethod
+    def _user(self, name) -> User:
+        raise NotImplementedError
+
+    def user(self, name) -> User:
+        return self._user(name)
+
+    @abc.abstractmethod
+    def _users(self) -> dict[str, User]:
+        raise NotImplementedError
+
+    def _user_by_email(self, email: str) -> User:
+        if self.__base_user_by_email is None:
+            ret: dict[str, User] = dict()
+            users = self._users()
+            for user in users.values():
+                ret[user.email] = user
+            self.__base_user_by_email = ret
+        return self.__base_user_by_email[email]
+
+    def user_by_email(self, email) -> User:
+        return self._user_by_email(email)
+
+    @property
+    def users(self) -> dict[str, User]:
+        return self._users()
+
+    @abc.abstractmethod
+    def _projects(self, name, flags: ProjectFlags) -> list[str]:
+        raise NotImplementedError
+
+    def projects(self, name, flags: ProjectFlags) -> list[str]:
+        return self._projects(name, flags)

src/python/jw/util/auth/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-mod.mk

src/python/jw/util/auth/dummy/Auth.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+
+from typing import Optional, Union
+
+from ...log import *
+from ... import Config
+from .. import Access
+from .. import Auth as AuthBase
+from .. import Group as GroupBase
+from .. import User as UserBase
+from .. import ProjectFlags
+
+class Group(GroupBase): # export
+
+    def __init__(self, auth: AuthBase, name: str):
+        self.__name = name
+        self.__auth = auth
+
+    def _name(self) -> str:
+        return self.__name
+
+class User(UserBase): # export
+
+    def __init__(self, auth: AuthBase, name: str, conf: Config):
+        self.__name = name
+        self.__conf = conf
+        self.__auth = auth
+        self.__groups: Optional[list[GroupBase]] = None
+        self.__email: str = conf['email']
+
+    @property
+    def conf(self):
+        return self.__conf
+
+    def _name(self) -> str:
+        return self.__name
+
+    def _groups(self) -> list[GroupBase]:
+        if self.__groups is None:
+            name: str = ''
+            ret: list[GroupBase] = []
+            for name in self.conf['groups']:
+                ret.append(Group(self.__auth, name))
+            self.__groups = ret
+        return self.__groups
+
+    def _email(self) -> str:
+        return self.__email
+
+class Auth(AuthBase): # export
+
+    def __init__(self, conf: Config):
+        super().__init__(conf)
+        self.___users: Optional[dict[str, UserBase]] = None
+        self.__groups = None
+        self.__current_user: UserBase|None = None
+        self.__user_by_email: Optional[dict[str, UserBase]] = None
+
+    @property
+    def __users(self) -> dict[str, UserBase]:
+        if self.___users is None:
+            ret: dict[str, UserBase] = {}
+            for name in self.conf.entries('user'):
+                conf = self.conf.branch('user.' + name)
+                ret[name] = User(self, name, conf)
+            self.___users = ret
+        return self.___users
+
+    def _access(self, what: str, access_type: Optional[Access], who: User|GroupBase|None) -> bool: # type: ignore
+        slog(WARNING, f'Returning False for {access_type} access to resource {what} by {who}')
+        return False
+
+    def _user(self, name) -> UserBase:
+        return self.__users[name]
+
+    def _users(self) -> dict[str, UserBase]:
+        return self.__users
+
+    def _current_user(self) -> UserBase:
+        if self.__current_user is None:
+            self.__current_user = self._user(self.conf['current_user'])
+        return self.__current_user
+
+    def _user_by_email(self, email: str) -> UserBase:
+        if self.__user_by_email is None:
+            ret: dict[str, UserBase] = dict()
+            for user in self.__users.values():
+                ret[user.email] = user
+            self.__user_by_email = ret
+        return self.__user_by_email[email]
+
+    def _projects(self, name, flags: ProjectFlags) -> list[str]:
+        return []

src/python/jw/util/auth/dummy/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-mod.mk

src/python/jw/util/auth/ldap/Auth.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+
+from typing import Optional, Union
+
+import ldap
+
+from ...log import *
+from ...ldap import bind
+from ...Config import Config
+from .. import Access
+from .. import Auth as AuthBase
+from .. import Group as GroupBase
+from .. import User as UserBase
+from .. import ProjectFlags
+
+class Group(GroupBase): # export
+
+    def __init__(self, auth: AuthBase, name: str):
+        self.__name = name
+        self.__auth = auth
+
+    def _name(self) -> str:
+        return self.__name
+
+class User(UserBase):
+
+    def __init__(
+                     self,
+                     auth: AuthBase,
+                     name: str,
+                     cn: str,
+                     email: str
+                 ):
+
+        self.__auth = auth
+        self.__name = name
+        self.__cn = cn
+        self.__email = email
+        self.__groups: Optional[list[GroupBase]] = None
+
+    def _name(self) -> str:
+        return self.__name
+
+    def _groups(self) -> list[GroupBase]:
+        raise NotImplementedError
+
+    def _email(self) -> str:
+        return self.__email
+
+    def _display_name(self) -> str:
+        return self.__cn
+
+class Auth(AuthBase): # export
+
+    def __init__(self, conf: Config):
+        super().__init__(conf)
+        self.___users: Optional[dict[str, UserBase]] = None
+        self.___user_by_email: Optional[dict[str, User]] = None
+        self.__groups = None
+        self.__current_user: User|None = None
+        self.__user_base_dn = conf['user_base_dn']
+        self.__conn = self.__bind()
+        self.__dummy = self.load(conf, 'dummy')
+
+    def __bind(self):
+        return bind(self.conf)
+
+    @property
+    def __users(self) -> dict[str, UserBase]:
+        if self.___users is None:
+            ret: dict[str, UserBase] = {}
+            ret_by_email: dict[str, User] = {}
+            for res in self.__conn.find(
+                    self.__user_base_dn,
+                    ldap.SCOPE_SUBTREE,
+                    "objectClass=inetOrgPerson",
+                    ('uid', 'cn', 'uidNumber', 'mail', 'maildrop')
+                ):
+                try:
+                    display_name = None
+                    if 'displayName' in res[1]:
+                        cn = res[1]['displayName'][0].decode('utf-8')
+                    else:
+                        cn = res[1]['cn'][0].decode('utf-8')
+                    uid = res[1]['uid'][0].decode('utf-8')
+                    uidNumber = res[1]['uidNumber'][0].decode('utf-8')
+                    emails = []
+                    #for attr in ['mail', 'maildrop']:
+                    for attr in ['mail']:
+                        if attr in res[1]:
+                            for entry in res[1][attr]:
+                                emails.append(entry.decode('utf-8'))
+                    if not emails:
+                        slog(DEBUG, f'No email for user "{uid}", skipping')
+                        continue
+                    user = User(self, name=uid, cn=cn, email=emails[0])
+                    ret[uid] = user
+                    for email in emails:
+                        ret_by_email[email] = user
+                except Exception as e:
+                    slog(WARNING, f'Exception {e}')
+                    raise
+                    continue
+            for dummy_user in self.__dummy.users.values():
+                ret[dummy_user.name] = dummy_user
+            self.___users = ret
+            self.___user_by_email = ret_by_email
+        return self.___users
+
+    @property
+    def __user_by_email(self) -> dict[str, UserBase]:
+        if self.___user_by_email is None:
+            self.__users
+        return self.___user_by_email # type: ignore # We are sure that ___user_by_email is not None at this point
+
+    def _access(self, what: str, access_type: Optional[Access], who: User|GroupBase|None) -> bool: # type: ignore
+        slog(WARNING, f'Returning False for {access_type} access to resource {what} by {who}')
+        return False
+
+    def _user(self, name) -> UserBase:
+        try:
+            return self.__users[name]
+        except:
+            slog(ERR, f'No such user: "{name}"')
+            raise
+
+    def _user_by_email(self, email: str) -> UserBase:
+        return self.__user_by_email[email]
+
+    def _current_user(self) -> User:
+        raise NotImplementedError
+
+    def _users(self) -> dict[str, UserBase]:
+        return self.__users
+
+    def _projects(self, name, flags: ProjectFlags) -> list[str]:
+        if flags & ProjectFlags.Contributing:
+            # TODO: Ask LDAP
+            slog(WARNING, f'Querying LDAP for projects a user contributes to is not implemented, ignoring')
+        return []

src/python/jw/util/auth/ldap/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-mod.mk

src/python/jw/util/cast.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+
+import pytimeparse, os
+from datetime import datetime, timedelta
+from collections import OrderedDict
+
+from .log import *
+
+_int_chars = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
+
+def _strip(s_, throw=True, log_level=ERR):
+    s = s_.strip()
+    if len(s) != 0:
+        return s
+    msg = f'Tried to strip empty string "{s_}" to int'
+    if throw:
+        raise Exception(msg)
+    slog(log_level, msg)
+    return None
+
+def cast_str_to_timedelta(s_: str, throw=True, log_level=DEBUG): # export
+    s = _strip(s_, throw=throw, log_level=log_level)
+    try:
+        return (True, timedelta(seconds=pytimeparse.parse(s_)))
+    except Exception as e:
+        msg = f'Could not convert string "{s_}" to time ({e})'
+        if throw:
+            raise Exception(msg)
+        slog(log_level, msg)
+        return (False, None)
+
+def cast_str_to_int(s_: str, throw=True, log_level=DEBUG): # export
+    s = _strip(s_, throw=throw, log_level=log_level)
+    if s[0] == '-':
+        s = s[1:]
+    for c in s:
+        if not c in _int_chars:
+            break
+        else:
+            return (True, int(s_))
+    msg = f'Could not convert string "{s_}" to int'
+    if throw:
+        raise Exception(msg)
+    slog(log_level, msg)
+    return (False, None)
+
+def cast_str_to_bool(s_: str, throw=True, log_level=DEBUG): # export
+    s = _strip(s_, throw=throw, log_level=log_level).lower()
+    if s in ['true', 'yes', '1']:
+        return (True, True)
+    if s in ['false', 'no', '0']:
+        return (True, False)
+    msg = f'Could not convert string "{s_}" to bool'
+    if throw:
+        raise Exception(msg)
+    slog(log_level, msg)
+    return (False, None)
+
+_str_cast_functions = OrderedDict({
+        bool: cast_str_to_bool,
+        int: cast_str_to_int,
+        timedelta: cast_str_to_timedelta
+
+})
+
+def guess_type(s: str, default=None, log_level=DEBUG, throw=False): # export
+    if s is None:
+        raise Exception('None string passed to guess_type()')
+    for tp, func in _str_cast_functions.items():
+        try:
+            success, value = func(s, log_level=OFF, throw=False)
+            if success:
+                return tp
+        except:
+            continue
+    msg = f'Failed to guess type of string "{s}"'
+    if throw:
+        raise Exception(msg)
+    slog(log_level, msg)
+    return default
+
+def from_str(s: str, target_type=None, default_type=None, throw=True, log_level=WARNING, caller=None): # export
+    if target_type is None:
+        target_type = guess_type(s, default_type)
+        if target_type is None:
+            msg = f'Could not deduce type to cast to from string "{s}"'
+            if throw:
+                raise Exception(msg)
+            slog(log_level, msg)
+            return None
+    result = _str_cast_functions[target_type](s, throw=throw, log_level=log_level)
+    if result[0]:
+        return result[1]
+    msg = f'Failed to cast string "{s}" to type {target_type}'
+    if throw:
+        raise Exception(msg)
+    slog(log_level, msg)
+    return None
+
+def from_env(key: str, default=None, target_type=None, default_type=None, throw=True, log_level=WARNING, caller=None): # export
+    val = os.getenv(key)
+    if val is None:
+        return default
+    if target_type is None and default is not None:
+        target_type = type(default)
+    return from_str(val, target_type=target_type, default_type=default_type, throw=throw, log_level=log_level, caller=caller)
+
+# deprecated name
+def cast_str(s: str, target_type=None, default_type=None, throw=True, log_level=WARNING, caller=None):
+    return from_str(s, target_type=target_type, default_type=None, throw=True, log_level=WARNING, caller=None)

src/python/jw/util/db/DataBase.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+
+from typing import Any
+
+import abc
+from contextlib import contextmanager
+
+from ..Config import Config
+from .schema.Schema import Schema
+from ..Cmds import Cmds
+from .Session import Session
+from ..log import *
+
+class DataBase(abc.ABC):
+
+    def __init__(self, schema: Schema, conf: Config) -> None:
+        self.__conf = conf
+        self.__schema = schema
+        conf.dump(NOTICE, "Initializing database with configuration")
+
+    @abc.abstractmethod
+    def _create_session(self):
+        pass
+
+    def _delete_session(self, session: Session):
+        del session
+
+    @property
+    def schema(self):
+        return self.__schema
+
+    @property
+    def conf(self):
+        return self.__conf
+
+    @contextmanager
+    def session(self):
+        ret = self._create_session()
+        try:
+           yield ret
+        finally:
+            self._delete_session(ret)

src/python/jw/util/db/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-mod.mk

src/python/jw/util/db/Session.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+
+import abc
+
+class Session(abc.ABC): # export
+
+    def __init__(self, db):
+        self.__db = db
+
+    @property
+    def db(self):
+        return self.__db

src/python/jw/util/db/TableIoHandler.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+
+from typing import Any, List, Union, Optional, Dict
+from abc import ABC, abstractmethod
+import re, csv, json
+
+from ..log import *
+from ..cast import cast_str
+from .schema.Schema import Schema
+
+from .rows import *
+
+TType = Union[Any, Dict[str, Any]]
+
+class TableIoHandler(ABC): # export
+
+    def __init__(self, schema: Schema):
+        self.__table_meta = None
+        self.__schema = schema
+
+    @property
+    def _table_meta(self):
+        if self.__table_meta is None:
+            self.__table_meta = self.__schema.table_by_model_name(
+                    self.__class__.__name__, throw=True)
+        return self.__table_meta
+
+    @property
+    def _table_name(self):
+        return self._table_meta.name
+
+    @property
+    def _primary_keys(self):
+        return self._table_meta.primary_keys
+
+    def _check_non_nullable(self, rows):
+        buf = []
+        non_nullable = self.__table_meta.not_null_insertible_columns
+        try:
+            rows_check_not_null(rows, non_nullable, buf=buf)
+        except:
+            cn = self.__class__.__name__
+            tn = self._table_name
+            d = '========================================================='
+            slog_m(ERR, f'{d} Null values in {cn}\n')
+            for key in non_nullable:
+                buf = rows_check_not_null(rows, key, log_prio=OFF, throw=False)
+                if not buf:
+                    continue
+                slog_m(ERR, f'\n{d} Null values in {cn} / {tn}: "{key}"\n')
+                use_cols=self.log_columns
+                if key not in use_cols:
+                    use_cols.append(key)
+                rows_dump(buf, use_cols=use_cols, log_prio=ERR)
+                rows_dump(buf, use_cols=use_cols, out_path=f'/tmp/missing_{key}_in_{tn}.html', heading=f'Missing "{key}" in table {tn}')
+            raise
+
+    @property
+    def log_columns(self):
+        return self._table_meta.log_columns
+
+    @abstractmethod
+    def _load(self, uri: str, reference) -> TType:
+        pass
+
+    @abstractmethod
+    def _store(self, uri: str, data: TType):
+        pass
+
+    def load(self, uri: str, reference, check_duplicates=False, write_csv=None) -> TType:
+        slog(INFO, f'Reading table "{self._table_name}" from "{uri}"')
+        ret = self._load(uri, reference)
+        if check_duplicates:
+            slog(INFO, f'Checking duplicates in {self._table_name}')
+            d = rows_duplicates(ret)
+            if len(d):
+                slog(ERR, f'Duplicates {d}')
+                raise Exception("Duplicates")
+        self._check_non_nullable(ret)
+        #schema_missing_foreign_key_values(self._table_name, ret, reference)
+        return ret
+
+    def store(self, uri: str, data: TType) -> None:
+        return self._store(uri, data)

src/python/jw/util/db/query/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-mod.mk

src/python/jw/util/db/query/Queries.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+
+from typing import Any
+
+import abc
+
+from ...log import *
+from ...misc import load_classes
+from ...Cmds import Cmds
+from ..DataBase import DataBase
+from ..schema.Schema import Schema
+from .Query import Query as QueryBase
+from .QueryResult import QueryResult
+
+class Queries(abc.ABC): # export
+
+    class Query(QueryBase):
+
+        def __init__(self: Any, query: QueryBase, name: str, location: str, func: Any):
+            self.__query = query
+            self.__func = func
+            self.__location = location
+            self.__name = name
+
+        # -- implement API
+
+        def _run(self, session, *args, **kwargs) -> QueryResult:
+            return self.__func(session, *args, **kwargs)
+
+        def _register(self):
+            raise Exception('Can\'t call register on this object')
+
+        def _column_names(self) -> list[str]:
+            return self.__query.column_names
+
+        @property
+        def name(self):
+            return self.__name
+
+    def __init__(self, db: DataBase) -> None:
+        assert(isinstance(db, DataBase))
+        self.__db = db
+        self.__queries: dict[str, Any] = dict()
+
+    def __getitem__(self, name) -> Query:
+        try:
+            return self.__queries[name]
+        except Exception as e:
+            slog(ERR, f'Failed to retrieve query {name} ({e})')
+            slog_m(ERR, '\n'.join(self.__queries.keys()))
+            raise
+
+    def keys(self):
+        return self.__queries.keys()
+
+    @property
+    def db(self) -> DataBase:
+        return self.__db
+
+    def load(self, modules: list[str], cls=QueryBase):
+        for path in modules:
+            slog(INFO, f'Loading modules from {path}')
+            for c in load_classes(path, cls):
+                c(self).register()
+
+    @property
+    def schema(self) -> Schema:
+        return self.__db.schema
+
+    def add(self, query: QueryBase, query_name: str, location: str, func: Any):
+        slog(INFO, f'Adding query "{query_name}" on location "{location}"')
+        assert(isinstance(query_name, str))
+        assert(isinstance(location, str))
+        #ret = self.Query(query, func)
+        ret = self.Query(query, query_name, location, func)
+        #setattr(ret, 'name', name)
+        self.__queries[query_name] = ret
+        self._add(ret)
+
+    # To be overriden in case the derived class wants to keep track
+    def _add(self, query):
+        pass

src/python/jw/util/db/query/Query.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+
+from typing import Any
+
+import abc
+
+from ...log import *
+from ...misc import load_classes
+from ...Cmds import Cmds
+from ..DataBase import DataBase
+from ..Session import Session
+from .QueryResult import QueryResult
+#from .Queries import Queries
+
+class Query(abc.ABC): # export
+
+    def __init__(self, parent: Any) -> None:
+        self.__parent = parent
+
+    # -- pure virtuals
+
+    @abc.abstractmethod
+    def _run(self, session: Session, *args, **kwargs) -> QueryResult:
+        raise Exception('Called pure virtual _run()')
+
+    @abc.abstractmethod
+    def _register(self):
+        raise Exception('Called pure virtual _register()')
+
+    @abc.abstractmethod
+    def _column_names(self) -> list[str]:
+        raise Exception('Called pure virtual _column()')
+
+    # -- used by Queries class
+
+    def register(self):
+        return self._register()
+
+    # -- to be used
+
+    def _add(self, query_name: str, location: str, func: Any):
+        return self.__parent.add(self, query_name, location, func)
+
+    def run(self, session: Session, *args, **kwargs) -> QueryResult:
+        return self._run(session, *args, **kwargs)
+
+    @property
+    def parent(self):
+        return self.__parent
+
+    @property
+    def db(self) -> DataBase:
+        return self.__parent.db
+
+    @property
+    def schema(self):
+        return self.__parent.db.schema
+
+    @property
+    def column_names(self) -> list[str]:
+        return self._column_names()

src/python/jw/util/db/query/QueryResult.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+
+from typing import Any, Union
+
+import abc
+from enum import Enum, auto
+
+from ...log import *
+from ...Cmds import Cmds
+from ..DataBase import DataBase
+from ..Session import Session
+
+class ResType(Enum): # export
+    Statement    = auto()
+    Scalars      = auto()
+    One          = auto()
+    First        = auto()
+    Pages        = auto()
+
+class QueryResult(abc.ABC): # export
+
+    def __init__(self, session: Session, query: Any) -> None:
+        self.__query = query
+        self.__session = session
+
+    @property
+    def db(self) -> DataBase:
+        return self.__query.db
+
+    @property
+    def query(self) -> DataBase:
+        return self.__query
+
+    @property
+    def session(self) -> Session:
+        return self.__session
+
+    @property
+    def schema(self):
+        return self.__query.db.schema
+
+    def rows(self) -> list[Any]:
+        return self._cast(ResType.Scalars)
+
+    def pages(self, per_page=20, page=1) -> Any:
+        return self._cast(ResType.Pages, per_page=per_page, page=page)
+
+    def one(self) -> Any:
+        return self._cast(ResType.One)
+
+    def first(self) -> Any:
+        return self._cast(ResType.First)
+
+    @property
+    def statement(self) -> Any:
+        return self._cast(ResType.Statement)
+
+    # -- pure virtuals
+
+    @abc.abstractmethod
+    def _cast(self, res_type: ResType, **kwargs) -> Union[Any|list[Any]]:
+        pass

src/python/jw/util/db/rows.py

@@ -0,0 +1,250 @@
+# -*- coding: utf-8 -*-
+
+import io, os, re, textwrap, json, csv
+from tabulate import tabulate # type: ignore
+
+from ..log import *
+
+def rows_pretty(rows): # export
+    if type(rows) == dict:
+        rows = [rows]
+    out = []
+    for row in rows:
+        out.append(json.dumps(row, sort_keys=True, indent=4, default=str))
+    return '\n'.join(out)
+
+def rows_duplicates(rows, log_prio=INFO, caller=None): # export
+    def __equal(r1, r2):
+        for col in set(r1.keys()) | set(r2.keys()):
+            if col in r1:
+                if col not in r2:
+                    return False
+            else:
+                if col in r2:
+                    return False
+            if r1[col] != r2[col]:
+                return False
+        return True
+    ret = []
+    last = len(rows) - 1
+    i = last
+    while last > 0:
+        for i in reversed(range(0, last-1)):
+            if __equal(rows[last], rows[i]):
+                ret.append(last)
+                last -= 1
+                break
+        last -= 1
+    return ret
+
+def rows_remove(rows, callback=None, candidates=None, log_prio=INFO, caller=None): # export
+
+    def __is_remove_candidate(row):
+        for remove_row in candidates:
+            for col, val in row.items():
+                if not col in remove_row.keys():
+                    break
+                if val != remove_row[col]:
+                    break
+            else:
+                return True
+        return False
+
+    if caller is None:
+        caller = get_caller_pos()
+    if callback is None:
+        if candidates is not None:
+            callback = __is_remove_candidate
+        else:
+            raise Exception('No criterion to remove rows')
+    remove = list()
+    index = -1
+    for row in rows:
+        index += 1
+        if callback(row):
+            remove.append(index)
+            continue
+    for index in reversed(remove):
+        slog(log_prio, f'Removing row {rows[index]}', caller=caller)
+        del rows[index]
+
+def rows_select(rows, rules): # export
+    ret = []
+    for row in rows:
+        for rule in rules:
+            if type(rule) == tuple():
+                search_rule = rule[0]
+            else:
+                search_rule = rule
+            for col_name, expr in search_rule.items():
+                if not re.search(expr, row[col_name]):
+                    break
+            else:
+                ret.append(row)
+                break
+    return ret
+
+def rows_rewrite_regex(rows, rules): # export
+    for row in rows:
+        for rule in rules:
+            try:
+                for col_name, expr in rule[0].items():
+                    if not re.search(expr, row[col_name]):
+                        break
+                else:
+                    for exec_col_name, exec_val in rule[1].items():
+                        slog(INFO, f'Rewriting {row} {row.get(exec_col_name)} -> {exec_val}')
+                        row[exec_col_name] = exec_val
+            except Exception as e:
+                slog(ERR, f'Failed to run rule {rule} against {row} ({e})')
+                raise
+
+def rows_check_not_null(rows, keys, log_prio=WARNING, buf=None, stat_key=None, throw=True, caller=None): # export
+    if type(keys) == str:
+        keys = [keys]
+    if caller is None:
+        caller = get_caller_pos()
+    count = 0
+    stats = dict()
+    if buf is None:
+        buf = []
+    else:
+        buf.clear()
+    for row in rows:
+        for key in keys:
+            if row.get(key) is None:
+                slog(log_prio, f'{key} is missing in row {row}', caller=caller)
+                buf.append(row)
+                if stat_key is not None:
+                    stat_val = row[stat_key]
+                    if not stat_val in stats.keys():
+                        stats[stat_val] = 0
+                    stats[stat_val] += 1
+                count += 1
+                break
+    if count > 0:
+        if stat_key is not None:
+            i = 0
+            for k, v in reversed(sorted(stats.items(), key=lambda item: item[1])):
+                i += 1
+                slog(ERR, f'{i:>3}. {k:<23}: {v}', caller=caller)
+        if throw:
+            raise Exception(f'Found {count} rows violating null-constraint for keys {keys}')
+    return buf
+
+def rows_dumps(rows, log_prio=INFO, caller=None, use_cols=None, skip_cols=None, table_name=None, out_path='log', heading=None, lead=None, tablefmt=None): # export
+
+    headers = 'keys'
+    dump_rows = rows
+    if use_cols is not None:
+        #dump_rows = {col: rows[col] for col in use_cols}
+        new_dump_rows = []
+        for row in dump_rows:
+            new_dump_rows.append({col: row.get(col) for col in use_cols})
+        dump_rows = new_dump_rows
+    if skip_cols is not None:
+        new_dump_rows = []
+        for row in dump_rows:
+            new_row = {}
+            for col, val in row.items():
+                if col in skip_cols:
+                    continue
+                new_row[col] = val
+            new_dump_rows.append(new_row)
+        dump_rows = new_dump_rows
+    out = header = footer = ""
+    match tablefmt:
+        case 'html':
+            if heading is not None:
+                heading = f'<h1>{heading}</h1>\n'
+            if type(lead) == str:
+                lead = f'<div class="lead">\n    {lead}\n</div>\n'
+            elif type(lead) == list:
+                l = '<ul>\n'
+                for li in lead:
+                    l += f'<li>{li}</li>\n'
+                l += '</ul>\n'
+                lead = l
+            header=textwrap.dedent('''\
+                <html>
+                    <head>
+                        <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
+                        <script>
+                            $(document).ready(function()
+                            {
+                                $("tr:odd").css({ "background-color":"#ffa"});
+                            });
+                        </script>
+                        <style>
+                            tbody td {
+                                /* padding: 30px; */
+                            }
+                            tbody tr:nth-child(odd) {
+                                background-color: #4C8BF5;
+                            }
+                        </style>
+                    </head>
+                    <body>
+                ''')
+            footer = textwrap.dedent('''
+                    </body>
+                </html>
+                ''')
+        case _:
+            if type(heading) == str:
+                heading = '\n' + heading
+            if type(lead) == str:
+                pass
+            elif type(lead) == list:
+                l =''
+                for li in lead:
+                    l += f'  - {li}\n'
+                lead = '\n\n' + l + '\n'
+
+    if heading is None:
+        heading = ''
+    if lead is None:
+        lead = ''
+
+    return header + heading + lead + tabulate(dump_rows, headers=headers, tablefmt=tablefmt) + footer
+
+def rows_dump(rows, log_prio=INFO, caller=None, use_cols=None, skip_cols=None, table_name=None, out_path='log', heading=None, lead=None, tablefmt=None): # export
+
+    if not prio_gets_logged(log_prio):
+        return
+
+    if caller is None:
+        caller = get_caller_pos()
+    if tablefmt is None and out_path:
+        tablefmt = os.path.splitext(out_path)[1][1:]
+
+    out = rows_dumps(rows, log_prio=log_prio, caller=caller, use_cols=use_cols, skip_cols=skip_cols, table_name=table_name, heading=heading, lead=lead, tablefmt=tablefmt)
+
+    match out_path:
+        case 'log':
+            slog_m(log_prio, out, caller=caller)
+        case _:
+            with open(out_path, 'w') as fp:
+                fp.write(out)
+
+def rows_to_csv(rows, use_tmpfile=False): # export
+    def __write(rows, out):
+        writer = csv.DictWriter(out, fieldnames=field_names, delimiter=';', quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
+        writer.writeheader()
+        for row in rows:
+            writer.writerow(row)
+
+    field_names = []
+    for row in rows:
+        for col in row.keys():
+            if col not in field_names:
+                field_names.append(col)
+    if True:
+        out = io.StringIO()
+        __write(rows, out)
+        return out.getvalue()
+    import tempfile
+    with tempfile.TemporaryFile(mode='w', newline='') as out:
+        __write(rows, out)
+        out.seek(0)
+        return out.read()

src/python/jw/util/db/schema/Column.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+
+from typing import Optional, Any
+
+import abc
+
+from .DataType import DataType
+from ...log import *
+
+class Column(abc.ABC): # export
+
+    def __init__(self, table, name, data_type: DataType):
+        self.__name: str = name
+        self.__table: Any = table
+        self.__is_nullable: Optional[bool] = None
+        self.__is_null_insertible: Optional[bool] = None
+        self.__is_primary_key: Optional[bool] = None
+        self.__default_value: Optional[Any] = None
+        self.__default_value_cached: bool = False
+        self.__is_auto_increment: Optional[bool] = None
+        self.__translate: Optional[bool] = None
+        self.__data_type: DataType = data_type
+        self.__foreign_keys: Optional[Any] = None
+        self.__foreign_keys_cached: bool = False
+        self.__foreign_keys_by_table: Optional[dict[str, Any]] = None
+
+    def __repr__(self):
+        return f'{self.__table.name}.{self.__name}: {self.__data_type}'
+
+    def __eq__(self, rhs) -> bool:
+        if isinstance(rhs, Column):
+            if self.__table != rhs.__table:
+                return False
+            if self.__name != rhs.__name:
+                return False
+            return True
+        elif isinstance(rhs, str):
+            if self.__name != rhs:
+                return False
+            return True
+        throw(ERR, f'Tried to compare column {self} to type {type(rhs)}: {rhs}')
+        return False # Unreachable but requested by mypy
+
+    @property
+    def name(self) -> str:
+        return self.__name
+
+    @property
+    def data_type(self):
+        return self.__data_type
+
+    @property
+    def table(self) -> str:
+        return self.__table
+
+    @property
+    def is_nullable(self) -> bool:
+        if self.__is_nullable is None:
+            self.__is_nullable = self.__name in self.__table.nullable_columns
+        return self.__is_nullable
+
+    @property
+    def is_null_insertible(self):
+        if self.__is_null_insertible is None:
+            ret = False
+            if self.is_nullable:
+                ret = True
+            elif self.is_auto_increment:
+                ret = True
+            elif self.default_value is not None:
+                ret = True
+            self.__is_null_insertible = ret
+        return self.__is_null_insertible
+
+    @property
+    def is_primary_key(self) -> bool:
+        if self.__is_primary_key is None:
+            self.__is_primary_key = self.__name in self.__table.primary_keys
+        return self.__is_primary_key
+
+    @property
+    def is_auto_increment(self) -> bool:
+        if self.__is_auto_increment is None:
+            self.__is_auto_increment = self.__name in self.__table.auto_increment_columns
+        return self.__is_auto_increment
+
+    @property
+    def translate(self) -> bool:
+        if self.__translate is None:
+            self.__translate = self.__name in self.__table.translate_columns
+        return self.__translate
+
+    @property
+    def default_value(self) -> Optional[Any]:
+        if self.__default_value_cached is False:
+            self.__default_value = self.__table.column_default(self.name)
+            self.__default_value_cached = True
+        return self.__default_value
+
+    # Returns Column object on parent table
+    @property
+    def foreign_keys(self) -> Optional[Any]:
+        if not self.__foreign_keys_cached:
+            fks: list[Any] = []
+            for cfk in self.__table.foreign_key_constraints:
+                for fk in cfk:
+                    if fk.child_column == self:
+                        fks.append(fk.parent_column)
+            self.__foreign_keys_cached = True
+            self.__foreign_keys = fks if fks else None
+        return self.__foreign_keys
+
+    def foreign_key(self, table) -> Optional[Any]:
+        if self.__foreign_keys_by_table is None:
+            self.__foreign_keys_by_table = dict()
+            for col in self.foreign_keys: # type: ignore # Any not iterable
+                assert(col.table.name not in self.__foreign_keys_by_table)
+                self.__foreign_keys_by_table[col.table.name] = col
+        table_name = table if isinstance(table, str) else table.name
+        return self.__foreign_keys_by_table.get(table_name)

src/python/jw/util/db/schema/ColumnSet.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+
+from typing import Optional, Iterable, Any
+
+class ColumnSet: # export
+
+    def __init__(self, *args: list[Any], columns: list[Any]=[], table: Optional[Any]=None, names: Optional[list[str]]=None):
+        self.__columns: list[Any] = [*args]
+        self.__columns.extend(columns)
+        self.__table = table
+        if names is not None:
+            assert(table is not None)
+            for name in names:
+                self.__columns.append(table.column(name))
+        if self.__table is not None:
+            for col in columns:
+                assert(col.table == self.__table)
+
+    def __len__(self):
+        return len(self.__columns)
+
+    def __iter__(self):
+        yield from self.__columns
+
+    def __repr__(self):
+        return '|'.join([col.name for col in self.__columns])
+
+    def __getitem__(self, index):
+        return self.__columns[index]
+
+    def __eq__(self, rhs):
+        if self.__table != rhs.__table:
+            return False
+        if len(self.__columns) != len(rhs.__columns):
+            return False
+        for i in range(0, len(self.__columns)):
+            if self.__columns[i].name != rhs.__columns[i].name:
+                return False
+        return True
+
+    @property
+    def columns(self) -> list[Any]:
+        return self.__columns

src/python/jw/util/db/schema/CompositeForeignKey.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+
+from typing import Optional, Any
+
+from ...log import *
+
+from .ColumnSet import ColumnSet
+from .SingleForeignKey import SingleForeignKey
+
+class CompositeForeignKey: # export
+
+    def __init__(self, child_col_set: ColumnSet, parent_col_set: ColumnSet): # TODO: Implement alternative ways to construct
+
+        def __table(s):
+            ret = None
+            for c in s:
+                if ret is None:
+                    ret = c.table
+                else:
+                    assert(ret == c.table)
+            assert(ret is not None)
+            return ret
+
+        self.__child_col_set = child_col_set
+        self.__parent_col_set = parent_col_set
+        self.__child_table = __table(child_col_set)
+        self.__parent_table = __table(parent_col_set)
+
+        assert(len(self.__child_col_set) == len(self.__parent_col_set))
+        self.__len = len(self.__child_col_set)
+        self.__column_relations: Optional[list[SingleForeignKey]] = None
+        self.__parent_columns_by_child_column: Optional[dict[str, Any]] = None
+        self.__child_columns_by_parent_column: Optional[dict[str, Any]] = None
+
+    def __table_rel_str(self):
+        return f'{self.__child_table.name} => {self.__parent_table.name}'
+
+    def __cols_rel_str(self, child, parent):
+        return f'{child.name} -> {parent.name}'
+
+    def __len__(self):
+        return self.__len
+
+    def __iter__(self):
+        yield from self.column_relations
+
+    def __repr__(self):
+        ret = self.__table_rel_str()
+        ret += ': ' + ', '.join([self.__cols_rel_str(rel.child_column, rel.parent_column) for rel in self.column_relations])
+        return ret
+
+    def __eq__(self, rhs):
+        if rhs.__child_col_set != self.__child_col_set:
+            return False
+        if rhs.__parent_col_set != self.__parent_col_set:
+            return False
+        return True
+
+    @property
+    def child_table(self) -> Any:
+        return self.__child_table
+
+    @property
+    def parent_table(self) -> Any:
+        return self.__parent_table
+
+    @property
+    def child_columns(self) -> ColumnSet:
+        return self.__child_col_set
+
+    @property
+    def parent_columns(self) -> ColumnSet:
+        return self.__parent_col_set
+
+    def parent_column(self, child_column) -> Any:
+        child_column_name = child_column if isinstance(child_column, str) else child_column.name
+        if self.__parent_columns_by_child_column is None:
+            d: dict[str, Any] = {}
+            assert(len(self.__child_col_set) == len(self.__parent_col_set))
+            for i in range(0, len(self.__child_col_set)):
+                d[self.__child_col_set[i].name] = self.__parent_col_set[i]
+            self.__parent_columns_by_child_column = d
+        return self.__parent_columns_by_child_column[child_column]
+
+    def child_column(self, parent_column) -> Any:
+        slog(WARNING, f'{self}: Looking for child column belonging to parent column "{parent_column}"')
+        parent_column_name = parent_column if isinstance(parent_column, str) else parent_column.name
+        if self.__child_columns_by_parent_column is None:
+            d: dict[str, Any] = {}
+            assert(len(self.__parent_col_set) == len(self.__child_col_set))
+            for i in range(0, len(self.__parent_col_set)):
+                d[self.__parent_col_set[i].name] = self.__child_col_set[i]
+            self.__child_columns_by_parent_column = d
+        return self.__child_columns_by_parent_column[parent_column]
+
+    @property
+    def column_relations(self) -> list[Any]:
+        ret = []
+        if self.__column_relations is None:
+            for i in range(0, self.__len):
+                ret.append(SingleForeignKey(self.__child_col_set[i], self.__parent_col_set[i]))
+            self.__column_relations = ret
+        return self.__column_relations

src/python/jw/util/db/schema/DataType.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+
+from typing import Optional
+from enum import Enum, auto
+from datetime import datetime
+
+from ...log import *
+
+class Id(Enum):
+    Integer = auto()
+    SmallInteger = auto()
+    Currency = auto()
+    Single = auto()
+    DateTime = auto()
+    String = auto()
+    Text = auto()
+    Invalid = auto()
+
+def py_type(type_id: Id) -> type: # export
+
+    match type_id:
+        case Id.Integer:
+            return int
+        case Id.SmallInteger:
+            return int
+        case Id.Currency:
+            return int
+        case Id.Single:
+            return int
+        case Id.DateTime:
+            return datetime
+        case Id.String:
+            return str
+        case Id.Text:
+            return str
+        case Id.DateTime:
+            return datetime
+
+    raise Exception(f'Unknown column type-id "{type_id}"')
+
+class DataType: # export
+
+    def __init__(self, type_id: Id, size: Optional[int]=None):
+        if not isinstance(type_id, Id):
+            throw(ERR, f'Passed type id "{type_id}" with unsupported data type {type(type_id)}')
+        if size is not None:
+            assert(isinstance(size, int))
+            assert(size > 0)
+        self.__id = type_id
+        self.__size = size
+
+    def __repr__(self):
+        ret = f'{self.__id.name}'
+        if self.__size is not None:
+            ret += f'({self.__size})'
+        return ret
+
+    def __eq__(self, rhs):
+        if self.__id != rhs.__id:
+            return False
+        if self.__size != rhs.__size:
+            return False
+        return True
+
+    @property
+    def type_id(self) -> Id:
+        return self.__id
+
+    @property
+    def size(self) -> Optional[int]:
+        return self.__size
+
+    @property
+    def py_type(self) -> type:
+        return py_type(self.__id)
+
+    @property
+    def py_type_str(self) -> str:
+        return self.py_type.__name__
+
+    @property
+    def py_type_annotation(self) -> str:
+        return self.py_type_str # FIXME: This is not always correct

src/python/jw/util/db/schema/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-mod.mk

src/python/jw/util/db/schema/Schema.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+
+from typing import Optional, Iterable
+
+import abc
+
+from ...log import *
+
+from .Table import Table
+from .Column import Column
+from .DataType import DataType
+from .CompositeForeignKey import CompositeForeignKey
+
+class Schema(abc.ABC): # export
+
+    def __init__(self) -> None:
+        self.___tables: Optional[list[Table]] = None
+        self.__foreign_keys: Optional[list[CompositeForeignKey]] = None
+        self.__access_defining_columns: Optional[list[str]] = None
+
+    @property
+    def __tables(self):
+        if self.___tables is None:
+            ret = dict()
+            for name in self._table_names():
+                slog(DEBUG, f'Caching metadata for table "{name}"')
+                assert(isinstance(name, str))
+                ret[name] = self._table(name)
+            self.___tables = ret
+        return self.___tables
+
+    # ------ API to be implemented
+
+    @abc.abstractmethod
+    def _table_names(self) -> Iterable[str]:
+        throw(ERR, "Called pure virtual base class method")
+        return []
+
+    @abc.abstractmethod
+    def _table(self, name: str) -> Table:
+        throw(ERR, "Called pure virtual base class method")
+        return None # type: ignore
+
+    @abc.abstractmethod
+    def _foreign_keys(self) -> list[CompositeForeignKey]:
+        pass
+
+    @abc.abstractmethod
+    def _access_defining_columns(self):
+        pass
+
+    @abc.abstractmethod
+    def _model_module_search_paths(self) ->  list[tuple[str, type]]:
+        pass
+
+    # ------ API to be called
+
+    def __len__(self):
+        return len(self.__tables)
+
+    def __iter__(self):
+        yield from self.__tables.values()
+
+    def __repr__(self):
+        return '|'.join([table.name for table in self.__tables])
+
+    def __getitem__(self, index):
+        return self.__tables[index]
+
+    @property
+    def table_names(self) -> Iterable[str]:
+        return self.__tables.keys()
+
+    @property
+    def tables(self) -> Iterable[Table]:
+        return self.__tables.values()
+
+    @property
+    def access_defining_columns(self):
+        if self.__access_defining_columns is None:
+            self.__access_defining_columns = self._access_defining_columns()
+        return self.__access_defining_columns
+
+    @property
+    def foreign_key_constraints(self) -> list[CompositeForeignKey]:
+        if self.__foreign_keys is None:
+            self.__foreign_keys = self._foreign_keys()
+        return self.__foreign_keys
+
+    def table(self, name: str) -> Table:
+        return self.__tables[name]
+
+    def table_by_model_name(self, name: str, throw=False) -> Table:
+        for table in self.__tables.values():
+            if table.model_name == name:
+                return table
+        if throw:
+            raise Exception(f'Table "{name}" not found in database metadata')
+        return None # type: ignore
+
+    def primary_keys(self, table_name: str) -> Iterable[str]:
+        return self.__tables[table_name].primary_keys
+
+    def columns(self, table_name: str) -> Iterable[Column]:
+        return self.__tables[table_name].columns
+
+    @property
+    def model_module_search_paths(self) ->  list[tuple[str, type]]:
+        return self._model_module_search_paths()

src/python/jw/util/db/schema/SingleForeignKey.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+
+from typing import Optional, Any
+
+from .Column import Column
+from .ColumnSet import ColumnSet
+
+class SingleForeignKey:
+
+    def __init__(self, child_col: Column, parent_col: Column):
+        self.__child_col = child_col
+        self.__parent_col = parent_col
+        self.__iterable = (self.__child_col, self.__parent_col)
+
+    def __len__(self):
+        return 2
+
+    def __iter__(self):
+        yield from self.__iterable
+
+    def __repr__(self):
+        return f'{self.__child_col.table.name}.{self.__child_col.name} -> {self.__parent_col.table.name}.{self.__parent_col.name}'
+
+    def __getitem__(self, index):
+        return self.__iterable[index]
+
+    @property
+    def child_column(self):
+        return self.__child_col
+
+    @property
+    def parent_column(self):
+        return self.__parent_col

src/python/jw/util/db/schema/Table.py

@@ -0,0 +1,469 @@
+# -*- coding: utf-8 -*-
+
+from typing import Optional, Union, Iterable, Self, Any # TODO: Need any for many things, as I can't figure out how to avoid circular imports from here
+
+import abc
+from collections import OrderedDict
+from urllib.parse import quote_plus
+
+from ...log import *
+from ...misc import load_class
+
+from .ColumnSet import ColumnSet
+from .DataType import DataType
+from .CompositeForeignKey import CompositeForeignKey
+from .Column import Column
+
+class Table(abc.ABC): # export
+
+    def __init__(self, schema, name: str):
+        assert(isinstance(name, str))
+        self.__schema = schema
+        self.__name = name
+
+        self.___columns: Optional[OrderedDict[str, Any]] = None
+        self.___foreign_key_parent_tables: Optional[OrderedDict[str, Any]]  = None
+
+        self.__primary_keys: Optional[Iterable[str]] = None
+        self.__unique_constraints: Optional[list[ColumnSet]] = None
+        self.__foreign_key_constraints: Optional[list[CompositeForeignKey]] = None
+        self.__nullable_columns: Optional[Iterable[str]]  = None
+        self.__non_nullable_columns: Optional[Iterable[str]]  = None
+        self.__null_insertible_columns: Optional[Iterable[str]]  = None
+        self.__not_null_insertible_columns: Optional[Iterable[str]]  = None
+        self.__log_columns: Optional[Iterable[str]]  = None
+        self.__edit_columns: Optional[Iterable[str]]  = None
+        self.__translate_columns: Optional[Iterable[str]]  = None
+        self.__display_columns: Optional[Iterable[str]]  = None
+        self.__default_sort_columns: Optional[Iterable[str]]  = None
+        self.__column_default: Optional[dict[str, Any]]  = None
+        self.__base_location_rule: Optional[Iterable[str]]  = None
+        self.__location_rule: Optional[Iterable[str]]  = None
+        self.__row_location_rule: Optional[Iterable[str]]  = None
+        self.__add_row_location_rule: Optional[Iterable[str]]  = None
+        self.___add_child_row_location_rules: Optional[dict[str, str]]  = None
+        self.__foreign_keys_to_parent_table: Optional[OrderedDict[str, Any]] = None
+        self.__relationships: Optional[list[tuple[str, Self]]] = None
+        self.__model_class: Optional[Any] = None
+        self.___relationship_by_foreign_table: Optional[dict[str, Self]] = None
+
+    @property
+    def __columns(self) -> OrderedDict[str, Any]:
+        if self.___columns is None:
+            ret: OrderedDict[str, Any] = OrderedDict()
+            for name in self._column_names():
+                ret[name] = Column(self, name, self._column_data_type(name))
+            self.___columns = ret
+        return self.___columns
+
+    @property
+    def __foreign_key_parent_tables(self) -> OrderedDict[str, Any]:
+        if self.___foreign_key_parent_tables is None:
+            self.___foreign_key_parent_tables = OrderedDict()
+            for cfk in self.foreign_key_constraints:
+                self.___foreign_key_parent_tables[cfk.parent_table.name] = cfk.parent_table
+        return self.___foreign_key_parent_tables
+
+    @property
+    def __relationship_by_foreign_table(self) -> dict[str, Self]:
+        if self.___relationship_by_foreign_table is None:
+            ret: dict[str, Self] = dict()
+            for member_name, table_name in self.relationships:
+                ret[member_name] = self.schema[table_name]
+            self.___relationship_by_foreign_table = ret
+        return self.___relationship_by_foreign_table
+
+    @property
+    def __add_child_row_location_rules(self) -> dict[str, str]:
+        if self.___add_child_row_location_rules is None:
+            ret: dict[str, str] = {}
+            for foreign_table_name, foreign_table in self.__relationship_by_foreign_table.items():
+                if len([self.foreign_keys_to_parent_table(foreign_table)]):
+                    rule = self._add_child_row_location_rule(foreign_table_name)
+                    if rule is None:
+                        continue
+                    ret[foreign_table_name] = rule
+            self.___add_child_row_location_rules = ret
+        return self.___add_child_row_location_rules
+
+    def __len__(self):
+        return len(self.__columns)
+
+    def __iter__(self):
+        yield from self.__columns.values()
+
+    def __repr__(self) -> str:
+        return self.__name
+
+    def __getitem__(self, index):
+        return self.__columns[index]
+
+    def __eq__(self, rhs) -> bool:
+        if isinstance(rhs, Table):
+            if self.__name != rhs.__name:
+                return False
+            return True
+        elif isinstance(rhs, str):
+            if self.__name != rhs:
+                return False
+            return True
+        throw(ERR, f'Tried to compare table {self} to type {type(rhs)}: {rhs}')
+        return False # Unreachable but requested by mypy
+
+    def __hash__(self) -> int:
+        return hash(self.name)
+
+    # -- To be reimplemented
+
+    @abc.abstractmethod
+    def _column_names(self) -> Iterable[str]:
+        pass
+
+    @abc.abstractmethod
+    def _column_data_type(self, name) -> DataType:
+        pass
+
+    @abc.abstractmethod
+    def _primary_keys(self) -> Iterable[str]:
+        pass
+
+    @abc.abstractmethod
+    def _log_columns(self) -> Iterable[str]:
+        pass
+
+    def _edit_columns(self) -> Iterable[str]:
+        return self._log_columns()
+
+    @abc.abstractmethod
+    def _display_columns(self) -> Optional[Iterable[str]]:
+        return None
+        #return self._primary_keys()
+
+    @abc.abstractmethod
+    def _default_sort_columns(self) -> Optional[Iterable[str]]:
+        return None
+
+    @abc.abstractmethod
+    def _nullable_columns(self) -> Iterable[str]:
+        pass
+
+    @abc.abstractmethod
+    def _auto_increment_columns(self) -> Iterable[str]:
+        pass
+
+    @abc.abstractmethod
+    def _translate_columns(self) -> Iterable[str]:
+        pass
+
+    @abc.abstractmethod
+    def _column_default(self, name) -> Any:
+        pass
+
+    @abc.abstractmethod
+    def _unique_constraints(self) -> list[list[str]]:
+        pass
+
+    def _model_name(self) -> Optional[str]:
+        slog(WARNING, f'Returning None model name for table {self.name}')
+        return None
+
+    def _model_module_search_paths(self) ->  list[tuple[str, type]]:
+        return self.schema.model_module_search_paths # Fall back to Schema-global default
+
+    @abc.abstractmethod
+    def _query_name(self) -> str:
+        return 'tbl/' + self.__name
+
+    def _relationships(self) -> list[Self]:
+        return []
+
+    @abc.abstractmethod
+    def _row_query_name(self) -> str:
+        return 'row/' + self.__name
+
+    # -- common URL schema for all data
+    def _base_location_rule(self) -> Optional[str]:
+        return f'/{self.name}'
+
+    def _location_rule(self) -> Optional[str]:
+        ret = ''
+        for col in self.__schema.access_defining_columns:
+            if col in self.primary_keys:
+                ret += f'/<{col}>'
+        ret += self.base_location_rule
+        return ret
+
+    def _row_location_rule(self) -> Optional[str]:
+        ret = self._location_rule()
+        if ret is not None:
+            for col in self.primary_keys:
+                if col not in self.__schema.access_defining_columns:
+                    ret += f'/<{col}>'
+        return ret
+
+    def _add_row_location_rule(self) -> Optional[str]:
+        rule = self._location_rule()
+        if rule is None:
+            return None
+        return rule + '/new'
+
+    def _add_child_row_location_rule(self, parent_table_name: str) -> Optional[str]:
+        parent_table = self.schema[parent_table_name]
+        ret = self._add_row_location_rule()
+        if ret is None:
+            return None
+        for cfk in self.foreign_keys_to_parent_table(parent_table):
+            for fk in cfk:
+                token = f'/<{fk.child_column.name}>'
+                if ret.find(token) != -1:
+                    continue
+                ret += f'/<{fk.child_column.name}>'
+        return ret
+
+    # -- To be used
+
+    def column_default(self, name) -> Any:
+        if self.__column_default is None:
+            ret: dict[str, Any] = dict()
+            for name in self.column_names:
+                ret[name] = self._column_default(name)
+            self.__column_default = ret
+        return self.__column_default[name]
+
+    @property
+    def columns(self):
+        return self.__columns.values()
+
+    def column(self, name):
+        return self.__columns[name]
+
+    @property
+    def column_names(self) -> Iterable[str]:
+        return self.__columns.keys()
+
+    @property
+    def name(self) -> str:
+        return self.__name
+
+    @property
+    def schema(self):
+        return self.__schema
+
+    @property
+    def model_name(self) -> Optional[str]:
+        return self._model_name()
+
+    @property
+    def model_class(self) -> Any:
+        if self.__model_class is None:
+            model_name = self.model_name
+            if model_name is None:
+                return None
+            pattern = r'^' + model_name + '$'
+            for module_path, base_class in self._model_module_search_paths():
+                ret = load_class(module_path, base_class, class_name_filter=pattern)
+                if ret is not None:
+                    self.__model_class = ret
+                    break
+            else:
+                throw(ERR, f'No model class found for model {self.model_name}')
+        return self.__model_class
+
+    def query_name(self) -> str:
+        return self._query_name()
+
+    def row_query_name(self) -> str:
+        return self._row_query_name()
+
+    @property
+    def base_location_rule(self):
+        if self.__base_location_rule is None:
+            self.__base_location_rule = self._base_location_rule()
+        return self.__base_location_rule
+
+    @property
+    def location_rule(self):
+        if self.__location_rule is None:
+            self.__location_rule = self._location_rule()
+        return self.__location_rule
+
+    def location(self, *args, **kwargs):
+        ret = self.location_rule
+        for token, val in kwargs.items(): # FIXME: Poor man's row location assembly
+            ret = re.sub(f'<{token}>', quote_plus(quote_plus(str(val))), ret)
+        return ret
+
+    @property
+    def row_location_rule(self):
+        if self.__row_location_rule is None:
+            self.__row_location_rule = self._row_location_rule()
+        return self.__row_location_rule
+
+    def row_location(self, *args, **kwargs):
+        ret = self.row_location_rule
+        for col in self.primary_keys:
+            if col in kwargs: # FIXME: Poor man's row location assembly
+                ret = re.sub(f'<{col}>', quote_plus(quote_plus(str(kwargs[col]))), ret)
+        return ret
+
+    @property
+    def add_row_location_rule(self):
+        if self.__add_row_location_rule is None:
+            self.__add_row_location_rule = self._add_row_location_rule()
+        return self.__add_row_location_rule
+
+    def add_row_location(self, *args, **kwargs) -> Optional[str]:
+        ret = self.add_row_location_rule
+        for col in self.primary_keys:
+            if col in kwargs: # FIXME: Poor man's row location assembly
+                ret = re.sub(f'<{col}>', quote_plus(quote_plus(str(kwargs[col]))), ret)
+        return ret
+
+    @property
+    def add_child_row_location_rules(self) -> Iterable[str]:
+        return self.__add_child_row_location_rules.values()
+
+    def add_child_row_location_rule(self, child_table: Union[Self, str]) -> Optional[str]:
+        if isinstance(child_table, Table):
+            child_table = child_table.name
+        return self.__add_child_row_location_rules.get(child_table)
+
+    def add_child_row_location(self, parent_table: Union[Self, str], **kwargs) -> Optional[str]:
+        ret = self.add_child_row_location_rule(parent_table)
+        if isinstance(parent_table, str):
+            parent_table = self.schema[parent_table]
+        if ret is None:
+            return None
+        for cfk in self.foreign_keys_to_parent_table(parent_table):
+            for fk in cfk:
+                if fk.parent_column.name in kwargs:
+                    ret = re.sub(f'<{fk.child_column.name}>', quote_plus(quote_plus(str(kwargs[fk.parent_column.name]))), ret)
+        return ret
+
+    @property
+    def primary_keys(self) -> Iterable[str]:
+        if self.__primary_keys is None:
+            self.__primary_keys = self._primary_keys()
+        return self.__primary_keys
+
+    @property
+    def log_columns(self):
+        if self.__log_columns is None:
+            self.__log_columns = self._log_columns()
+        return self.__log_columns
+
+    @property
+    def edit_columns(self):
+        if self.__edit_columns is None:
+            self.__edit_columns = self._edit_columns()
+        return self.__edit_columns
+
+    @property
+    def display_columns(self):
+        if self.__display_columns is None:
+            self.__display_columns = self._display_columns()
+        return self.__display_columns
+
+    @property
+    def default_sort_columns(self):
+        if self.__default_sort_columns is None:
+            self.__default_sort_columns = self._default_sort_columns()
+        return self.__default_sort_columns
+
+    @property
+    def auto_increment_columns(self) -> Iterable[str]:
+        return self._auto_increment_columns()
+
+    @property
+    def translate_columns(self) -> Iterable[str]:
+        if self.__translate_columns is None:
+            self.__translate_columns = self._translate_columns()
+        return self.__translate_columns
+
+    @property
+    def nullable_columns(self) -> Iterable[str]:
+        if self.__nullable_columns is None:
+            self.__nullable_columns = self._nullable_columns()
+        return self.__nullable_columns
+
+    @property
+    def non_nullable_columns(self) -> Iterable[str]:
+        if self.__non_nullable_columns is None:
+            ret = []
+            all_cols = self.column_names
+            nullable_columns = self.nullable_columns
+            for col in all_cols:
+                if col not in nullable_columns:
+                    ret.append(col)
+            self.__non_nullable_columns = ret
+        return self.__non_nullable_columns
+
+    @property
+    def null_insertible_columns(self) -> Iterable[str]:
+        if self.__null_insertible_columns is None:
+            ret: list[str] = []
+            for col in self.__columns.values():
+                if col.is_null_insertible:
+                    ret.append(col.name)
+            self.__null_insertible_columns = ret
+        return self.__null_insertible_columns
+
+    @property
+    def not_null_insertible_columns(self) -> Iterable[str]:
+        if self.__not_null_insertible_columns is None:
+            ret: list[str] = []
+            for col in self.__columns.values():
+                if not col.is_null_insertible:
+                    ret.append(col.name)
+            self.__not_null_insertible_columns = ret
+        return self.__not_null_insertible_columns
+
+    @property
+    def unique_constraints(self) -> list[ColumnSet]:
+        if self.__unique_constraints is None:
+            ret: list[ColumnSet] = []
+            impl = self._unique_constraints()
+            if impl is not None:
+                for columns in impl:
+                    ret.append(ColumnSet(columns=columns))
+            self.__unique_constraints = ret
+        return self.__unique_constraints
+
+    @property
+    def foreign_key_constraints(self) -> list[CompositeForeignKey]:
+        if self.__foreign_key_constraints is None:
+            ret: list[Any] = []
+            for composite_key in self.__schema.foreign_key_constraints:
+                if composite_key.child_table == self:
+                    ret.append(composite_key)
+            self.__foreign_key_constraints = ret
+        return self.__foreign_key_constraints
+
+    @property
+    def foreign_key_parent_tables(self):
+        return self.__foreign_key_parent_tables.values()
+
+    def foreign_keys_to_parent_table(self, parent_table) -> Iterable[CompositeForeignKey]:
+        if self.__foreign_keys_to_parent_table is None:
+            self.__foreign_keys_to_parent_table = OrderedDict()
+            for cfk in self.foreign_key_constraints:
+                pt = cfk.parent_table.name
+                if pt not in self.__foreign_keys_to_parent_table:
+                    self.__foreign_keys_to_parent_table[pt] = []
+                self.__foreign_keys_to_parent_table[pt].append(cfk)
+        parent_table_name = parent_table if isinstance(parent_table, str) else parent_table.name
+        return self.__foreign_keys_to_parent_table[parent_table_name] if parent_table_name in self.__foreign_keys_to_parent_table else []
+
+    @property
+    def relationships(self) -> list[tuple[str, Self]]:
+        if self.__relationships is None:
+            ret = []
+            for member_name, table_name in self._relationships():
+                ret.append((member_name, self.schema[table_name]))
+            self.__relationships = ret
+        return self.__relationships
+
+    def relationship(self, table: Union[Self, str]) -> Optional[Self]:
+        if isinstance(table, Table):
+            table = table.name
+        return self.__relationship_by_foreign_table.get(table)

src/python/jw/util/db/schema/utils.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+
+from .Schema import Schema
+
+from ...log import *
+
+def check_schema(schema: Schema): # export
+    slog(NOTICE, f'There are {len(schema)} tables in the database')
+    for cfk in schema.foreign_key_constraints:
+        for fk in cfk:
+            if fk.child_column.data_type != fk.parent_column.data_type:
+                raise Exception(f'Type mismatch in foreign key {fk}: {fk.child_column.data_type} != {fk.parent_column.data_type}')

src/python/jw/util/graph/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-mod.mk

src/python/jw/util/graph/yed/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-mod.mk

src/python/jw/util/graph/yed/MapAttr2Shape.py

@@ -0,0 +1,191 @@
+# -*- coding: utf-8 -*-
+
+from collections.abc import Callable
+
+import xml.etree.ElementTree as ET
+
+from ...log import *
+
+class MapAttr2Shape: # export
+
+    def __init__(self, mappings: dict[str, str|Callable[[dict[str, str]], str]]|None=None) -> None:
+        self.__mappings = mappings if mappings is not None else {}
+        self.__shape_node_key = 'd25'
+        self.__ns_gml = "http://graphml.graphdrawing.org/xmlns"
+        self.__ns = {
+            # -- Standard GraphML
+            "":                   self.__ns_gml,
+            "xsi":                "http://www.w3.org/2001/XMLSchema-instance",
+            "xsi:schemaLocation": "http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd",
+
+            # -- YWorks GraphML
+            "java":               "http://www.yworks.com/xml/yfiles-common/1.0/java",
+            "sys":                "http://www.yworks.com/xml/yfiles-common/markup/primitives/2.0",
+            "x":                  "http://www.yworks.com/xml/yfiles-common/markup/2.0",
+            "y":                  "http://www.yworks.com/xml/graphml",
+            "yed":                "http://www.yworks.com/xml/yed/3",
+        }
+        # https://stackoverflow.com/questions/4997848/
+        for name, url in self.__ns.items():
+            ET.register_namespace(name, url)
+
+    def __keys(self, root) -> dict[str, str]:
+        ret: dict[str, str] = {}
+        for el in root.findall('key', self.__ns):
+            attr_name = el.get('attr.name')
+            id = el.get('id')
+            if attr_name is not None:
+                ret[attr_name] = id
+        return ret
+
+    def __value(self, node, key):
+        data = node.find(f'data[@key="{key}"]', self.__ns)
+        if data is None:
+            return None
+        return data.text
+
+    def __attribs(self, node, keys) -> dict[str, str]:
+        ret: dict[str, str] = {}
+        for name, key in keys.items():
+            val = self.__value(node, key)
+            if val is None:
+                continue
+            ret[name] = val
+        return ret
+
+    def __add_key_nodegraphics(self, root):
+        # <graphml><key for="node" id="d22" yfiles.type="nodegraphics"/></graphml>
+        el = ET.Element('key')
+        for attr, val in {
+            'id':          self.__shape_node_key,
+            'for':        'node',
+            'yfiles.type':'nodegraphics'
+        }.items():
+            el.set(attr, val)
+        root.append(el)
+
+    def __massage_node(self, node, keys: dict[str, str]) -> None:
+
+        def __add(parent, d: dict[str, Any]):
+            for tag, content in d.items():
+                if tag.find('y:') != -1:
+                    ns, tag = tag.split(':')
+                    tag = '{' + self.__ns[ns] + '}' + tag
+                attrib = content.get('a') or {}
+                el = ET.Element(tag, attrib=attrib)
+                text = content.get('t')
+                if text is not None:
+                    el.text = text
+                parent.append(el)
+                children = content.get('c')
+                if children is not None:
+                    __add(el, children)
+
+        default_values = {
+            'color': '#FFCC00',
+            'text': ''
+        }
+        values = {}
+
+        for key, default in default_values.items():
+            mapping = self.__mappings.get(key)
+            if mapping is None:
+                values[key] = default
+                continue
+            try:
+                if isinstance(mapping, str):
+                    values[key] = mapping
+                    continue
+                mapped = mapping(self.__attribs(node, keys))
+                values[key] = mapped or default
+            except:
+                pass
+
+        color = values['color']
+        text  = values['text']
+        has_text = 'true' if text else 'false'
+
+        width_text = round(len(text) * 5.75, 5) if text else 0
+        width_box = width_text + 10 if text else 30
+
+        shape = {
+            'data': {
+                'a': {'key': self.__shape_node_key},
+                'c': {
+                    'y:ShapeNode': {
+                        'a': {},
+                        'c': {
+                            'y:Geometry': {'a': {'height': '30.0', 'width': str(width_box), 'x': str(-(width_box / 2)), 'y':' -15.0'}},
+                            'y:Fill': {'a': {'color': color, 'transparent': 'false'}},
+                            'y:BorderStyle': {'a': {'color': '#000000', 'raised': 'false', 'type': 'line', 'width': '1.0'}},
+                            'y:NodeLabel': {
+                                'a': {
+                                 'alignment': 'center',
+                                 'autoSizePolicy': 'content',
+                                 'fontFamily': 'Dialog',
+                                 'fontSize': '12',
+                                 'fontStyle': 'plain',
+                                 'hasBackgroundColor': 'false',
+                                 'hasLineColor': 'false',
+                                 'hasText': has_text,
+                                 'height': '18',
+                                 'horizontalTextPosition': 'center',
+                                 'iconTextGap': '4',
+                                 'modelName': 'custom',
+                                 'textColor': '#000000',
+                                 'verticalTextPosition': 'bottom',
+                                 'visible': 'true',
+                                 'width': str(width_text),
+                                 'x': '13.0',
+                                 'y': '13.0',
+                                },
+                                'c': {
+                                   'y:LabelModel': {
+                                      'c': {
+                                          'y:SmartNodeLabelModel': {'a': {'distance': '4.0'}}
+                                      },
+                                   },
+                                  'y:ModelParameter': {
+                                      'c': {
+                                          'y:SmartNodeLabelModelParameter': {
+                                              'a': {
+                                                'labelRatioX':'0.0',
+                                                'labelRatioY': '0.0',
+                                                'nodeRatioX': '0.0',
+                                                'nodeRatioY': '0.0',
+                                                'offsetX': '0.0',
+                                                'offsetY': '0.0',
+                                                'upX': '0.0',
+                                                'upY': '-1.0',
+                                              }
+                                          }
+                                      }
+                                   }
+                                },
+                                't': text
+                             },
+                            'y:Shape': {'a': {'type': 'rectangle'}}
+                        }
+                    }
+                }
+            }
+        }
+
+        __add(node, shape)
+
+    def __massage_nodes(self, root) -> None:
+        keys = self.__keys(root)
+        graph = root.find(f'graph', self.__ns)
+        for node in graph:
+            self.__massage_node(node, keys)
+
+    def run(self, path_in: str, path_out: str) -> None:
+        parser = ET.XMLParser(encoding="utf-8")
+        tree = ET.parse(path_in, parser=parser)
+        root = tree.getroot()
+
+        self.__add_key_nodegraphics(root)
+        self.__massage_nodes(root)
+
+        ET.indent(tree, space='  ', level=0)
+        tree.write(path_out, xml_declaration=True, encoding='utf-8')

src/python/jw/util/ldap.py

@@ -0,0 +1,322 @@
+# -*- coding: utf-8 -*-
+
+import ldap, getpass, pathlib, copy
+from ldap.schema.models import ObjectClass
+from enum import Flag, auto
+import networkx as nx
+from typing import Any, Self
+from collections.abc import Callable
+
+from .Config import Config as BaseConfig
+from .log import *
+
+class Config:
+    def __init__(self, external: BaseConfig|None=None):
+        self.__external = external
+        for attr in ['ldap_uri', 'bind_dn', 'bind_pw', 'base_dn']:
+            setattr(self, '_Config__' + attr, None)
+
+    def __get(self, key: str, default: str):
+        if not self.__external:
+            return default
+        return self.__external.value(key, default=default)
+
+    @property
+    def ldap_uri(self):
+        if self.__ldap_uri is None:
+            for key in ['ldap_uri', 'uri']:
+                self.__ldap_uri = self.__get(key, default=None)
+                if self.__ldap_uri is not None:
+                    break
+            else:
+                self.__ldap_uri = 'ldap://ldap.janware.com'
+        return self.__ldap_uri
+    @ldap_uri.setter
+    def ldap_uri(self, rhs):
+        self.__ldap_uri = rhs
+
+    @property
+    def bind_dn(self):
+        if self.__bind_dn is None:
+            self.__bind_dn = self.__get('bind_dn', default=f'uid={getpass.getuser()},ou=users,dc=jannet,dc=de')
+        return self.__bind_dn
+    @bind_dn.setter
+    def bind_dn(self, rhs):
+        self.__bind_dn = rhs
+
+    @property
+    def bind_pw(self):
+        if self.__bind_pw is None:
+            for key in ['bind_pw', 'password']:
+                ret = self.__get(key, default=None)
+                if ret is not None:
+                    break
+            if ret is None:
+                ldap_secret_file = self.__get('secret_file', f'{pathlib.Path.home()}/.ldap.secret')
+                with open(ldap_secret_file, 'r') as file:
+                    ret = file.read()
+                file.closed
+            ret = ret.strip()
+            self.__bind_pw = ret
+        return self.__bind_pw
+    @bind_pw.setter
+    def bind_pw(self, rhs):
+        self.__bind_pw = rhs
+
+    @property
+    def base_dn(self):
+        if self.__base_dn is None:
+            self.__base_dn = self.__get('base_dn', default=f'dc=jannet,dc=de')
+        return self.__base_dn
+    @base_dn.setter
+    def base_dn(self, rhs):
+        self.__base_dn = rhs
+
+class Connection: # export
+
+    class AttrType(Flag):
+        Must = auto()
+        May  = auto()
+
+    def __init__(self, conf: Config|BaseConfig|None=None, backtrace=False):
+        uri: str|None = None
+        c = conf if isinstance(conf, Config) else Config(conf)
+        try:
+            uri = c.ldap_uri
+        except:
+            uri = c.uri
+        try:
+            ret = ldap.initialize(uri)
+            ret.start_tls_s()
+        except Exception as e:
+            slog(ERR, f'Failed to initialize LDAP connection to "{uri}" ({str(e)})')
+            raise
+        try:
+            rr = ret.bind_s(c.bind_dn, c.bind_pw) # method)
+        except Exception as e:
+            slog(ERR, f'Failed to bind to "{uri}" with dn "{c.bind_dn}" ({str(e)})')
+            raise
+        self.__ldap = ret
+        self.__backtrace = backtrace
+        self.__object_classes_by_oid: dict[str, ObjectClass]|None = None
+        self.__object_class_tree: nx.Graph|None = None
+        self.__object_classes_by_name: dict[str, ObjectClass]|None = None
+
+    @property
+    def ldap(self):
+        return self.__ldap
+
+    def add(self, attrs: dict[str, bytes], dn: str|None=None):
+        if dn is None:
+            if not 'dn' in attrs:
+                raise Exception('No DN to add an LDAP entry to')
+            attrs = copy.deepcopy(attrs)
+            del attrs['dn']
+        try:
+            slog(INFO, f'LDAP: Add [{dn}] -> {attrs}')
+            self.__ldap.add_s(dn, ldap.modlist.addModlist(attrs))
+        except Exception as e:
+            slog(ERR, f'{dn}: Failed to add entry {attrs} ({e})')
+            raise
+
+    def delete(self, dn: str, recursive=False, force_existence: bool=False):
+
+        def __walk_cb_delete(conn: Connection, entry, context):
+            self.walk(__walk_cb_delete, base=entry[0], scope=ldap.SCOPE_ONELEVEL, context=context)
+            self.__ldap.delete_s(entry[0])
+
+        try:
+            if recursive:
+                self.walk(__walk_cb_delete, dn, scope=ldap.SCOPE_ONELEVEL)
+                self.__ldap.delete_s(dn)
+            else:
+                self.__ldap.delete_s(dn)
+        except ldap.NO_SUCH_OBJECT as e:
+            if force_existence:
+                raise
+        except Exception as e:
+            slog(ERR, f'Failed to delete {dn} ({e})')
+            raise
+
+    def walk(
+            self,
+            callback: Callable[[Self, Any, Any], None],
+            base: str,
+            scope,
+            context=None,
+            filterstr=None,
+            attrlist=None,
+            attrsonly=0,
+            serverctrls=None,
+            clientctrls=None,
+            timeout=-1,
+            sizelimit=0,
+            decode: bool=False,
+            unroll: bool=False
+        ):
+
+        # TODO: Support ignored arguments
+        search_return = self.__ldap.search(base=base,
+                                           scope=scope,
+                                           filterstr=filterstr,
+                                           attrlist=attrlist,
+                                           attrsonly=attrsonly)
+        while True:
+            result_type, result_data = self.__ldap.result(search_return, 0)
+            if (result_data == []):
+                break
+            if result_type != ldap.RES_SEARCH_ENTRY:
+                continue
+            for entry in result_data:
+                if decode:
+                    entry = entry[0], {key: [val.decode() for val in vals] for key, vals in entry[1].items()}
+                if unroll and False:
+                    entry = entry[0], {key: val[0] for key, val in entry[1].items()}
+                try:
+                    callback(self, entry, context)
+                except Exception as e:
+                    msg = f'Exception {e}'
+                    if self.__backtrace:
+                        slog(ERR, msg)
+                        raise
+                    slog(WARNING, msg)
+                    continue
+
+    def find(self,
+            base: str,
+            scope,
+            filterstr=None,
+            attrlist=None,
+            attrsonly=0,
+            serverctrls=None,
+            clientctrls=None,
+            timeout=-1,
+            sizelimit=0,
+            assert_unique=False,
+            assert_not_empty=False,
+           ):
+
+        def __walk_cb_find(conn: Connection, entry: Any, context: Any):
+            result.append(entry)
+
+        def __search():
+            return f'{base} -> "{filterstr}"'
+
+        try:
+            result: list[Any] = []
+            self.walk(__walk_cb_find, base, scope=scope, filterstr=filterstr, attrlist=attrlist)
+        except Exception as e:
+            slog(ERR, f'Failed search {__search()} ({e})')
+            raise
+        if assert_not_empty and not result:
+            raise Exception(f'Empty result for search {__search()}')
+        if assert_unique and len(result) > 1:
+            raise Exception(f'Found {len(result)} results for search {__search()}')
+        return result
+
+    @property
+    def object_classes(self) -> dict[str, ObjectClass]:
+    #def object_classes(self):
+        if self.__object_classes_by_oid is None:
+            res = self.find(base='', scope=ldap.SCOPE_BASE, filterstr='(objectClass=*)', attrlist=['subschemaSubentry'])
+            dn = res[0][1]['subschemaSubentry'][0].decode('utf-8') # Usually yields cn=Subschema
+            res = self.find(base=dn, scope=ldap.SCOPE_BASE, filterstr='(objectClass=*)', attrlist=['*', '+'])
+            subschema_entry = res[0]
+            subschema_subentry = ldap.cidict.cidict(subschema_entry[1])
+            subschema = ldap.schema.SubSchema(subschema_subentry)
+            object_class_oids = subschema.listall(ObjectClass)
+            self.__object_classes_by_oid = {
+                oid: subschema.get_obj(ObjectClass, oid) for oid in object_class_oids
+            }
+        return self.__object_classes_by_oid
+
+    @property
+    def object_class_by_name(self) -> dict[str, ObjectClass]:
+        if self.__object_classes_by_name is None:
+            ret: dict[str, ObjectClass] = {}
+            self.__object_classes_by_name = ret
+            for oid, oc in self.object_classes.items():
+                ret[oid] = oc
+                for name in oc.names:
+                    ret[name] = oc
+                    ret[name.lower()] = oc
+        return self.__object_classes_by_name
+
+    def __oc_recurse_to_top(self, cur: str|ObjectClass, cb, context):
+        cur_oc = cur if isinstance(cur, ObjectClass) else self.object_class_by_name[cur.lower()]
+        for s in cur_oc.sup:
+            self.__oc_recurse_to_top(s, cb, context)
+        cb(cur_oc, context)
+
+    def object_class_path(self, leaf: str|ObjectClass):
+        def cb(oc, context):
+            ret.append(oc)
+        ret: list[str] = []
+        self.__oc_recurse_to_top(leaf, cb, None)
+        return reversed(ret)
+
+    @property
+    def object_class_tree(self) -> nx.Graph:
+
+        if self.__object_class_tree is None:
+
+            def collect(root, attr):
+                ret = set()
+                def cb(oc, attr):
+                    vals = getattr(oc, attr)
+                    if vals is None:
+                        return
+                    for val in vals:
+                        ret.add(val)
+                self.__oc_recurse_to_top(root, cb, attr)
+                return ret
+
+            kind = {
+                0: 'STRUCTURAL',
+                1: 'ABSTRACT',
+                2: 'AUXILIARY'
+            }
+            ret = nx.DiGraph()
+            for oid, oc in self.object_classes.items():
+                ret.add_node(
+                        oid,
+                        oid=oid,
+                        name=oc.names[0],
+                        kind=kind[oc.kind],
+                        must=', '.join(collect(oc, 'must')),
+                        may=', '.join(collect(oc, 'may'))
+                    )
+                for base_class in oc.sup:
+                    try:
+                        ret.add_edge(oid, self.object_class_by_name[base_class.lower()].oid)
+                    except Exception as e:
+                        slog(WARNING, f'Failed to add edge {oid}:{oc.names} -> {base_class} ({e})')
+            self.__object_class_tree = ret
+        return self.__object_class_tree
+
+    def object_class_attrs(self, oc: str|ObjectClass, required: AttrType = AttrType.Must, origins: bool=False) -> dict[str, set[str]]|set[str]:
+        all_attrs: set[str] = set()
+        attrs_by_origin: dict[str, set[str]] = {}
+        for oc in self.object_class_path(oc):
+            cur = set()
+            if required & self.AttrType.Must:
+                cur |= set(oc.must)
+            if required & self.AttrType.May:
+                cur |= set(oc.may)
+            if cur:
+                all_attrs |= cur
+                attrs_by_origin[oc] = cur
+        return attrs_by_origin if origins else all_attrs
+
+    def is_derived_class(self, name, base_candidate):
+        #oid = self.object_class_by_name[name].oid
+        #base_oid = self.object_class_by_name[base_candidate].oid
+        #if base_oid in [oc.oid for oc in self.object_class_path(name)]:
+        #    return True
+        return nx.has_path(self.object_class_tree, self.object_class_by_name[name.lower()].oid, self.object_class_by_name[base_candidate.lower()].oid)
+
+def default_config() -> Config: # export
+    return Config()
+
+def bind(conf: Config|BaseConfig|None=None) -> Connection:
+    return Connection(conf)

src/python/jw/util/log.py

@@ -0,0 +1,332 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+
+from typing import List, Tuple, Optional, Any
+
+import sys, re, io, syslog, inspect, unicodedata
+
+from os.path import basename
+from datetime import datetime
+
+from . import misc
+
+# --- python 2 / 3 compatibility stuff
+try:
+    basestring # type: ignore
+except NameError:
+    basestring = str
+
+_special_chars = {
+    '\a' : '\\a',
+    '\b' : '\\b',
+    '\t' : '\\t',
+    '\n' : '\\n',
+    '\v' : '\\v',
+    '\f' : '\\f',
+    '\r' : '\\r',
+}
+
+_special_char_regex = re.compile("(%s)" % "|".join(map(re.escape, _special_chars.keys())))
+
+_all_control_chars   = ''.join(chr(c) for c in range(sys.maxunicode) if unicodedata.category(chr(c)) in {'Cc'})
+_clean_str_regex     = re.compile(r'(\033\[[0-9]*m|[%s])' % re.escape(_all_control_chars))
+
+EMERG                = int(syslog.LOG_EMERG)
+ALERT                = int(syslog.LOG_ALERT)
+CRIT                 = int(syslog.LOG_CRIT)
+ERR                  = int(syslog.LOG_ERR)
+WARNING              = int(syslog.LOG_WARNING)
+NOTICE               = int(syslog.LOG_NOTICE)
+INFO                 = int(syslog.LOG_INFO)
+DEBUG                = int(syslog.LOG_DEBUG)
+DEVEL                = int(syslog.LOG_DEBUG + 1)
+OFF                  = DEVEL + 1
+
+_level               = NOTICE
+
+CONSOLE_FONT_BOLD    = '\033[1m'
+CONSOLE_FONT_RED     = '\033[31m'
+CONSOLE_FONT_GREEN   = '\033[32m'
+CONSOLE_FONT_YELLOW  = '\033[33m'
+CONSOLE_FONT_BLUE    = '\033[34m'
+
+CONSOLE_FONT_MAGENTA = '\033[35m'
+CONSOLE_FONT_CYAN    = '\033[36m'
+CONSOLE_FONT_WHITE   = '\033[37m'
+
+CONSOLE_FONT_BLINK   = '\033[5m'
+CONSOLE_FONT_OFF     = '\033[m'
+
+f_position           = 'position'
+f_module             = 'module'
+f_date               = 'date'
+f_stderr             = 'stderr'
+f_stdout             = 'stdout'
+f_prio               = 'prio'
+f_color              = 'color'
+f_default            = [ f_position, f_stderr, f_prio, f_color ]
+
+_flags               = set(f_default)
+_log_prefix          = ''
+_clean_log_prefix    = ''
+_file_name_len       = 20
+_module_name_len     = 50
+_log_file_streams: list[io.TextIOWrapper] = []
+
+_short_prio_str      = {
+    EMERG            : '<Y>',
+    ALERT            : '<A>',
+    CRIT             : '<C>',
+    ERR              : '<E>',
+    WARNING          : '<W>',
+    NOTICE           : '<N>',
+    INFO             : '<I>',
+    DEBUG            : '<D>',
+    DEVEL            : '<V>',
+}
+
+_prio_colors         = {
+    DEVEL            : [ "", "" ],
+    DEBUG            : [ "", "" ],
+    INFO             : [ CONSOLE_FONT_BLUE, CONSOLE_FONT_OFF ],
+    NOTICE           : [ CONSOLE_FONT_GREEN, CONSOLE_FONT_OFF ],
+    WARNING          : [ CONSOLE_FONT_YELLOW, CONSOLE_FONT_OFF ],
+    ERR              : [ CONSOLE_FONT_BOLD + CONSOLE_FONT_RED, CONSOLE_FONT_OFF ],
+    CRIT             : [ CONSOLE_FONT_BOLD + CONSOLE_FONT_MAGENTA, CONSOLE_FONT_OFF ],
+    ALERT            : [ CONSOLE_FONT_BOLD + CONSOLE_FONT_MAGENTA, CONSOLE_FONT_OFF ],
+    EMERG            : [ CONSOLE_FONT_BOLD + CONSOLE_FONT_MAGENTA, CONSOLE_FONT_OFF ],
+}
+
+class Stream:
+    def __init__(self, stream, flags):
+        self.stream = stream
+        self.flags = flags
+
+_streams: dict[int, Stream] = dict()
+_stream_descriptors = [reversed(range(1, 16))]
+
+def add_capture_stream(stream, flags=0x0):
+    ret = _stream_descriptors.pop()
+    _streams[ret] = Stream(stream=stream, flags=flags)
+    return ret
+
+def rm_capture_stream(sd):
+    del _streams[sd]
+    _stream_descriptors.append(sd)
+
+def prio_gets_logged(prio: int) -> bool: # export
+    if prio > _level:
+        return False
+    return True
+
+def log_level(s: Optional[str]=None) -> int: # export
+    if s is None:
+        return _level
+    return parse_log_prio_str(s)
+
+def get_caller_pos(up: int = 1, kwargs: Optional[dict[str, Any]] = None) -> Tuple[str, str, int]:
+    if kwargs and 'caller' in kwargs:
+        r = kwargs['caller']
+        del kwargs['caller']
+        return r
+    caller = inspect.stack()[up+1]
+    mod = inspect.getmodule(caller[0])
+    mod_name = '' if mod is None else mod.__name__
+    return (mod_name, basename(caller.filename), caller.lineno)
+
+def slog_m(prio: int, *args, **kwargs) -> None: # export
+    if prio > _level:
+        return
+    if len(args):
+        margs = ''
+        for a in args:
+            if isinstance(a, list):
+                margs += '\n'.join([str(elem) for elem in a])
+                continue
+            margs += ' ' + str(a)
+    if 'caller' not in kwargs:
+        caller = get_caller_pos(1)
+    else:
+        caller = kwargs['caller']
+        del kwargs['caller']
+    for line in margs[1:].split('\n'):
+        slog(prio, line, **kwargs, caller=caller)
+
+def slog(prio: int, *args, only_printable: bool=False, **kwargs) -> None: # export
+
+    if prio > _level:
+        return
+
+    msg = ''
+    color_on = ''
+    color_off = ''
+
+    if f_date in _flags:
+        msg += datetime.now().strftime("%b %d %H:%M:%S.%f ")
+
+    if f_prio in _flags:
+        msg += _short_prio_str[prio] + ' '
+
+    if f_position in _flags:
+
+        if 'caller' in kwargs:
+            mod, name, line = kwargs['caller']
+        else:
+            mod, name, line = get_caller_pos(1)
+
+        if f_module in _flags:
+            msg += misc.pad(mod, _module_name_len)
+
+        msg += misc.pad(name, _file_name_len) + '[' + misc.pad(str(line), 4, True) + ']'
+
+    if f_color in _flags:
+        color_on, color_off = console_color_chars(prio)
+
+    margs = ''
+    if len(args):
+        for a in args:
+            margs += ' ' + str(a)
+        if only_printable:
+            margs = _special_char_regex.sub(lambda mo: _special_chars[mo.string[mo.start():mo.end()]], margs)
+            margs = re.sub('[\x01-\x1f]', '.', margs)
+
+    for file in _log_file_streams:
+        print(msg + _clean_log_prefix + margs, file=file)
+
+    msg += _log_prefix
+
+    if not len(msg):
+        return
+
+    if len(margs):
+        msg += color_on + margs + color_off
+
+    files = []
+    if 'capture' in kwargs:
+        files.append(kwargs['capture'])
+    elif _streams:
+        files = [s.stream for s in _streams.values()]
+    else:
+        if f_stdout in _flags:
+            files.append(sys.stdout)
+
+        if f_stderr in _flags:
+            files.append(sys.stderr)
+
+        if not len(files):
+            files = [ sys.stdout ]
+
+    for file in files:
+        print(msg, file=file)
+
+def throw(*args, prio=ERR, caller=None, **kwargs) -> None:
+    if caller is None:
+        caller = get_caller_pos(1)
+    msg = ' '.join([str(arg) for arg in args])
+    slog(prio, msg, caller=caller)
+    raise Exception(msg)
+
+def parse_log_prio_str(prio: str) -> int: # export
+    try:
+        r = int(prio)
+        if r < 0 or r > DEVEL:
+            raise Exception("Invalid log priority ", prio)
+    except ValueError:
+        map_prio_str_to_val = {
+            "EMERG"     : EMERG,
+            "emerg"     : EMERG,
+            "ALERT"     : ALERT,
+            "alert"     : ALERT,
+            "CRIT"      : CRIT,
+            "crit"      : CRIT,
+            "ERR"       : ERR,
+            "err"       : ERR,
+            "WARNING"   : WARNING,
+            "warning"   : WARNING,
+            "NOTICE"    : NOTICE,
+            "notice"    : NOTICE,
+            "INFO"      : INFO,
+            "info"      : INFO,
+            "DEBUG"     : DEBUG,
+            "debug"     : DEBUG,
+            "DEVEL"     : DEVEL,
+            "devel"     : DEVEL,
+            "OFF"       : OFF,
+            "off"       : OFF,
+        }
+        if prio in map_prio_str_to_val:
+            return map_prio_str_to_val[prio]
+    raise Exception("Unknown priority string \"", prio, "\"")
+
+def console_color_chars(prio: int) -> List[str]: # export
+    if not sys.stdout.isatty():
+        return [ '', '' ]
+    return _prio_colors[prio]
+
+def set_level(level_: str) -> None: # export
+    global _level
+    if isinstance(level_, basestring):
+        _level = parse_log_prio_str(level_)
+        return
+    _level = level_
+
+def set_flags(flags: str|None) -> str: # export
+    global _flags
+    ret = ','.join(_flags)
+    if flags is not None:
+        _flags = set(flags.split(','))
+    return ret
+
+    #syslog
+    #console
+    #color
+    #prio
+    #position
+    #ide
+    #trace_rename_thread_to_shorter
+    #trace_rename_thread_to_longer
+    #trace_inout
+    #skip_openlog
+    #id
+    #date
+    #pid
+    #highlight_first_error
+
+def append_to_prefix(prefix: str) -> str: # export
+    global _log_prefix
+    global _clean_log_prefix
+    r = _log_prefix
+    if prefix:
+        _log_prefix += prefix
+        _clean_log_prefix = _clean_str_regex.sub('', _log_prefix)
+    return r
+
+def remove_from_prefix(count) -> str: # export
+    if isinstance(count, str):
+        count = len(count)
+    global _log_prefix
+    global _clean_log_prefix
+    r = _log_prefix
+    _log_prefix = _log_prefix[:-count]
+    _clean_log_prefix = _clean_str_regex.sub('', _log_prefix)
+    return r
+
+def set_filename_length(l: int) -> int: # export
+    global _file_name_len
+    r = _file_name_len
+    if l:
+        _file_name_len = l
+    return r
+
+def set_module_name_length(l: int) -> int: # export
+    global _module_name_len
+    r = _module_name_len
+    if l:
+        _module_name_len = l
+    return r
+
+def add_log_file(path: str) -> None: # export
+    global _log_file_streams
+    fd = open(path, 'w', buffering=1)
+    _log_file_streams.append(fd)

src/python/jw/util/misc.py

@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+
+import os, errno, atexit, tempfile, filecmp, inspect, importlib, re
+
+from typing import Iterable
+
+from . import log
+
+_tmpfiles: set[str] = set()
+
+def _cleanup():
+    for f in _tmpfiles:
+        silentremove(f)
+
+def silentremove(filename): #export
+    try:
+        os.remove(filename)
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise # re-raise exception if a different error occurred
+
+def update_symlink(target, link_name):
+    try:
+        os.symlink(target, link_name)
+    except OSError as e:
+        if e.errno == errno.EEXIST:
+            os.remove(link_name)
+            os.symlink(target, link_name)
+        else:
+            raise e
+
+def pad(token: str, total_size: int, right_align: bool = False) -> str:
+    add = total_size - len(token)
+    if add <= 0:
+        return token
+    space = ' ' * add
+    if right_align:
+        return space + token
+    return token + space
+
+def atomic_store(contents, path): # export
+    if path[0:3] == '/dev':
+        with open(path, 'w') as outfile:
+            outfile.write(contents)
+        return
+    outfile = tempfile.NamedTemporaryFile(prefix=os.path.basename(path), delete=False, dir=os.path.dirname(path))
+    name = outfile.name
+    _tmpfiles.add(name)
+    outfile.write(contents)
+    outfile.close()
+    os.rename(name, path)
+    _tmpfiles.remove(name)
+
+# see https://stackoverflow.com/questions/2020014
+def object_builtin_name(o, full=True): # export
+    #if not full:
+    #    return o.__class__.__name__
+    module = o.__class__.__module__
+    if module is None or module == str.__class__.__module__:
+        return o.__class__.__name__  # Avoid reporting __builtin__
+    return module + '.' + o.__class__.__name__
+
+def get_derived_classes(mod, base, flt=None): # export
+    members = inspect.getmembers(mod, inspect.isclass)
+    r = []
+    for name, c in members:
+        log.slog(log.DEBUG, "found ", name)
+        if inspect.isabstract(c):
+            log.slog(log.DEBUG, "  is abstract")
+            continue
+        if not base in inspect.getmro(c):
+            log.slog(log.DEBUG, "  is not derived from", base, "only", inspect.getmro(c))
+            continue
+        if flt and not re.match(flt, name):
+            log.slog(log.DEBUG, '  "{}.{}" has wrong name'.format(mod, name))
+            continue
+        r.append(c)
+    return r
+
+def load_classes(path, baseclass, flt=None): # export
+    r = []
+    for p in path.split(':'):
+        mod = importlib.import_module(path)
+        log.slog(log.DEBUG, "importing ", path)
+        r.extend(get_derived_classes(mod, baseclass, flt))
+    return r
+
+def load_class(module_path, baseclass, class_name_filter=None): # export
+    mod = importlib.import_module(module_path)
+    classes = get_derived_classes(mod, baseclass, flt=class_name_filter)
+    if len(classes) == 0:
+        raise Exception(f'no class matching "{class_name_filter}" of type "{baseclass}" found in module "{module_path}"')
+    if len(classes) > 1:
+        raise Exception(f'{len(classes)} classes matching "{class_name_filter}" of type "{baseclass}" found in module "{module_path}"')
+    return classes[0]
+
+def load_class_names(path, baseclass, flt=None, remove_flt=False): # export
+    classes = load_classes(path, baseclass, flt)
+    r = []
+    for c in classes:
+        name = c.__name__
+        if flt and remove_flt:
+            name = re.subst(flt, "", name)
+        if name not in r:
+            r.append(name)
+        else:
+            pass
+            #log.slog(log.WARNING, "{} is already in in {}".format(name, r))
+    return r
+
+def load_object(module_path, baseclass, class_name_filter=None, *args, **kwargs): # export
+    return load_class(module_path, baseclass, class_name_filter=class_name_filter)(*args, **kwargs)
+
+def load_function(module_path, name): # export
+    mod = importlib.import_module(module_path)
+    return getattr(mod, name)
+
+def commit_tmpfile(tmp: str, path: str) -> None: # export
+    caller = log.get_caller_pos()
+    if os.path.isfile(path) and filecmp.cmp(tmp, path):
+        log.slog(log.INFO, "{} is up to date".format(path), caller=caller)
+        os.unlink(tmp)
+    else:
+        log.slog(log.NOTICE, "saving {}".format(path), caller=caller)
+        os.rename(path + '.tmp', path)
+
+def multi_regex_edit(spec, strings): # export
+    for cmd in spec:
+        if len(cmd) < 2:
+            raise Exception('Invalid command in multi_regex_edit(): {}'.format(str(cmd)))
+        if cmd[0] == 'sub':
+            rx = re.compile(cmd[1])
+            replacement = cmd[2]
+            r = []
+            for l in strings:
+                r.append(re.sub(rx, replacement, l))
+            strings = r
+            continue
+        if cmd[0] == 'del':
+            rx = re.compile(cmd[1])
+            r = []
+            for l in strings:
+                if rx.search(l) is not None:
+                    continue
+                r.append(l)
+            strings = r
+            continue
+        if cmd[0] == 'match':
+            rx = re.compile(cmd[1])
+            r = []
+            for l in strings:
+                if rx.search(l) is not None:
+                    r.append(l)
+            strings = r
+            continue
+        raise Exception('Invalid command in multi_regex_edit(): {}'.format(str(cmd)))
+    return strings
+
+def dump(prio: int, objects: Iterable, *args, **kwargs) -> None: # export
+    caller = log.get_caller_pos(kwargs=kwargs)
+    log.slog(prio, ",---------- {}".format(' '.join(args)), caller=caller)
+    prefix = " | "
+    log.append_to_prefix(prefix)
+    i = 1
+    for o in objects:
+        o.dump(prio, "{} ({})".format(i, o.__class__.__name__), caller=caller, **kwargs)
+        i += 1
+    log.remove_from_prefix(prefix)
+    log.slog(prio, "`---------- {}".format(' '.join(args)), caller=caller)
+
+atexit.register(_cleanup)

src/python/jw/util/multi_key_dict.py

@@ -0,0 +1,621 @@
+# -*- coding: utf-8 -*-
+
+'''
+Created on 26 May 2013
+
+@author: lukasz.forynski
+
+@brief: Implementation of the multi-key dictionary.
+
+https://github.com/formiaczek/python_data_structures
+___________________________________
+
+ Copyright (c) 2014 Lukasz Forynski <lukasz.forynski@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this
+software and associated documentation files (the "Software"), to deal in the Software
+without restriction, including without limitation the rights to use, copy, modify, merge,
+publish, distribute, sub-license, and/or sell copies of the Software, and to permit persons
+to whom the Software is furnished to do so, subject to the following conditions:
+
+- The above copyright notice and this permission notice shall be included in all copies
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+'''
+
+import platform
+_python3 = int(platform.python_version_tuple()[0]) >= 3
+
+class multi_key_dict(object):
+    """ The purpose of this type is to provide a multi-key dictionary.
+    This kind of dictionary has a similar interface to the standard dictionary, and indeed if used 
+    with single key key elements - it's behaviour is the same as for a standard dict().
+
+    However it also allows for creation of elements using multiple keys (using tuples/lists).
+    Such elements can be accessed using either of those keys (e.g read/updated/deleted).
+    Dictionary provides also an extended interface for iterating over items and keys by the key type.
+    This can be useful e.g.: when creating dictionaries with (index,name) allowing one to iterate over
+    items using either: names or indexes. It can be useful for many many other similar use-cases,
+    and there is no limit to the number of keys used to map to the value.
+
+    There are also methods to find other keys mapping to the same value as the specified keys etc.
+    Refer to examples and test code to see it in action.
+
+    simple example:
+        k = multi_key_dict()
+        k[100] = 'hundred'  # add item to the dictionary (as for normal dictionary)
+
+        # but also:
+        # below creates entry with two possible key types: int and str, 
+        # mapping all keys to the assigned value
+        k[1000, 'kilo', 'k'] = 'kilo (x1000)'
+        print k[1000]   # will print 'kilo (x1000)'
+        print k['k']  # will also print 'kilo (x1000)'
+
+        # the same way objects can be updated, and if an object is updated using one key, the new value will
+        # be accessible using any other key, e.g. for example above:
+        k['kilo'] = 'kilo'
+        print k[1000] # will print 'kilo' as value was updated
+    """
+
+    def __init__(self, mapping_or_iterable=None, **kwargs):
+        """ Initializes dictionary from an optional positional argument and a possibly empty set of keyword arguments."""
+        self.items_dict = {}
+        if mapping_or_iterable is not None:
+            if type(mapping_or_iterable) is dict:
+                mapping_or_iterable = mapping_or_iterable.items()
+            for kv in mapping_or_iterable:
+                if len(kv) != 2:
+                    raise Exception('Iterable should contain tuples with exactly two values but specified: {0}.'.format(kv))
+                self[kv[0]] = kv[1]
+        for keys, value in kwargs.items():
+            self[keys] = value
+
+    def __getitem__(self, key):
+        """ Return the value at index specified as key."""
+        return self.items_dict[self.__dict__[str(type(key))][key]]
+
+    def __setitem__(self, keys, value):
+        """ Set the value at index (or list of indexes) specified as keys.
+            Note, that if multiple key list is specified, either: 
+              - none of keys should map to an existing item already (item creation), or 
+              - all of keys should map to exactly the same item (as previously created)
+                  (item update)
+            If this is not the case - KeyError is raised. """
+        if(type(keys) in [tuple, list]):
+            at_least_one_key_exists = False
+            num_of_keys_we_have = 0
+
+            for x in keys:
+                try:
+                    self.__getitem__(x)
+                    num_of_keys_we_have += 1
+                except Exception as err:
+                    continue
+
+            if num_of_keys_we_have:
+                all_select_same_item = True
+                direct_key = None
+                for key in keys:
+                    key_type = str(type(key))
+                    try:
+                        if not direct_key:
+                            direct_key = self.__dict__[key_type][key]
+                        else:
+                            new = self.__dict__[key_type][key]
+                            if new != direct_key:
+                                all_select_same_item = False
+                                break
+                    except Exception as err:
+                        all_select_same_item = False
+                        break;
+
+                if not all_select_same_item:
+                    raise KeyError(', '.join(str(key) for key in keys))
+
+            first_key = keys[0] # combination if keys is allowed, simply use the first one
+        else:
+            first_key = keys
+
+        key_type = str(type(first_key)) # find the intermediate dictionary..
+        if first_key in self:
+            self.items_dict[self.__dict__[key_type][first_key]] = value # .. and update the object if it exists..
+        else:
+            if(type(keys) not in [tuple, list]):
+                key = keys
+                keys = [keys]
+            self.__add_item(value, keys) # .. or create it - if it doesn't
+
+    def __delitem__(self, key):
+        """ Called to implement deletion of self[key]."""
+        key_type = str(type(key))
+
+        if (key in self and
+            self.items_dict and
+            (self.__dict__[key_type][key] in self.items_dict) ):
+            intermediate_key = self.__dict__[key_type][key]
+
+            # remove the item in main dictionary 
+            del self.items_dict[intermediate_key]
+
+            # and remove all references (if there were other keys)
+            for k in self.get_other_keys(key, True):
+                key_type = str(type(k))
+                if (key_type in self.__dict__ and k in self.__dict__[key_type]):
+                    del self.__dict__[key_type][k]
+
+        else:
+            raise KeyError(key)
+
+    def __contains__(self, key):
+        """ Returns True if this object contains an item referenced by the key."""
+        key_type = str(type(key))
+        if key_type in self.__dict__:
+            if key in self.__dict__[key_type]:
+                return True
+
+        return False
+
+    def has_key(self, key):
+        """ Returns True if this object contains an item referenced by the key."""
+        return key in self
+
+    def get_other_keys(self, key, including_current=False):
+        """ Returns list of other keys that are mapped to the same value as specified key. 
+            @param key - key for which other keys should be returned.
+            @param including_current if set to True - key will also appear on this list."""
+        other_keys = []
+        if key in self:
+            other_keys.extend(self.__dict__[str(type(key))][key])
+            if not including_current:
+                other_keys.remove(key)
+        return other_keys
+
+    def iteritems(self, key_type=None, return_all_keys=False):
+        """ Returns an iterator over the dictionary's (key, value) pairs.
+            @param key_type if specified, iterator will be returning only (key,value) pairs for this type of key.
+                   Otherwise (if not specified) ((keys,...), value) 
+                   i.e. (tuple of keys, values) pairs for all items in this dictionary will be generated.
+            @param return_all_keys if set to True - tuple of keys is retuned instead of a key of this type."""
+
+        if key_type is None:
+            for item in self.items_dict.items():
+                yield item
+            return
+        used_keys = set()
+        key = str(key_type)
+        if key in self.__dict__:
+            for key, keys in self.__dict__[key].items():
+                if keys in used_keys:
+                    continue
+                used_keys.add(keys)
+                value = self.items_dict[keys]
+                if not return_all_keys:
+                    keys = tuple(k for k in keys if isinstance(k, key_type))
+                yield keys, value
+
+    def iterkeys(self, key_type=None, return_all_keys=False):
+        """ Returns an iterator over the dictionary's keys.
+            @param key_type if specified, iterator for a dictionary of this type will be used. 
+                   Otherwise (if not specified) tuples containing all (multiple) keys
+                   for this dictionary will be generated.
+            @param return_all_keys if set to True - tuple of keys is retuned instead of a key of this type."""
+        if(key_type is not None):
+            the_key = str(key_type)
+            if the_key in self.__dict__:
+                for key in self.__dict__[the_key].keys():
+                    if return_all_keys:
+                        yield self.__dict__[the_key][key]
+                    else:
+                        yield key            
+        else:
+            for keys in self.items_dict.keys():
+                yield keys
+
+    def itervalues(self, key_type=None):
+        """ Returns an iterator over the dictionary's values.
+            @param key_type if specified, iterator will be returning only values pointed by keys of this type.
+                   Otherwise (if not specified) all values in this dictinary will be generated."""
+        if(key_type is not None):
+            intermediate_key = str(key_type)
+            if intermediate_key in self.__dict__:
+                for direct_key in self.__dict__[intermediate_key].values():
+                    yield self.items_dict[direct_key]
+        else:
+            for value in self.items_dict.values():
+                yield value
+
+    if _python3:
+        items = iteritems
+    else:
+        def items(self, key_type=None, return_all_keys=False):
+            return list(self.iteritems(key_type, return_all_keys))
+        items.__doc__ = iteritems.__doc__
+
+    def keys(self, key_type=None):
+        """ Returns a copy of the dictionary's keys.
+            @param key_type if specified, only keys for this type will be returned.
+                 Otherwise list of tuples containing all (multiple) keys will be returned."""
+        if key_type is not None:
+            intermediate_key = str(key_type)
+            if intermediate_key in self.__dict__:
+                return self.__dict__[intermediate_key].keys()
+        else:
+            all_keys = {} # in order to preserve keys() type (dict_keys for python3) 
+            for keys in self.items_dict.keys():
+                all_keys[keys] = None
+            return all_keys.keys()
+
+    def values(self, key_type=None):
+        """ Returns a copy of the dictionary's values.
+            @param key_type if specified, only values pointed by keys of this type will be returned.
+                 Otherwise list of all values contained in this dictionary will be returned."""
+        if(key_type is not None):
+            all_items = {} # in order to preserve keys() type (dict_values for python3) 
+            keys_used = set()
+            direct_key = str(key_type)
+            if direct_key in self.__dict__:
+                for intermediate_key in self.__dict__[direct_key].values():
+                    if not intermediate_key in keys_used:
+                        all_items[intermediate_key] = self.items_dict[intermediate_key]
+                        keys_used.add(intermediate_key)
+            return all_items.values()
+        else:
+            return self.items_dict.values()
+
+    def __len__(self):
+        """ Returns number of objects in dictionary."""
+        length = 0
+        if 'items_dict' in self.__dict__:
+            length = len(self.items_dict)
+        return length
+
+    def __add_item(self, item, keys=None):
+        """ Internal method to add an item to the multi-key dictionary"""
+        if(not keys or not len(keys)):
+            raise Exception('Error in %s.__add_item(%s, keys=tuple/list of items): need to specify a tuple/list containing at least one key!'
+                            % (self.__class__.__name__, str(item)))
+        direct_key = tuple(keys) # put all keys in a tuple, and use it as a key
+        for key in keys:
+            key_type = str(type(key))
+
+            # store direct key as a value in an intermediate dictionary
+            if(not key_type in self.__dict__):
+                self.__setattr__(key_type, dict())
+            self.__dict__[key_type][key] = direct_key
+
+            # store the value in the actual dictionary
+            if(not 'items_dict' in self.__dict__):
+                self.items_dict = dict()
+            self.items_dict[direct_key] = item
+
+    def get(self, key, default=None):
+        """ Return the value at index specified as key."""
+        if key in self:
+            return self.items_dict[self.__dict__[str(type(key))][key]]
+        else:
+            return default
+
+    def __str__(self):
+        items = []
+        str_repr = lambda x: '\'%s\'' % x if type(x) == str else str(x)
+        if hasattr(self, 'items_dict'):
+            for (keys, value) in self.items():
+                keys_str = [str_repr(k) for k in keys]
+                items.append('(%s): %s' % (', '.join(keys_str),
+                                           str_repr(value)))
+        dict_str = '{%s}' % ( ', '.join(items))
+        return dict_str
+
+def test_multi_key_dict():
+    contains_all = lambda cont, in_items: not (False in [c in cont for c in in_items])
+
+    m = multi_key_dict()
+    assert( len(m) == 0 ), 'expected len(m) == 0'
+    all_keys = list()
+
+    m['aa', 12, 32, 'mmm'] = 123  # create a value with multiple keys..
+    assert( len(m) == 1 ), 'expected len(m) == 1'
+    all_keys.append(('aa', 'mmm', 32, 12)) # store it for later
+
+    # try retrieving other keys mapped to the same value using one of them
+    res = m.get_other_keys('aa')
+    expected = ['mmm', 32, 12]
+    assert(set(res) == set(expected)), 'get_other_keys(\'aa\'): {0} other than expected: {1} '.format(res, expected)
+
+    # try retrieving other keys mapped to the same value using one of them: also include this key
+    res = m.get_other_keys(32, True)
+    expected = ['aa', 'mmm', 32, 12]
+    assert(set(res) == set(expected)), 'get_other_keys(32): {0} other than expected: {1} '.format(res, expected)
+
+    assert( m.has_key('aa') == True ), 'expected m.has_key(\'aa\') == True'
+    assert( m.has_key('aab') == False ), 'expected m.has_key(\'aab\') == False'
+
+    assert( m.has_key(12) == True ), 'expected m.has_key(12) == True'
+    assert( m.has_key(13) == False ), 'expected m.has_key(13) == False'
+    assert( m.has_key(32) == True ), 'expected m.has_key(32) == True'
+
+    m['something else'] = 'abcd'
+    assert( len(m) == 2 ), 'expected len(m) == 2'
+    all_keys.append(('something else',)) # store for later
+
+    m[23] = 0
+    assert( len(m) == 3 ), 'expected len(m) == 3'
+    all_keys.append((23,)) # store for later
+
+    # check if it's possible to read this value back using either of keys
+    assert( m['aa'] == 123 ), 'expected m[\'aa\'] == 123'
+    assert( m[12] == 123 ), 'expected m[12] == 123'
+    assert( m[32] == 123 ), 'expected m[32] == 123'
+    assert( m['mmm'] == 123 ), 'expected m[\'mmm\'] == 123'
+
+    # now update value and again - confirm it back - using different keys..
+    m['aa'] = 45
+    assert( m['aa'] == 45 ), 'expected m[\'aa\'] == 45'
+    assert( m[12] == 45 ), 'expected m[12] == 45'
+    assert( m[32] == 45 ), 'expected m[32] == 45'
+    assert( m['mmm'] == 45 ), 'expected m[\'mmm\'] == 45'
+
+    m[12] = '4'
+    assert( m['aa'] == '4' ), 'expected m[\'aa\'] == \'4\''
+    assert( m[12] == '4' ), 'expected m[12] == \'4\''
+
+    # test __str__
+    m_str_exp = '{(23): 0, (\'aa\', \'mmm\', 32, 12): \'4\', (\'something else\'): \'abcd\'}'
+    m_str = str(m)
+    assert (len(m_str) > 0), 'str(m) should not be empty!'    
+    assert (m_str[0] == '{'), 'str(m) should start with \'{\', but does with \'%c\'' % m_str[0]
+    assert (m_str[-1] == '}'), 'str(m) should end with \'}\', but does with \'%c\'' % m_str[-1]
+
+    # check if all key-values are there as expected. They might be sorted differently
+    def get_values_from_str(dict_str):
+        sorted_keys_and_values = []
+        for k in dict_str.split(', ('):
+            keys, val = k.strip('{}() ').replace(')', '').split(':')
+            keys = tuple(sorted([k.strip() for k in keys.split(',')]))
+            sorted_keys_and_values.append((keys, val))
+        return sorted_keys_and_values
+    exp = get_values_from_str(m_str_exp)
+    act = get_values_from_str(m_str)
+    assert (set(act) == set(exp)), 'str(m) values: \'{0}\' are not {1} '.format(act, exp)
+
+    # try accessing / creating new (keys)-> value mapping whilst one of these
+    # keys already maps to a value in this dictionaries
+    try:
+        m['aa', 'bb'] = 'something new'
+        assert(False), 'Should not allow adding multiple-keys when one of keys (\'aa\') already exists!'
+    except KeyError as err:
+        pass
+
+    # now check if we can get all possible keys (formed in a list of tuples)
+    # each tuple containing all keys)
+    res = sorted([sorted([str(x) for x in k]) for k in m.keys()])
+    expected = sorted([sorted([str(x) for x in k]) for k in all_keys])
+    assert(res == expected), 'unexpected values from m.keys(), got:\n%s\n expected:\n%s' %(res, expected) 
+
+    # check default items (which will unpack tupe with key(s) and value)
+    num_of_elements = 0
+    for keys, value in m.items():
+        sorted_keys = sorted([str(k) for k in keys])
+        num_of_elements += 1
+        assert(sorted_keys in expected), 'm.items(): unexpected keys: %s' % (sorted_keys)
+        assert(m[keys[0]] == value), 'm.items(): unexpected value: %s (keys: %s)' % (value, keys)
+    assert(num_of_elements > 0), 'm.items() returned generator that did not produce anything'
+
+    # test default iterkeys()
+    num_of_elements = 0
+    for keys in m.keys():
+        num_of_elements += 1
+        keys_s = sorted([str(k) for k in keys])
+        assert(keys_s in expected), 'm.keys(): unexpected keys: {0}'.format(keys_s)
+
+    assert(num_of_elements > 0), 'm.iterkeys() returned generator that did not produce anything'
+
+    # test iterkeys(int, True): useful to get all info from the dictionary
+    # dictionary is iterated over the type specified, but all keys are returned.
+    num_of_elements = 0
+    for keys in m.iterkeys(int, True):
+        keys_s = sorted([str(k) for k in keys])
+        num_of_elements += 1
+        assert(keys_s in expected), 'm.iterkeys(int, True): unexpected keys: {0}'.format(keys_s)
+    assert(num_of_elements > 0), 'm.iterkeys(int, True) returned generator that did not produce anything'
+
+
+    # test values for different types of keys()
+    expected = set([0, '4'])
+    res = set(m.values(int))
+    assert (res == expected), 'm.values(int) are {0}, but expected: {1}.'.format(res, expected)
+
+    expected = sorted(['4', 'abcd'])
+    res = sorted(m.values(str)) 
+    assert (res == expected), 'm.values(str) are {0}, but expected: {1}.'.format(res, expected)
+
+    current_values = set([0, '4', 'abcd']) # default (should give all values)
+    res = set(m.values()) 
+    assert (res == current_values), 'm.values() are {0}, but expected: {1}.'.format(res, current_values)
+
+    #test itervalues() (default) - should return all values. (Itervalues for other types are tested below)
+    vals = set()
+    for value in m.itervalues():
+        vals.add(value)
+    assert (current_values == vals), 'itervalues(): expected {0}, but collected {1}'.format(current_values, vals)
+
+    #test items(int)
+    items_for_int = sorted([((12, 32), '4'), ((23,), 0)])
+    assert (items_for_int == sorted(m.items(int))), 'items(int): expected {0}, but collected {1}'.format(items_for_int, 
+                                                                                                     sorted(m.items(int)))
+
+    # test items(str)
+    items_for_str = set([(('aa','mmm'), '4'), (('something else',), 'abcd')])
+    res = set(m.items(str))
+    assert (set(res) == items_for_str), 'items(str): expected {0}, but collected {1}'.format(items_for_str, res)
+
+    # test items() (default - all items)
+    # we tested keys(), values(), and __get_item__ above so here we'll re-create all_items using that 
+    all_items = set()
+    keys = m.keys()
+    values = m.values()
+    for k in keys:
+        all_items.add( (tuple(k), m[k[0]]) )
+
+    res = set(m.items())
+    assert (all_items == res), 'items() (all items): expected {0},\n\t\t\t\tbut collected {1}'.format(all_items, res)
+
+    # now test deletion..
+    curr_len = len(m)
+    del m[12]
+    assert( len(m) == curr_len - 1 ), 'expected len(m) == %d' % (curr_len - 1)
+    assert(not m.has_key(12)), 'expected deleted key to no longer be found!'
+
+    # try again 
+    try:
+        del m['aa']
+        assert(False), 'cant remove again: item m[\'aa\'] should not exist!'
+    except KeyError as err:
+        pass
+
+    # try to access non-existing 
+    try:
+        k =  m['aa']
+        assert(False), 'removed item m[\'aa\'] should not exist!'
+    except KeyError as err:
+        pass
+
+    # try to access non-existing with a different key 
+    try:
+        k =  m[12]
+        assert(False), 'removed item m[12] should not exist!'
+    except KeyError as err:
+        pass
+
+    # prepare for other tests (also testing creation of new items)
+    del m
+    m = multi_key_dict()
+    tst_range = list(range(10, 40)) + list(range(50, 70))
+    for i in tst_range:
+        m[i] = i # will create a dictionary, where keys are same as items
+
+    # test items()
+    for key, value in m.items(int):
+        assert(key == (value,)), 'items(int): expected {0}, but received {1}'.format(key, value)
+
+    # test iterkeys()
+    num_of_elements = 0
+    returned_keys = set()
+    for key in m.iterkeys(int):
+        returned_keys.add(key)
+        num_of_elements += 1
+    assert(num_of_elements > 0), 'm.iteritems(int) returned generator that did not produce anything'
+    assert (returned_keys == set(tst_range)), 'iterkeys(int): expected {0}, but received {1}'.format(expected, key)
+
+
+    #test itervalues(int)
+    num_of_elements = 0
+    returned_values  = set()
+    for value in m.itervalues(int):
+        returned_values.add(value)
+        num_of_elements += 1
+    assert (num_of_elements > 0), 'm.itervalues(int) returned generator that did not produce anything'
+    assert (returned_values == set(tst_range)), 'itervalues(int): expected {0}, but received {1}'.format(expected, value)
+
+    # test values(int)
+    res = sorted([x for x in m.values(int)])
+    assert (res == tst_range), 'm.values(int) is not as expected.'
+
+    # test keys()
+    assert (set(m.keys(int)) == set(tst_range)), 'm.keys(int) is not as expected.'
+
+    # test setitem with multiple keys
+    m['xy', 999, 'abcd'] = 'teststr'
+    try:
+        m['xy', 998] = 'otherstr'
+        assert(False), 'creating / updating m[\'xy\', 998] should fail!'
+    except KeyError as err:
+        pass
+
+    # test setitem with multiple keys
+    m['cd'] = 'somethingelse'
+    try:
+        m['cd', 999] = 'otherstr'
+        assert(False), 'creating / updating m[\'cd\', 999] should fail!'
+    except KeyError as err:
+        pass
+
+    m['xy', 999] = 'otherstr'
+    assert (m['xy']  == 'otherstr'), 'm[\'xy\'] is not as expected.'
+    assert (m[999]   == 'otherstr'), 'm[999] is not as expected.'
+    assert (m['abcd'] == 'otherstr'), 'm[\'abcd\'] is not as expected.'
+
+    m['abcd', 'xy']   =  'another'
+    assert (m['xy']  == 'another'), 'm[\'xy\'] is not == \'another\'.'
+    assert (m[999]   == 'another'), 'm[999] is not == \'another\''
+    assert (m['abcd'] == 'another'), 'm[\'abcd\'] is not  == \'another\'.'
+
+    # test get functionality of basic dictionaries
+    m['CanIGet'] = 'yes'
+    assert (m.get('CanIGet') == 'yes')
+    assert (m.get('ICantGet') == None)
+    assert (m.get('ICantGet', "Ok") == "Ok")
+
+    k = multi_key_dict()
+    k['1:12', 1] = 'key_has_:'
+    k.items() # should not cause any problems to have : in key
+    assert (k[1] == 'key_has_:'), 'k[1] is not equal to \'abc:def:ghi\''
+
+    import datetime
+    n = datetime.datetime.now()
+    l = multi_key_dict()
+    l[n] = 'now' # use datetime obj as a key
+
+    #test keys..
+    res = [x for x in l.keys()][0] # for python3 keys() returns dict_keys dictionarly
+    expected = n,
+    assert(expected == res), 'Expected \"{0}\", but got: \"{1}\"'.format(expected, res)
+
+    res = [x for x in l.keys(datetime.datetime)][0]
+    assert(n == res), 'Expected {0} as a key, but got: {1}'.format(n, res)
+    
+    res = [x for x in l.values()] # for python3 keys() returns dict_values dictionarly
+    expected = ['now']
+    assert(res == expected), 'Expected values: {0}, but got: {1}'.format(expected, res)
+
+    # test items..
+    exp_items = [((n,), 'now')]
+    r = list(l.items())
+    assert(r == exp_items), 'Expected for items(): tuple of keys: {0}, but got: {1}'.format(r, exp_items) 
+    assert(exp_items[0][1] == 'now'), 'Expected for items(): value: {0}, but got: {1}'.format('now', 
+                                                                                              exp_items[0][1])
+
+    x = multi_key_dict({('k', 'kilo'):1000, ('M', 'MEGA', 1000000):1000000}, milli=0.01)
+    assert (x['k'] == 1000), 'x[\'k\'] is not equal to 1000'
+    x['kilo'] = 'kilo'
+    assert (x['kilo'] == 'kilo'), 'x[\'kilo\'] is not equal to \'kilo\''
+
+    y = multi_key_dict([(('two', 'duo'), 2), (('one', 'uno'), 1), ('three', 3)])
+
+    assert (y['two'] == 2), 'y[\'two\'] is not equal to 2'
+    y['one'] = 'one'
+    assert (y['one'] == 'one'), 'y[\'one\'] is not equal to \'one\''
+
+    try:
+        y = multi_key_dict([(('two', 'duo'), 2), ('one', 'uno', 1), ('three', 3)])
+        assert(False), 'creating dictionary using iterable with tuples of size > 2 should fail!'
+    except:
+        pass
+
+    print ('All test passed OK!')
+
+__all__ = ["multi_key_dict"]
+
+if __name__ == '__main__':
+    try:
+        test_multi_key_dict()
+    except KeyboardInterrupt:
+        print ('\n(interrupted by user)')
+

src/python/jw/util/stree/Makefile

@@ -0,0 +1,4 @@
+TOPDIR = ../../../../..
+
+include $(TOPDIR)/make/proj.mk
+include $(JWBDIR)/make/py-mod.mk

src/python/jw/util/stree/StringTree.py

@@ -0,0 +1,302 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from typing import Any, List, Optional, Union
+
+import re, fnmatch
+from collections import OrderedDict
+from enum import Enum, auto
+
+from ..log import *
+
+def quote(s):
+    if is_quoted(s):
+        return s
+    s = s.strip()
+    if len(s) > 0:
+        if s[0] == '"':
+            return "'" + s + "'"
+    return '"' + s + '"'
+
+def is_quoted(s: str) -> bool:
+    if isinstance(s, StringTree):
+        return False
+    s = s.strip()
+    if len(s) < 2:
+        return False
+    d = s[0]
+    if d == s[-1] and d in [ '"', "'" ]:
+        return True
+    return False
+
+def cleanup_string(s: str) -> str:
+    if isinstance(s, StringTree):
+        return s
+    s = s.strip()
+    if is_quoted(s):
+        return s[1:-1].replace('\\' + s[0], s[0])
+    return s
+
+class StringTree: # export
+
+    def __init__(self, path: str, content: str, parent: StringTree|None=None) -> None:
+        slog(DEBUG, f'Constructing StringTree(path="{path}", content="{content}")')
+        self.__parent = parent
+        self.children: OrderedDict[str, StringTree] = OrderedDict()
+        self.content: Optional[str] = None
+        self.__set(path, content)
+
+        assert(hasattr(self, "content"))
+        #assert self.content is not None
+
+    # root (content = [ symbols ])
+    #  symbols (content = [ regex ])
+    #    regex ( content ='[ \n\t\r]+' )
+
+    # root (content = root, children = [ symbols ])
+    #  symbols (content = symbols, children = [ regex ])
+    #    regex ( content = regex, children = [ '[ \n\t\r]+' ] )
+    #      '[ \n\t\r]+)' ( content = '\n\t\r]+)', children = [] )
+
+    def __adopt_children(self, parent):
+        assert isinstance(parent, StringTree)
+        slog(DEBUG, f'At {self.content}: Adopting children of {parent}')
+        #parent.dump(INFO, "These children are added")
+        self.content = parent.content
+        for name, c in parent.children.items():
+            if not name in self.children.keys():
+                slog(DEBUG, f'At {self.content}: Adding new child {c}')
+                self.children[name] = c
+            else:
+                self.children[name].__adopt_children(c)
+
+    def __set(self, path_, content, split=True):
+        slog(DEBUG, ('At "{}": '.format(str(self.content)) if hasattr(self, "content") else "") + f'Setting "{path_}" -> "{content}"')
+        #assert self.content != str(content) # Not sure what the idea behind this was. It often goes off, and all works fine without.
+        if content is not None and not type(content) in [str, StringTree]:
+            raise Exception("Tried to add content of unsupported type {}".format(type(content).__name__))
+        if path_ is None:
+            if isinstance(content, str):
+                self.content = cleanup_string(content)
+            elif isinstance(content, StringTree):
+                self.__adopt_children(content)
+            else:
+                raise Exception("Tried to add content of unsupported type {}".format(type(content).__name__))
+            slog(DEBUG, " -- content = >" + str(content) + "<, self.content = >" + str(self.content) + "<")
+            return self
+        path = cleanup_string(path_)
+        components = path.split('.') if split else [ path ]
+        l = len(components)
+        if len(path) == 0 or l == 0:
+            #assert self.content is None or (isinstance(content, StringTree) and content.content == self.content)
+            if isinstance(content, StringTree):
+                #assert isinstance(content, StringTree), "Type: " + type(content).__name__
+                self.__adopt_children(content)
+            else:
+                if self.content != content:
+                    #self.content = cleanup_string(content)
+                    slog(DEBUG, f'Changing content: "{self.content}" ->"{content}"')
+                    assert(content != '"[a-zA-Z0-9+_*/-]"')
+                    self.content = content
+                    #assert(content != "'antlr_doesnt_understand_vertical_tab'")
+                    #self.children[content] = StringTree(None, content)
+            return self
+
+        #assert self.content is not None, "tried to set empty content to {}".format(path_)
+
+        nibble = components[0]
+        rest = '.'.join(components[1:])
+        if nibble not in self.children:
+            self.children[nibble] = StringTree('', content=nibble, parent=self)
+        if l > 1:
+            assert len(rest) > 0
+            return self.children[nibble].__set(rest, content=content)
+        # last component, a.k.a. leaf
+        if content is not None:
+            gc = content if isinstance(content, StringTree) else StringTree('', content=content, parent=self.children[nibble])
+            # Make sure no existing grand child is updated. It would reside too
+            # far up in the grand child OrderedDict, we need it last
+            if gc.content in self.children[nibble].children:
+                del self.children[nibble].children[gc.content]
+            self.children[nibble].children[gc.content] = gc
+        return self.children[nibble]
+
+    def __str__(self) -> str:
+        return 'st:"{}"'.format(self.content)
+
+    def __getitem__(self, path: str) -> str:
+        r = self.get(path)
+        if r is None:
+            raise KeyError(path)
+        return r.value() # type: ignore
+
+    def __setitem__(self, key, value):
+        return self.__set(key, value)
+
+    def __dump(self, prio, indent=0, **kwargs):
+        caller = kwargs['caller'] if 'caller' in kwargs.keys() else get_caller_pos(1)
+        slog(prio, '|' + (' ' * indent) + str(self.content), caller=caller)
+        indent += 2
+        for name, child in self.children.items():
+            child.__dump(prio, indent=indent, caller=caller)
+
+    @property
+    def path(self):
+        if self.__parent is None:
+            return ''
+        prefix = self.__parent.path
+        if len(prefix):
+            prefix += '.'
+        return prefix + self.content
+
+    def keys(self):
+        return self.children.keys()
+
+    def items(self):
+        return self.children.items()
+
+    def set_content(self, content):
+        if content is None:
+            raise Exception("Tried to set none content")
+        content = cleanup_string(content)
+        if len(content) == 0:
+            raise Exception("Tried to set empty content")
+        self.content = content
+
+    def add(self, path: str, content: Optional[Union[str, StringTree]] = None, split: bool = True) -> StringTree:
+        slog(DEBUG, f'-- At "{self.content}": Adding "{path}" -> "{content}"')
+        return self.__set(path, content, split)
+
+    def get(self, path_: str) -> Optional[StringTree]:
+        slog(DEBUG, 'looking for "{}" in "{}"'.format(path_, self.content))
+        assert not isinstance(path_, int)
+        path = cleanup_string(path_)
+        if len(path) == 0:
+            slog(DEBUG, "returning myself")
+            return self
+        if is_quoted(path_):
+            if not path in self.children.keys():
+                return None
+            return self.children[path]
+        components = path.split('.')
+        if len(components) == 0:
+            return self
+        name = cleanup_string(components[0])
+        if not hasattr(self, "children"):
+            return None
+        if not name in self.children.keys():
+            slog(DEBUG, "Name \"" + name + "\" is not in children of", self.content)
+            for child in self.children:
+                slog(DEBUG, "child = ", child)
+            return None
+        relpath = '.'.join(components[1:])
+        return self.children[name].get(relpath)
+
+    def value(self, path = None, default=None) -> Optional[str]:
+        if path:
+            child = self.get(path)
+            if child is None:
+                if default:
+                    return default
+                return None
+            return child.value()
+        if len(self.children) == 0:
+            raise Exception('tried to get value from leaf "{}"'.format(self.content))
+        slog(DEBUG, f'Returning value from children {self.children}')
+        return self.children[next(reversed(self.children))].content # type: ignore
+
+    @property
+    def parent(self):
+        return self.__parent
+
+    @property
+    def root(self):
+        if self.__parent is None:
+            return self
+        return self.__parent.root
+
+    def child_list(self, depth_first: bool=True) -> List[StringTree]:
+        if depth_first == False:
+            raise Exception("tried to retrieve child list with breadth-first search, not yet implemented")
+        r = []
+        for name, c in self.children.items():
+            r.append(c)
+            r.extend(c.child_list())
+        return r
+
+    def dump(self, prio: int, *args, **kwargs) -> None:
+        caller = kwargs['caller'] if 'caller' in kwargs.keys() else get_caller_pos(1)
+        msg = ''
+        if args is not None:
+            msg = ' ' + ' '.join(args) + ' '
+        slog(prio, ",------------" + msg + "----------- >", caller=caller)
+        self.__dump(prio, indent=0, caller=caller)
+        slog(prio, "`------------" + msg + "----------- <", caller=caller)
+
+    class Match(Enum):
+        Equal      = auto()
+        RegExArg   = auto()
+        RegExConf  = auto()
+        GlobArg    = auto()
+        GlobConf   = auto()
+
+    def __find(self, key: str|None, val: str|None, match: Match, depth_first: bool):
+
+        def __children():
+            for name, child in self.children.items():
+                ret.extend(child.__find(key, val, match, depth_first))
+
+        def __self():
+            _val = self.value()
+            _content = self.content
+            try:
+                if (
+                      (key == _content and matcher(val, _val))
+                   or (key is None     and matcher(val, _val))
+                   or (key == _content and val is None)
+                ):
+                    ret.append(self)
+            except Exception as e:
+                if isinstance(e, re.PatternError):
+                    pass
+                else:
+                    raise
+
+        def __debug_matcher(matcher, log_level=DEBUG):
+            def __matcher(x, y):
+                slog(log_level, f'Comparing "{x}" ~ "{y}"')
+                return matcher(x, y)
+            return __matcher
+
+        if not self.children:
+            return []
+
+        matcher = lambda x, y: x == y
+        match match:
+            case self.Match.Equal:
+                pass
+            case self.Match.RegExArg:
+                matcher = lambda x, y: re.search(x, y) is not None
+            case self.Match.RegExConf:
+                matcher = lambda x, y: re.search(y, x) is not None
+            case self.Match.GlobArg:
+                matcher = lambda x, y: fnmatch.fnmatch(y, x)
+            case self.Match.GlobConf:
+                matcher = lambda x, y: fnmatch.fnmatch(x, y)
+            case _:
+                raise NotImplementedError(f'Matcher {match} is not yet implemented')
+
+        ret: list[StringTree] = []
+
+        if depth_first:
+            __children()
+            __self()
+        else:
+            __self()
+            __children()
+
+        return ret
+
+    def find(self, key: str|None=None, val: str|None=None, match: Match=Match.Equal, depth_first: bool=False):
+        return [ node.parent.path for node in self.__find(key, val, match=match, depth_first=depth_first)]

src/python/jw/util/stree/serdes.py

@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+
+import os, glob
+
+from .StringTree import *
+from ..log import *
+
+def _cleanup_line(line: str) -> str:
+    line = line.strip()
+    r = ''
+    in_quote = None
+    for c in line:
+        # slog(DEBUG, "in_quote = >" + str(in_quote) + "<")
+        if in_quote is not None:
+            if c == in_quote:
+                in_quote = None
+        else:
+            if c in [ '"', "'" ]:
+                in_quote = c
+            elif in_quote is None and c == '#':
+                return r.strip()
+        r += c
+    if len(r) >= 2 and r[0] in [ '"', "'" ] and r[-1] == r[0]:
+        return r[1:-1]
+    return r
+
+def parse(s: str, allow_full_lines: bool=True, root_content: str='root') -> StringTree: # export
+    slog_m(DEBUG, "--->--- parsing --->---\n" + s + "\n---<--- parsing ---<---\n")
+    root = StringTree('', content=root_content)
+    sec = ''
+    for line in s.splitlines():
+        slog(DEBUG, f'Parsing: "{line}"')
+        line = _cleanup_line(line)
+        #slog(DEBUG, "cleaned line=", line)
+        if not len(line):
+            continue
+        if line[0] == '[':
+            if line[-1] == ']':
+                sec = line[1:-1]
+            elif line[-1] == '[':
+                if len(sec):
+                    sec += '.'
+                sec += line[1:-1]
+            else:
+                raise Exception("failed to parse section line", line)
+            if root.get(sec) is None:
+                root.add(sec)
+            continue
+        elif line[0] == ']':
+            assert(len(sec) > 0)
+            sec = '.'.join(sec.split('.')[0:-1])
+            continue
+        lhs = ''
+        rhs = None
+        for c in line:
+            if rhs is None:
+                if c == '=':
+                    rhs = ''
+                    continue
+                lhs += c
+                continue
+            rhs += c
+
+        split = True
+        if rhs is None:
+            if not allow_full_lines:
+                raise Exception("failed to parse assignment", line)
+            rhs = 'empty'
+            split = False
+        root.add(sec + '.' + cleanup_string(lhs), cleanup_string(rhs), split=split)
+    return root
+
+def _read_lines_from_one_path(path: str, throw=True, level=0, log_prio=INFO, paths_buf=None):
+    try:
+        with open(path, 'r') as infile:
+            slog(log_prio, 'Reading {}"{}"'.format(' ' * level * 2, path))
+            if paths_buf is not None:
+                paths_buf.append(path)
+            ret = []
+            for line in infile: # lines are all trailed by \n
+                m = re.search(r'^\s*(-)*include\s+(\S+)', line)
+                if m:
+                    optional = m.group(1) == '-'
+                    include_path = m.group(2)
+                    if include_path[0] != '/':
+                        dir_name = os.path.dirname(path)
+                        if len(dir_name):
+                            include_path = dir_name + '/' + include_path
+                    include_lines = _read_lines(include_path, throw=(not optional), level=level+1, paths_buf=paths_buf)
+                    if include_lines is None:
+                        slog(DEBUG, f'{path}: Failed to process "{line}"')
+                        continue
+                    ret.extend(include_lines)
+                    continue
+                ret.append(line)
+            return ret
+    except Exception as e:
+        slog(ERR, f'Failed to read file "{path}": {str(e)}')
+        if throw or not isinstance(e, FileNotFoundError):
+            raise
+    return None
+
+def _read_lines(path: str, throw=True, level=0, log_prio=INFO, paths_buf=None):
+    paths = glob.glob(path)
+    ret = []
+    for p in paths:
+        rr = _read_lines_from_one_path(p, throw=throw, level=level, log_prio=log_prio, paths_buf=paths_buf)
+        if rr is None:
+            return None
+        ret.extend(rr)
+    return ret
+
+def read(path: str, root_content: str='root', log_prio=INFO, paths_buf=None) -> StringTree: # export
+    lines = _read_lines_from_one_path(path, log_prio=log_prio, paths_buf=paths_buf)
+    s = ''.join(lines)
+    return parse(s, root_content=root_content)