jw-pkg/src/python/jw/pkg/cmds/projects/CmdBuild.py

# -*- coding: utf-8 -*-

import os, re, sys, subprocess, datetime, time
from argparse import Namespace, ArgumentParser
from functools import lru_cache

from ...lib.log import *
from ..Cmd import Cmd
from ..CmdProjects import CmdProjects
from ...App import Scope

class CmdBuild(Cmd): # export

    def __init__(self, parent: CmdProjects) -> None:
        super().__init__(parent, 'build', help='janware software project build tool')

    def add_arguments(self, parser: ArgumentParser) -> None:
        super().add_arguments(parser)
        parser.add_argument('--exclude', default='', help='Space seperated ist of modules to be excluded from build')
        parser.add_argument('-n', '--dry-run', action='store_true',
            default=False, help='Don\'t build anything, just print what would be done.')
        parser.add_argument('-O', '--build-order', action='store_true',
            default=False, help='Don\'t build anything, just print the build order.')
        parser.add_argument('-I', '--ignore-deps', action='store_true',
            default=False, help='Don\'t build dependencies, i.e. build only modules specified on the command line')
        parser.add_argument('target', default='all', help='Build target')
        parser.add_argument('modules', nargs='+', default='', help='Modules to be built')

    async def _run(self, args: Namespace) -> None:

        @lru_cache(maxsize=None)
        def read_deps(cur, prereq_type):
            # dep cache doesn't make a difference at all
            if prereq_type in dep_cache:
                if cur in dep_cache[prereq_type]:
                    return dep_cache[prereq_type][cur]
            else:
                dep_cache[prereq_type]: dict[str, str] = {}

            ret = self.app.get_project_refs([ cur ], ['pkg.requires.jw'],
                prereq_type, scope = Scope.Subtree, add_self=False, names_only=True)
            log(DEBUG, 'prerequisites = ' + ' '.join(ret))
            if cur in ret:
                ret.remove(cur)
            log(DEBUG, 'inserting', prereq_type, "prerequisites of", cur, ":", ' '.join(ret))
            dep_cache[prereq_type][cur] = ret
            return ret

        def add_dep_tree(cur, prereq_types, tree, all_deps):
            log(DEBUG, "adding prerequisites " + ' '.join(prereq_types) + " of module " + cur)
            if cur in all_deps:
                log(DEBUG, 'already handled module ' + cur)
                return 0
            deps = set()
            all_deps.add(cur)
            for t in prereq_types:
                log(DEBUG, "checking prereqisites of type " + t)
                deps.update(read_deps(cur, t))
            for d in deps:
                add_dep_tree(d, prereq_types, tree, all_deps)
            tree[cur] = deps
            return len(deps)

        def calculate_order(order, modules, prereq_types):
            all_deps = set()
            dep_tree = {}
            for m in modules:
                log(DEBUG, "--- adding dependency tree of module " + m)
                add_dep_tree(m, prereq_types, dep_tree, all_deps)
            while len(all_deps):
                # Find any leaf
                for d in all_deps:
                    if not len(dep_tree[d]): # Dependency d doesn't have dependencies itself
                        break # found
                else: # no Leaf found
                    print(all_deps)
                    raise Exception("fatal: the dependencies between these modules are unresolvable")
                order.append(d) # do it
                # bookkeep it
                all_deps.remove(d)
                for k in dep_tree.keys():
                    if d in dep_tree[k]:
                        dep_tree[k].remove(d)
            return 1

        def run_make(module, target, cur_project, num_projects):
            #make_cmd = "make " + target + " 2>&1"
            make_cmd = [ "make", target ]
            path = self.app.find_dir(module, pretty=False)
            delim_len = 120
            delim = '---- [%d/%d]: running %s in %s -' % (cur_project, num_projects, make_cmd, path)
            delim = delim + '-' * (delim_len - len(delim))

            print(',' + delim + ' >')

            patt = self.app.is_excluded_from_build(module)
            if patt is not None:
                    print('| Configured to skip build on platform >' + patt + '<')
                    print('`' + delim + ' <')
                    return

            os.chdir(path)
            p = subprocess.Popen(make_cmd, shell=False, stdout=subprocess.PIPE, stderr=None, close_fds=True)
            for line in iter(p.stdout.readline, b''):
                line = line.decode(sys.stdout.encoding)
                sys.stdout.write('| ' + line) # avoid extra newlines from print()
                sys.stdout.flush()
            p.wait()
            print('`' + delim + ' <')
            if p.returncode:
                print(' '.join(make_cmd) + ' failed')
                raise Exception(time.strftime("%Y-%m-%d %H:%M") + ": failed to make target " + target + " in module " + module + " below base " + self.app.projs_root)

        def run_make_on_modules(modules, order, target):
            cur_project = 0
            num_projects = len(order)
            if target in ["clean", "distclean"]:
                for m in reversed(order):
                    cur_project += 1
                    run_make(m, target, cur_project, num_projects)
                    if m in modules:
                        modules.remove(m)
                    if not len(modules):
                        print("all modules cleaned")
                        return
            else:
                for m in order:
                    cur_project += 1
                    run_make(m, target, cur_project, num_projects)

        def run(args):

            log(DEBUG, "----------------------------------------- running ", ' '.join(sys.argv))

            modules = args.modules
            exclude = args.exclude.split()
            target = args.target

            env_exclude = os.getenv('BUILD_EXCLUDE', '')
            if len(env_exclude):
                print("exluding modules from environment: " + env_exclude)
                exclude += " " + env_exclude

            # -- build
            order = []

            glob_prereq_types = [ "build" ]
            if re.match("pkg-.*", target) is not None:
                glob_prereq_types = [ "build", "run", "release", "devel" ]

            if target != 'order' and not args.build_order:
                print("using prerequisite types " + ' '.join(glob_prereq_types))
                print("calculating order for modules ... ")

            calculate_order(order, modules, glob_prereq_types)
            if args.ignore_deps:
                order = [m for m in order if m in args.modules]
            order = [m for m in order if m not in exclude]
            if target == 'order' or args.build_order:
                print(' '.join(order))
                exit(0)

            cur_project = 0
            print("Building target %s in %d projects:" % (target, len(order)))
            for m in order:
                cur_project += 1
                print("  %3d %s" % (cur_project, m))

            if args.dry_run:
                exit(0)

            run_make_on_modules(modules, order, target)

            print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))

        dep_cache: dict[dict[str, str]] = {}

        run(args)