From e739b8f0b85d39ebdb1afc0809f6b3daed7a766b Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 25 Jun 2026 18:09:56 +0200 Subject: [PATCH 1/4] python-tools.sh: Next attempt to satisfy formatter Commit 55060486 does not satisfy yapf in all places of the source code, so have another attempt at it. Signed-off-by: Jan Lindemann --- scripts/python-tools.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/python-tools.sh b/scripts/python-tools.sh index aa83d8f5..b9724178 100644 --- a/scripts/python-tools.sh +++ b/scripts/python-tools.sh @@ -81,12 +81,12 @@ cmd_create_init() echo "import `module_path $f` as $f" __add_seen $f done - echo fi if [ ${#seen[@]} -eq 0 ]; then echo "__all__ = []" else + echo echo "__all__ = [" for dst_type in ${!seen[@]}; do echo " \"$dst_type\"," From 8c093733f43b144faa9a6947111cb214d4a7295e Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 25 Jun 2026 17:04:40 +0200 Subject: [PATCH 2/4] test/ .. /help: Fail over missing toplevel "make all" The help integration test currently only succeeds because CI doesn't run "make all" before "make test". That way, the checked out repository lacks the generated __init__.py files needed for some modular subcommands to be fully loaded, and hence, the test should have failed. The entire machinery only worked because the subcommands in question are not not essential to building jw-pkg itself: "secrets" and "posix" The fix is to 1. Change CI to run "make all" before "make test" (not here) 2. Fix the test in jw-pkg Signed-off-by: Jan Lindemann --- .../integration/jw-pkg/help/test-expected.txt | 131 +++++++++++++++++- 1 file changed, 129 insertions(+), 2 deletions(-) diff --git a/test/integration/jw-pkg/help/test-expected.txt b/test/integration/jw-pkg/help/test-expected.txt index dec09556..4ebbee24 100644 --- a/test/integration/jw-pkg/help/test-expected.txt +++ b/test/integration/jw-pkg/help/test-expected.txt @@ -163,12 +163,55 @@ options: %{codename}, %{gnu-triplet}, %{os-cascade}, %{os-release}, %{pkg-ext} (default: %{cascade}) ============= Running: jw-pkg.py --log-level info posix --help -usage: jw-pkg.py posix [-h] +usage: jw-pkg.py posix [-h] ... Perform various operations on a distro through its POSIX utility interface options: -h, --help show this help message and exit + +Available subcommands of posix: + + copy Copy files + tar Handle tar archives +============= Running: jw-pkg.py --log-level info posix copy --help +usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-F] src dst + +Copy files + +positional arguments: + src Source file URI + dst Destination file URI + +options: + -h, --help show this help message and exit + -o, --owner OWNER Destination file owner (default: None) + -g, --group GROUP Destination file group (default: None) + -m, --mode MODE Destination file mode (default: None) + -F, --fixed-strings Don't expand macros in and (default: False) +============= Running: jw-pkg.py --log-level info posix tar --help +usage: jw-pkg.py posix tar [-h] ... + +Handle tar archives + +options: + -h, --help show this help message and exit + +Available subcommands of tar: + + x Extract a tar archive +============= Running: jw-pkg.py --log-level info posix tar x --help +usage: jw-pkg.py posix tar x [-h] -f ARCHIVE_PATH dst + +Extract a tar archive + +positional arguments: + dst Destination root URI + +options: + -h, --help show this help message and exit + -f, --archive-path ARCHIVE_PATH + Archive path ============= Running: jw-pkg.py --log-level info projects --help usage: jw-pkg.py projects [-h] ... @@ -689,9 +732,93 @@ positional arguments: options: -h, --help show this help message and exit ============= Running: jw-pkg.py --log-level info secrets --help -usage: jw-pkg.py secrets [-h] +usage: jw-pkg.py secrets [-h] ... Manage package secrets +options: + -h, --help show this help message and exit + +Available subcommands of secrets: + + compile-templates Compile package template files + install Install secrets from various sources as static secrets + onto the target + list-compilation-output + List package compilation output files + list-secrets List package secret files + list-templates List package template files + rm-compilation-output + Remove package compilation output files +============= Running: jw-pkg.py --log-level info secrets compile-templates --help +usage: jw-pkg.py secrets compile-templates [-h] [--owner OWNER] + [--group GROUP] [--mode MODE] + [packages ...] + +Compile package template files + +positional arguments: + packages Package names (default: None) + +options: + -h, --help show this help message and exit + --owner, -o OWNER Default output file owner (default: None) + --group, -g GROUP Default output file group (default: None) + --mode, -m MODE Default output file mode (default: None) +============= Running: jw-pkg.py --log-level info secrets install --help +usage: jw-pkg.py secrets install [-h] [--only-missing] src [packages ...] + +Install secrets from various sources as static secrets onto the target + +positional arguments: + src URI of secret source + packages Package names (default: None) + +options: + -h, --help show this help message and exit + --only-missing Install only secrets not already on the target (default: + False) +============= Running: jw-pkg.py --log-level info secrets list-compilation-output --help +usage: jw-pkg.py secrets list-compilation-output [-h] [--all] [packages ...] + +List package compilation output files + +positional arguments: + packages Package names (default: None) + +options: + -h, --help show this help message and exit + --all Show all output targets, including non-existent files (default: + False) +============= Running: jw-pkg.py --log-level info secrets list-secrets --help +usage: jw-pkg.py secrets list-secrets [-h] [--all] [packages ...] + +List package secret files + +positional arguments: + packages Package names (default: None) + +options: + -h, --help show this help message and exit + --all Show all secret paths, including non-existent files (default: + False) +============= Running: jw-pkg.py --log-level info secrets list-templates --help +usage: jw-pkg.py secrets list-templates [-h] [packages ...] + +List package template files + +positional arguments: + packages Package names (default: None) + +options: + -h, --help show this help message and exit +============= Running: jw-pkg.py --log-level info secrets rm-compilation-output --help +usage: jw-pkg.py secrets rm-compilation-output [-h] [packages ...] + +Remove package compilation output files + +positional arguments: + packages Package names (default: None) + options: -h, --help show this help message and exit From 57180e3596208e72b78c5490f2d03b406bce11a7 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 25 Jun 2026 15:11:22 +0200 Subject: [PATCH 3/4] jw.pkg.lib.Uri: Add unit test The Uri class provides URL parsing and manipulation utilities used throughout jw-pkg. Add a unit test covering URL parsing, credential handling, path manipulation, and safe string formatting. Signed-off-by: Jan Lindemann --- test/Makefile | 2 + test/unit/Makefile | 4 ++ test/unit/python/Makefile | 4 ++ test/unit/python/lib/Makefile | 4 ++ test/unit/python/lib/Uri/Makefile | 7 +++ test/unit/python/lib/Uri/test.py | 88 +++++++++++++++++++++++++++++++ 6 files changed, 109 insertions(+) create mode 100644 test/unit/Makefile create mode 100644 test/unit/python/Makefile create mode 100644 test/unit/python/lib/Makefile create mode 100644 test/unit/python/lib/Uri/Makefile create mode 100644 test/unit/python/lib/Uri/test.py diff --git a/test/Makefile b/test/Makefile index a9018b59..959f094f 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,4 +1,6 @@ TOPDIR = .. +ORDERED_SUBDIRS = unit integration + include $(TOPDIR)/make/proj.mk include $(JWBDIR)/make/dirs.mk diff --git a/test/unit/Makefile b/test/unit/Makefile new file mode 100644 index 00000000..b1ec41b6 --- /dev/null +++ b/test/unit/Makefile @@ -0,0 +1,4 @@ +TOPDIR = ../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/dirs.mk diff --git a/test/unit/python/Makefile b/test/unit/python/Makefile new file mode 100644 index 00000000..62d684a7 --- /dev/null +++ b/test/unit/python/Makefile @@ -0,0 +1,4 @@ +TOPDIR = ../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/dirs.mk diff --git a/test/unit/python/lib/Makefile b/test/unit/python/lib/Makefile new file mode 100644 index 00000000..5e331b8e --- /dev/null +++ b/test/unit/python/lib/Makefile @@ -0,0 +1,4 @@ +TOPDIR = ../../../.. + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/dirs.mk diff --git a/test/unit/python/lib/Uri/Makefile b/test/unit/python/lib/Uri/Makefile new file mode 100644 index 00000000..a2efda8e --- /dev/null +++ b/test/unit/python/lib/Uri/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../../../../../ + +include $(TOPDIR)/make/proj.mk +include $(JWBDIR)/make/py-run.mk + +all: +test: run diff --git a/test/unit/python/lib/Uri/test.py b/test/unit/python/lib/Uri/test.py new file mode 100644 index 00000000..109e4b24 --- /dev/null +++ b/test/unit/python/lib/Uri/test.py @@ -0,0 +1,88 @@ +import sys, os + +# Mirror the source tree structure so imports resolve correctly +src_root = os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'src', 'python') +sys.path.insert(0, src_root) + +from jw.pkg.lib.Uri import Uri + +# Basic URL parsing +u = Uri('ssh://john.doe@host:22/path/to/repo') +assert u.protocol == 'ssh' +assert u.hostname == 'host' +assert u.port == 22 +assert u.path == '/path/to/repo' + +# Username and password +u = Uri('https://user:pass@example.com/repo') +assert u.username == 'user' +assert u.password == 'pass' + +# Credentials in output +u = Uri('https://user:secret@example.com/repo') +assert 'secret' in u.full # full shows password +assert '' in u.safe_full_with_username # safe version hides password + +# Scheme defaults to file +u = Uri('/local/path') +assert u.scheme == 'file://' +assert u.protocol == 'file' + +# Pimp returns existing Uri unchanged +u1 = Uri('ssh://host/path') +u2 = Uri.pimp(u1) +assert u2 is u1 + +# New with path (add) +u = Uri('ssh://host/path') +u2 = u.new_add_path('subdir') +assert u2.to_string == 'ssh://host/path/subdir' + +# New with path (replace) - to_string is updated +u3 = u.new_replace_path('/other') +assert u3.to_string == 'ssh://host/other' + +# Authority includes credentials +u = Uri('ssh://user:pass@host:22/path') +assert u.authority == 'userpass@host:22' + +# Origin excludes credentials +assert u.origin == 'host:22' + +# ID and safe version hide password +assert u.id == 'ssh://user:@host:22' +assert '' in u.safe_full_with_username + +# Full includes path and credentials +assert u.full == 'ssh://userpass@host:22/path' + +# Simple URLs +u = Uri('https://example.com/repo.git') +assert u.hostname == 'example.com' +assert u.path == '/repo.git' +assert u.basename == 'repo.git' + +# Username setter +u = Uri('ssh://host/path') +u.set_username('newuser') +assert u.username == 'newuser' + +# Password setter +u = Uri('ssh://host/path') +u.set_password('newpass') +assert u.password == 'newpass' + +# No port +u = Uri('ssh://host/path') +assert u.port is None +assert u.port_str is None + +# to_string returns original +u = Uri('ssh://user:pass@host:22/path') +assert u.to_string == 'ssh://user:pass@host:22/path' + +# String representation hides credentials +u = Uri('https://user:secret@example.com/repo.git') +assert '' in str(u) + +print('All Uri tests passed') From 45f8e8cc89e6a095a2db8260625c596c46268798 Mon Sep 17 00:00:00 2001 From: Jan Lindemann Date: Thu, 25 Jun 2026 18:09:56 +0200 Subject: [PATCH 4/4] test + python-tools.sh: Fix toplevel "make all" fallout After a pipeline change, CI now runs "make all" in a repo's root, which uncovers two problems: 1. The help integration test only succeeded as long CI didn't run "make all" before "make test". That way, the checked out repository lacked the generated __init__.py files needed for some modular subcommands to be fully loaded, and hence, the test should have failed. The entire machinery only worked because the subcommands in question are not not essential to building jw-pkg itself: "secrets" and "posix". So, this commit adapts the help integration test to the new reality. 2. Regarding python-tools.sh: Commit 55060486 satisfies yapf in some places of the source code, but in others not anylonger. So patch python-tools.sh's newline handling again. While not thematically similar, both fixes get baked into one commit to satisfy the requirement that every single commit needs to pass "make clean all check test" individually. Signed-off-by: Jan Lindemann --- scripts/python-tools.sh | 2 +- .../integration/jw-pkg/help/test-expected.txt | 131 +++++++++++++++++- 2 files changed, 130 insertions(+), 3 deletions(-) diff --git a/scripts/python-tools.sh b/scripts/python-tools.sh index aa83d8f5..b9724178 100644 --- a/scripts/python-tools.sh +++ b/scripts/python-tools.sh @@ -81,12 +81,12 @@ cmd_create_init() echo "import `module_path $f` as $f" __add_seen $f done - echo fi if [ ${#seen[@]} -eq 0 ]; then echo "__all__ = []" else + echo echo "__all__ = [" for dst_type in ${!seen[@]}; do echo " \"$dst_type\"," diff --git a/test/integration/jw-pkg/help/test-expected.txt b/test/integration/jw-pkg/help/test-expected.txt index dec09556..4ebbee24 100644 --- a/test/integration/jw-pkg/help/test-expected.txt +++ b/test/integration/jw-pkg/help/test-expected.txt @@ -163,12 +163,55 @@ options: %{codename}, %{gnu-triplet}, %{os-cascade}, %{os-release}, %{pkg-ext} (default: %{cascade}) ============= Running: jw-pkg.py --log-level info posix --help -usage: jw-pkg.py posix [-h] +usage: jw-pkg.py posix [-h] ... Perform various operations on a distro through its POSIX utility interface options: -h, --help show this help message and exit + +Available subcommands of posix: + + copy Copy files + tar Handle tar archives +============= Running: jw-pkg.py --log-level info posix copy --help +usage: jw-pkg.py posix copy [-h] [-o OWNER] [-g GROUP] [-m MODE] [-F] src dst + +Copy files + +positional arguments: + src Source file URI + dst Destination file URI + +options: + -h, --help show this help message and exit + -o, --owner OWNER Destination file owner (default: None) + -g, --group GROUP Destination file group (default: None) + -m, --mode MODE Destination file mode (default: None) + -F, --fixed-strings Don't expand macros in and (default: False) +============= Running: jw-pkg.py --log-level info posix tar --help +usage: jw-pkg.py posix tar [-h] ... + +Handle tar archives + +options: + -h, --help show this help message and exit + +Available subcommands of tar: + + x Extract a tar archive +============= Running: jw-pkg.py --log-level info posix tar x --help +usage: jw-pkg.py posix tar x [-h] -f ARCHIVE_PATH dst + +Extract a tar archive + +positional arguments: + dst Destination root URI + +options: + -h, --help show this help message and exit + -f, --archive-path ARCHIVE_PATH + Archive path ============= Running: jw-pkg.py --log-level info projects --help usage: jw-pkg.py projects [-h] ... @@ -689,9 +732,93 @@ positional arguments: options: -h, --help show this help message and exit ============= Running: jw-pkg.py --log-level info secrets --help -usage: jw-pkg.py secrets [-h] +usage: jw-pkg.py secrets [-h] ... Manage package secrets +options: + -h, --help show this help message and exit + +Available subcommands of secrets: + + compile-templates Compile package template files + install Install secrets from various sources as static secrets + onto the target + list-compilation-output + List package compilation output files + list-secrets List package secret files + list-templates List package template files + rm-compilation-output + Remove package compilation output files +============= Running: jw-pkg.py --log-level info secrets compile-templates --help +usage: jw-pkg.py secrets compile-templates [-h] [--owner OWNER] + [--group GROUP] [--mode MODE] + [packages ...] + +Compile package template files + +positional arguments: + packages Package names (default: None) + +options: + -h, --help show this help message and exit + --owner, -o OWNER Default output file owner (default: None) + --group, -g GROUP Default output file group (default: None) + --mode, -m MODE Default output file mode (default: None) +============= Running: jw-pkg.py --log-level info secrets install --help +usage: jw-pkg.py secrets install [-h] [--only-missing] src [packages ...] + +Install secrets from various sources as static secrets onto the target + +positional arguments: + src URI of secret source + packages Package names (default: None) + +options: + -h, --help show this help message and exit + --only-missing Install only secrets not already on the target (default: + False) +============= Running: jw-pkg.py --log-level info secrets list-compilation-output --help +usage: jw-pkg.py secrets list-compilation-output [-h] [--all] [packages ...] + +List package compilation output files + +positional arguments: + packages Package names (default: None) + +options: + -h, --help show this help message and exit + --all Show all output targets, including non-existent files (default: + False) +============= Running: jw-pkg.py --log-level info secrets list-secrets --help +usage: jw-pkg.py secrets list-secrets [-h] [--all] [packages ...] + +List package secret files + +positional arguments: + packages Package names (default: None) + +options: + -h, --help show this help message and exit + --all Show all secret paths, including non-existent files (default: + False) +============= Running: jw-pkg.py --log-level info secrets list-templates --help +usage: jw-pkg.py secrets list-templates [-h] [packages ...] + +List package template files + +positional arguments: + packages Package names (default: None) + +options: + -h, --help show this help message and exit +============= Running: jw-pkg.py --log-level info secrets rm-compilation-output --help +usage: jw-pkg.py secrets rm-compilation-output [-h] [packages ...] + +Remove package compilation output files + +positional arguments: + packages Package names (default: None) + options: -h, --help show this help message and exit