From 55ead78e80566d4d26d7d0862beef102f1384ad7 Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Wed, 7 Oct 2026 14:47:12 +0200 Subject: [PATCH 01/11] Run manifest loading and generate layer in generate_app_layer.py --- tools/app_layer.py | 5 +++++ tools/generate_app_layer.py | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tools/app_layer.py b/tools/app_layer.py index e69de29..c39aea9 100644 --- a/tools/app_layer.py +++ b/tools/app_layer.py @@ -0,0 +1,5 @@ +from pathlib import Path +from app_manifest import Manifest + +def generate_layers(manifests: list[Manifest], output: Path) -> None: + """Generate bitbake recipes based on application manifests""" diff --git a/tools/generate_app_layer.py b/tools/generate_app_layer.py index 4e995bf..93bb4ca 100644 --- a/tools/generate_app_layer.py +++ b/tools/generate_app_layer.py @@ -3,6 +3,13 @@ import argparse from pathlib import Path +if __package__: + from . import app_layer + from .app_manifest import load_manifests +else: + import app_layer + from app_manifest import load_manifests + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) @@ -18,11 +25,13 @@ def main() -> None: required=True, help="Destination for the generated BitBake layer", ) - _ = parser.parse_args() + args = parser.parse_args() - # TODO: Read and validate manifests with app_manifest before writing output - # TODO: Generate the layer from the validated data with app_layer - parser.exit(1, "error: application layer generation is not implemented yet\n") + try: + manifests = load_manifests(args.apps_dir) + app_layer.generate_layer(manifests, args.output) + except (ValueError, OSError) as error: + parser.exit(1, f"error: {error}\n") if __name__ == "__main__": From 5b0d04ec86a6533e66074f7c66c927ef7d6303b5 Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Wed, 7 Oct 2026 15:21:53 +0200 Subject: [PATCH 02/11] Start on recipe generation --- tools/app_layer.py | 25 +++++++++++++++++++++++-- tools/generate_app_layer.py | 2 +- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/tools/app_layer.py b/tools/app_layer.py index c39aea9..13042e0 100644 --- a/tools/app_layer.py +++ b/tools/app_layer.py @@ -1,5 +1,26 @@ +"""Orchestrate application recipes and generated layer output""" + from pathlib import Path -from app_manifest import Manifest + +if __package__: + from .app_manifest import Manifest +else: + from app_manifest import Manifest + def generate_layers(manifests: list[Manifest], output: Path) -> None: - """Generate bitbake recipes based on application manifests""" + """Build the generated layer from validated manifests, rendering everything before writes""" + enabled = sorted( + (manifest for manifest in manifests if manifest.enabled), + key=lambda manifest: manifest.id, + ) + files: list[tuple[Path, str]] = [] + + for manifest in enabled: + files.extend(render_application_files(manifest)) + for service in sorted(manifest.services, key=lambda service: service.id): + files.extend(render_service_files(manifest, service)) + + files.append((Path("conf/layer.conf"), render_layer_config(enabled))) + validate_layer_output(files, output, manifests) + replace_layer(output, files) diff --git a/tools/generate_app_layer.py b/tools/generate_app_layer.py index 93bb4ca..5513ad2 100644 --- a/tools/generate_app_layer.py +++ b/tools/generate_app_layer.py @@ -29,7 +29,7 @@ def main() -> None: try: manifests = load_manifests(args.apps_dir) - app_layer.generate_layer(manifests, args.output) + app_layer.generate_layers(manifests, args.output) except (ValueError, OSError) as error: parser.exit(1, f"error: {error}\n") From 1476c416e3dba43b013d602ca2c3d6a05ee846ae Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Wed, 7 Oct 2026 15:35:52 +0200 Subject: [PATCH 03/11] Implement replace_layer in app_layer.py --- tools/app_layer.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tools/app_layer.py b/tools/app_layer.py index 13042e0..7b6b7c9 100644 --- a/tools/app_layer.py +++ b/tools/app_layer.py @@ -1,6 +1,8 @@ """Orchestrate application recipes and generated layer output""" from pathlib import Path +import shutil +import tempfile if __package__: from .app_manifest import Manifest @@ -24,3 +26,45 @@ def generate_layers(manifests: list[Manifest], output: Path) -> None: files.append((Path("conf/layer.conf"), render_layer_config(enabled))) validate_layer_output(files, output, manifests) replace_layer(output, files) + + +def replace_layer(output: Path, files: list[tuple[Path, str]]) -> None: + """Replace a validated generated-only destination with UTF-8 files + + Stage beside the destination, then swap directories and remove stale output + Restore the previous layer if installation fails, retaining its backup if + restoration also fails + """ + if output.is_symlink() or (output.exists() and not output.is_dir()): + raise ValueError(f"{output}: expected a generated layer directory") + + output.parent.mkdir(parents=True, exist_ok=True) + workspace = Path(tempfile.mkdtemp(prefix=f".{output.name}-", dir=output.parent)) + staged = workspace / "new" + backup = workspace / "old" + installed = False + + try: + staged.mkdir() + for relative, text in files: + if relative.is_absolute() or ".." in relative.parts: + raise ValueError(f"{relative}: expected layer-relative path") + + destination = staged / relative + destination.parent.mkdir(parents=True, exist_ok=True) + + with destination.open("x", encoding="utf-8", newline="\n") as stream: + _ = stream.write(text) + + if output.exists(): + _ = output.rename(backup) + try: + _ = staged.rename(output) + except OSError: + if backup.exists(): + _ = backup.rename(output) + raise + installed = True + finally: + if installed or not backup.exists(): + shutil.rmtree(workspace) From 8ca1061f7df039d0f7938af3e40ffbd7e9d2be29 Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Wed, 7 Oct 2026 15:36:28 +0200 Subject: [PATCH 04/11] Add tests for app layer generation --- tools/tests/test_app_layer.py | 93 +++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 tools/tests/test_app_layer.py diff --git a/tools/tests/test_app_layer.py b/tools/tests/test_app_layer.py new file mode 100644 index 0000000..fc41454 --- /dev/null +++ b/tools/tests/test_app_layer.py @@ -0,0 +1,93 @@ +"""Generated layer replacement using isolated filesystem data""" + +from pathlib import Path + +import pytest + +from tools.app_layer import replace_layer + + +@pytest.fixture +def files() -> list[tuple[Path, str]]: + return [ + (Path("conf/layer.conf"), '# Generated layer\nBBPATH .= ":${LAYERDIR}"\n'), + (Path("recipes-apps/example/example.bb"), 'SUMMARY = "Example æøå"\n'), + ] + + +def test_replace_layer(tmp_path: Path, files: list[tuple[Path, str]]) -> None: + output = tmp_path / "generated/meta-apps" + replace_layer(output, files) + stale = output / "recipes-apps/removed/removed.bb" + stale.parent.mkdir() + _ = stale.write_text("obsolete", encoding="utf-8") + unrelated = output.parent / "keep.txt" + _ = unrelated.write_text("keep", encoding="utf-8") + + for _ in range(2): + replace_layer(output, files) + assert {path.relative_to(output) for path in output.rglob("*") if path.is_file()} == { + path for path, _ in files + } + for path, content in files: + assert (output / path).read_bytes() == content.encode("utf-8") + assert unrelated.read_text() == "keep" + assert set(output.parent.iterdir()) == {output, unrelated} + + replace_layer(output, []) + assert output.is_dir() + assert list(output.iterdir()) == [] + + +@pytest.mark.parametrize("failure", ["write", "install", "restore"]) +def test_failure_preserves_previous_layer( + tmp_path: Path, files: list[tuple[Path, str]], + monkeypatch: pytest.MonkeyPatch, failure: str, +) -> None: + output = tmp_path / "meta-apps" + output.mkdir() + _ = (output / "previous.bb").write_text("previous", encoding="utf-8") + original_rename = Path.rename + + def fail_install(source: Path, target: Path) -> Path: + if target == output and (source.name == "new" or failure == "restore"): + raise OSError("rename failed") + return original_rename(source, target) + + if failure == "write": + files = files + [(Path("bad.txt"), "\ud800")] + expected = UnicodeError + else: + monkeypatch.setattr(Path, "rename", fail_install) + expected = OSError + + with pytest.raises(expected): + replace_layer(output, files) + + if failure == "restore": + backups = list(tmp_path.glob(".meta-apps-*/old/previous.bb")) + assert len(backups) == 1 + assert backups[0].read_text() == "previous" + else: + assert (output / "previous.bb").read_text() == "previous" + assert set(output.iterdir()) == {output / "previous.bb"} + assert list(tmp_path.iterdir()) == [output] + + +@pytest.mark.parametrize("invalid", [Path("../escape"), Path("/escape")]) +def test_rejects_paths_outside_layer(tmp_path: Path, invalid: Path) -> None: + with pytest.raises(ValueError, match="layer-relative path"): + replace_layer(tmp_path / "meta-apps", [(invalid, "bad")]) + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize("kind", ["file", "symlink"]) +def test_rejects_non_directory_output(tmp_path: Path, kind: str) -> None: + output = tmp_path / "meta-apps" + if kind == "file": + _ = output.write_text("keep", encoding="utf-8") + else: + output.symlink_to(tmp_path, target_is_directory=True) + with pytest.raises(ValueError, match="generated layer directory"): + replace_layer(output, []) + assert output.is_symlink() if kind == "symlink" else output.read_text() == "keep" From e3b7b68e450b104a26b19308e4419854091d56f1 Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Wed, 7 Oct 2026 16:40:13 +0200 Subject: [PATCH 05/11] Implement app recipe generation --- tools/app_layer.py | 88 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/tools/app_layer.py b/tools/app_layer.py index 7b6b7c9..92ac63e 100644 --- a/tools/app_layer.py +++ b/tools/app_layer.py @@ -1,6 +1,9 @@ """Orchestrate application recipes and generated layer output""" +import hashlib +import json from pathlib import Path +import re import shutil import tempfile @@ -10,6 +13,91 @@ from app_manifest import Manifest +def render_application_files(manifest: Manifest) -> list[tuple[Path, str]]: + """Render one application recipe from validated bundle paths without writing files""" + bundle = manifest.manifest_path.parent + source = manifest.app.source_dir.relative_to(bundle) + license_file = manifest.license_file.relative_to(bundle) + for field, path in (("bundle", bundle), ("app.source_dir", source), + ("license_file", license_file)): + if not re.fullmatch(r"[A-Za-z0-9_./+-]+", path.as_posix()): + raise ValueError( + f"{manifest.manifest_path}: {field}: cannot serialize path in BitBake " + "(use ASCII letters, digits, underscores, dots, slashes, + or -)" + ) + if any(char in manifest.license for char in '\\"$'): + raise ValueError( + f"{manifest.manifest_path}: license: cannot serialize quotes, backslashes " + "or dollar signs in a BitBake literal" + ) + + package = f"mosaic-app-{manifest.id}" + unpacked = "${UNPACKDIR}/" + bundle.name + checksum = hashlib.md5(manifest.license_file.read_bytes()).hexdigest() + build_dependencies = sorted({ + "qtbase", "qtdeclarative", "qtdeclarative-native", + *manifest.app.build_dependencies, + }) + runtime_dependencies = sorted({ + "qtbase-plugins", "qtdeclarative-qmlplugins", "qtwayland-plugins", + *manifest.app.runtime_dependencies, + *(f"mosaic-service-{manifest.id}-{service.id}" for service in manifest.services), + }) + metadata = { + "id": manifest.id, + "name": {"en": manifest.name}, + "version": manifest.version, + "applications": [{ + "id": manifest.id, + "code": manifest.app.executable, + "runtime": "native", + "runtimeParameters": {"arguments": manifest.app.arguments}, + "supportsApplicationInterface": False, + }], + } + # Escape dollars before embedding JSON so BitBake cannot expand launch data + serialized = json.dumps(metadata, ensure_ascii=True).replace("$", r"\u0024") + recipe = f'''SUMMARY = "Mosaic application {manifest.id}" +LICENSE = "{manifest.license}" +LIC_FILES_CHKSUM = "file://{unpacked}/{license_file.as_posix()};md5={checksum}" + +FILESEXTRAPATHS:prepend := "{bundle.parent.as_posix()}:" +SRC_URI = "file://{bundle.name};unpack=0" +S = "{unpacked}/{source.as_posix()}" + +inherit qt6-cmake + +DEPENDS += "{' '.join(build_dependencies)}" +RDEPENDS:${{PN}} += "{' '.join(runtime_dependencies)}" +EXTRA_OECMAKE += "-DMOSAIC_APP_EXECUTABLE={manifest.app.executable}" + +python mosaic_install_app_metadata() {{ + import json + import os + + metadata = json.loads({serialized!r}) + application = metadata["applications"][0] + application["code"] = os.path.join(d.getVar("bindir"), application["code"]) + executable = d.getVar("D") + application["code"] + if not os.path.isfile(executable) or not os.access(executable, os.X_OK): + bb.fatal("Expected executable at %s: set the CMake target OUTPUT_NAME " + "from MOSAIC_APP_EXECUTABLE and install it into CMAKE_INSTALL_BINDIR" + % executable) + + directory = d.getVar("D") + d.getVar("datadir") + "/mosaic/apps/" + metadata["id"] + os.makedirs(directory, exist_ok=True) + with open(os.path.join(directory, "info.yaml"), "w", encoding="utf-8") as stream: + stream.write("%YAML 1.2\\n---\\nformatVersion: 1\\nformatType: am-package\\n---\\n") + json.dump(metadata, stream, ensure_ascii=True, indent=2) + stream.write("\\n") +}} + +do_install[postfuncs] += "mosaic_install_app_metadata" +FILES:${{PN}} += "${{datadir}}/mosaic/apps/{manifest.id}" +''' + return [(Path(f"recipes-apps/{package}/{package}_{manifest.version}.bb"), recipe)] + + def generate_layers(manifests: list[Manifest], output: Path) -> None: """Build the generated layer from validated manifests, rendering everything before writes""" enabled = sorted( From e4fb73cadeed6fd3dae09e66337c993bf1aa68c7 Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Wed, 7 Oct 2026 16:42:08 +0200 Subject: [PATCH 06/11] Write some tests for app recipe generation --- tools/tests/test_application_recipe.py | 243 +++++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 tools/tests/test_application_recipe.py diff --git a/tools/tests/test_application_recipe.py b/tools/tests/test_application_recipe.py new file mode 100644 index 0000000..4f10b0b --- /dev/null +++ b/tools/tests/test_application_recipe.py @@ -0,0 +1,243 @@ +"""Application recipe rendering and staged CMake/QAM installation""" + +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import textwrap +from types import SimpleNamespace + +import pytest + +from tools.app_layer import render_application_files +from tools.app_manifest import Manifest, load_manifest + + +@pytest.fixture +def manifest(tmp_path: Path) -> Manifest: + bundle = tmp_path / "bundle" + source = bundle / "src/gui" + source.mkdir(parents=True) + (source / "CMakeLists.txt").write_text(textwrap.dedent('''\ + cmake_minimum_required(VERSION 3.16) + project(example LANGUAGES CXX) + include(GNUInstallDirs) + add_executable(internal_target main.cpp) + set_target_properties(internal_target PROPERTIES OUTPUT_NAME "${MOSAIC_APP_EXECUTABLE}") + install(TARGETS internal_target RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") + install(FILES resource.txt DESTINATION "${CMAKE_INSTALL_DATADIR}/mosaic/apps/example") + '''), encoding="utf-8") + (source / "main.cpp").write_text("int main() { return 0; }\n", encoding="utf-8") + (source / "resource.txt").write_text("Application resource\n", encoding="utf-8") + (bundle / "LICENSE").write_text("Synthetic test license\n", encoding="utf-8") + path = bundle / "mosaic-app.json" + path.write_text(json.dumps({ + "schema_version": 1, "id": "example", "name": "Example æøå", + "version": "1.2.3", "enabled": True, "include_in_prod": False, + "license": "MIT", "license_file": "LICENSE", + "app": {"source_dir": "src/gui", "executable": "visible-app"}, + }), encoding="utf-8") + return load_manifest(path) + + +def install_metadata(recipe: str, stage: Path, bindir: str = "/usr/bin", + datadir: str = "/usr/share") -> Path: + function = re.search(r"python mosaic_install_app_metadata\(\) \{\n(.*?)\n\}", recipe, re.S) + assert function is not None + body = textwrap.dedent(function[1]) + # No launch data may reach BitBake's expansion mechanism + assert "${" not in body + + def fatal(message: str) -> None: + raise RuntimeError(message) + + values = {"D": str(stage), "bindir": bindir, "datadir": datadir} + exec(compile(body, "generated recipe postfunc", "exec"), { + "d": SimpleNamespace(getVar=values.__getitem__), + "bb": SimpleNamespace(fatal=fatal), + }) + return stage / datadir.lstrip("/") / "mosaic/apps/example/info.yaml" + + +def read_metadata(path: Path) -> dict: + content = path.read_text(encoding="utf-8") + header = "%YAML 1.2\n---\nformatVersion: 1\nformatType: am-package\n---\n" + assert content.startswith(header) + return json.loads(content.removeprefix(header)) + + +def test_recipe_and_no_writes(manifest: Manifest) -> None: + bundle = manifest.manifest_path.parent + before = {path: path.read_bytes() for path in bundle.rglob("*") if path.is_file()} + files = render_application_files(manifest) + assert files == render_application_files(manifest) + assert len(files) == 1 + path, recipe = files[0] + assert path == Path("recipes-apps/mosaic-app-example/mosaic-app-example_1.2.3.bb") + checksum = hashlib.md5(before[manifest.license_file]).hexdigest() + assert f'LIC_FILES_CHKSUM = "file://${{UNPACKDIR}}/bundle/LICENSE;md5={checksum}"' in recipe + assert 'LICENSE = "MIT"' in recipe + assert f'FILESEXTRAPATHS:prepend := "{bundle.parent}:"' in recipe + assert 'SRC_URI = "file://bundle;unpack=0"' in recipe + assert 'S = "${UNPACKDIR}/bundle/src/gui"' in recipe + assert 'inherit qt6-cmake' in recipe + assert 'DEPENDS += "qtbase qtdeclarative qtdeclarative-native"' in recipe + assert 'RDEPENDS:${PN} += "qtbase-plugins qtdeclarative-qmlplugins qtwayland-plugins"' in recipe + assert 'EXTRA_OECMAKE += "-DMOSAIC_APP_EXECUTABLE=visible-app"' in recipe + assert 'do_install[postfuncs] += "mosaic_install_app_metadata"' in recipe + assert 'FILES:${PN} += "${datadir}/mosaic/apps/example"' in recipe + assert not re.search(r"^do_(configure|compile|install).*\(\)", recipe, re.M) + assert before == {path: path.read_bytes() for path in bundle.rglob("*") if path.is_file()} + assert list(bundle.parent.iterdir()) == [bundle] + + +def test_dependencies_and_selection_boundary(manifest: Manifest) -> None: + path = manifest.manifest_path + data = json.loads(path.read_text()) + data["app"]["build_dependencies"] = ["zlib", "qtbase", "zlib"] + data["app"]["runtime_dependencies"] = ["zlib", "qtwayland-plugins", "zlib"] + data["services"] = [] + for service_id in ("telemetry", "alarm"): + unit = path.parent / f"{service_id}.service" + unit.write_text("[Service]\nExecStart=/bin/true\n", encoding="utf-8") + data["services"].append({ + "id": service_id, "source_dir": "src/gui", "unit": unit.name, + "start": "boot", "build_dependencies": ["service-build-only"], + "runtime_dependencies": ["service-runtime-only"], + }) + data["app"]["runtime_dependencies"].append("mosaic-service-example-alarm") + path.write_text(json.dumps(data), encoding="utf-8") + files = render_application_files(load_manifest(path)) + recipe = files[0][1] + assert 'DEPENDS += "qtbase qtdeclarative qtdeclarative-native zlib"' in recipe + assert 'RDEPENDS:${PN} += "mosaic-service-example-alarm mosaic-service-example-telemetry qtbase-plugins qtdeclarative-qmlplugins qtwayland-plugins zlib"' in recipe + assert "service-build-only" not in recipe + assert "service-runtime-only" not in recipe + data.update(enabled=False, include_in_prod=True, profiles={"prod": {"settings": {"opaque": 1}}}) + data["services"].reverse() + data["app"]["build_dependencies"].reverse() + path.write_text(json.dumps(data), encoding="utf-8") + assert render_application_files(load_manifest(path)) == files + + +@pytest.mark.parametrize("arguments", [[], [ + "", " two words ", "æøå 日本語 😀", "'quoted' \"text\"", "\\path\\", + "${bindir}", "${@bb.fatal('must not expand')}", "$(touch nope)", + "line\nnext\r\t\x01\x85\u2028", r"\u0024{literal}", "---\n}\npython injected() {", +]]) +def test_metadata_round_trip(manifest: Manifest, tmp_path: Path, arguments: list[str]) -> None: + path = manifest.manifest_path + data = json.loads(path.read_text()) + data["name"] = ' Æøå "Display" \\ ${PN} ${@bb.fatal("no")} ' + data["app"]["arguments"] = arguments + path.write_text(json.dumps(data), encoding="utf-8") + manifest = load_manifest(path) + recipe = render_application_files(manifest)[0][1] + stage = tmp_path / "image" + executable = stage / "opt/bin/visible-app" + executable.parent.mkdir(parents=True) + executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + executable.chmod(0o755) + metadata_path = install_metadata(recipe, stage, "/opt/bin", "/opt/share") + assert read_metadata(metadata_path) == { + "id": "example", "name": {"en": data["name"]}, "version": "1.2.3", + "applications": [{ + "id": "example", "code": "/opt/bin/visible-app", "runtime": "native", + "runtimeParameters": {"arguments": arguments}, + "supportsApplicationInterface": False, + }], + } + original = metadata_path.read_bytes() + install_metadata(recipe, stage, "/opt/bin", "/opt/share") + assert metadata_path.read_bytes() == original + assert not executable.is_symlink() + + +@pytest.mark.parametrize("kind", ["missing", "not-executable", "directory", "wrong-name"]) +def test_install_requires_executable(manifest: Manifest, tmp_path: Path, kind: str) -> None: + stage = tmp_path / "image" + executable = stage / "usr/bin/visible-app" + executable.parent.mkdir(parents=True) + if kind == "not-executable": + executable.touch(mode=0o644) + elif kind == "directory": + executable.mkdir() + elif kind == "wrong-name": + executable.with_name("internal_target").touch(mode=0o755) + recipe = render_application_files(manifest)[0][1] + with pytest.raises(RuntimeError, match="OUTPUT_NAME.*MOSAIC_APP_EXECUTABLE.*CMAKE_INSTALL_BINDIR"): + install_metadata(recipe, stage) + assert not (stage / "usr/share/mosaic").exists() + + +@pytest.mark.parametrize("field", ["bundle", "app.source_dir", "license_file"]) +@pytest.mark.parametrize("unsafe", ["space here", '"quote', "${PN}", "back\\slash", ";parameter"]) +def test_unsafe_recipe_literals(manifest: Manifest, field: str, unsafe: str) -> None: + path = manifest.manifest_path + data = json.loads(path.read_text()) + if field == "bundle": + renamed = path.parent.with_name(unsafe) + path.parent.rename(renamed) + path = renamed / path.name + elif field == "app.source_dir": + manifest.app.source_dir.rename(path.parent / unsafe) + data["app"]["source_dir"] = unsafe + elif field == "license_file": + manifest.license_file.rename(path.parent / unsafe) + data["license_file"] = unsafe + path.write_text(json.dumps(data), encoding="utf-8") + validated = load_manifest(path) + with pytest.raises(ValueError, match=re.escape(field) + ": cannot serialize"): + render_application_files(validated) + + +@pytest.mark.parametrize("license", ['MIT "quoted"', "${LICENSE}", "MIT\\"]) +def test_unsafe_license(manifest: Manifest, license: str) -> None: + path = manifest.manifest_path + data = json.loads(path.read_text()) + data["license"] = license + path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(ValueError, match="license: cannot serialize"): + render_application_files(load_manifest(path)) + + +def test_bundle_root_source_and_nested_license(manifest: Manifest) -> None: + path = manifest.manifest_path + data = json.loads(path.read_text()) + (manifest.app.source_dir / "CMakeLists.txt").rename(path.parent / "CMakeLists.txt") + licenses = path.parent / "licenses" + licenses.mkdir() + manifest.license_file.rename(licenses / "COPYING") + (licenses / "COPYING").write_bytes(b"Different license contents\r\n") + data.update(license_file="licenses/COPYING", license="MIT & BSD-3-Clause") + data["app"]["source_dir"] = "." + path.write_text(json.dumps(data), encoding="utf-8") + recipe = render_application_files(load_manifest(path))[0][1] + checksum = hashlib.md5(b"Different license contents\r\n").hexdigest() + assert 'S = "${UNPACKDIR}/bundle/."' in recipe + assert f'file://${{UNPACKDIR}}/bundle/licenses/COPYING;md5={checksum}' in recipe + assert 'LICENSE = "MIT & BSD-3-Clause"' in recipe + + +@pytest.mark.skipif(shutil.which("cmake") is None, reason="CMake is not installed") +def test_native_cmake_install(manifest: Manifest, tmp_path: Path) -> None: + build = tmp_path / "cmake-build" + stage = tmp_path / "image" + subprocess.run([ + "cmake", "-S", str(manifest.app.source_dir), "-B", str(build), + "-DCMAKE_INSTALL_PREFIX=/usr", "-DCMAKE_INSTALL_BINDIR=bin", + "-DCMAKE_INSTALL_DATADIR=share", f"-DMOSAIC_APP_EXECUTABLE={manifest.app.executable}", + ], check=True, capture_output=True, text=True) + subprocess.run(["cmake", "--build", str(build)], check=True, capture_output=True, text=True) + subprocess.run(["cmake", "--install", str(build)], env={**os.environ, "DESTDIR": str(stage)}, + check=True, capture_output=True, text=True) + executable = stage / "usr/bin/visible-app" + assert executable.is_file() and os.access(executable, os.X_OK) + assert not (stage / "usr/bin/internal_target").exists() + assert (stage / "usr/share/mosaic/apps/example/resource.txt").read_text() == "Application resource\n" + recipe = render_application_files(manifest)[0][1] + metadata = read_metadata(install_metadata(recipe, stage)) + assert metadata["applications"][0]["code"] == "/usr/bin/visible-app" From 4f11a2a43dcd6c96a5e80bb114d94e96d02e1032 Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Wed, 7 Oct 2026 16:46:50 +0200 Subject: [PATCH 07/11] Update docs for app generation --- docs/app-generation.rst | 15 ++++++++------- docs/applications.rst | 42 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/docs/app-generation.rst b/docs/app-generation.rst index e6f6248..18cc8b5 100644 --- a/docs/app-generation.rst +++ b/docs/app-generation.rst @@ -45,12 +45,13 @@ bundles are excluded. Enabled bundles with ``include_in_prod: true`` are include in both profiles, while the remaining enabled bundles are included only in dev. Selecting an application also includes its required service packages. -Each application recipe installs a JSON launcher entry at -``/usr/share/mosaic/apps/.json`` containing its existing ``id``, ``name``, -``executable``, and ``arguments``. -The shell reads installed entries, displays each application's name, and starts -its executable with the listed arguments when selected after unlock. The launcher -owns discovery and startup of installed applications. +Each application recipe generates Qt Application Manager (QAM) metadata during +installation at ``${datadir}/mosaic/apps//info.yaml`` (normally +``/usr/share/mosaic/apps//info.yaml``). The application package includes this +directory. See :doc:`applications` for the CMake contract and metadata fields. +Shell integration must configure QAM discovery of this directory and display +installed application names. Native QAM applications require Wayland support +in the distro and compositor integration in the shell. Tool files ---------- @@ -67,7 +68,7 @@ file handling. It consists of three files under ``tools/``: data without writing generated files. ``app_layer.py`` - Generates recipes, launcher entries, and layer configuration from the + Generates recipes and layer configuration from the validated data, and removes obsolete output. Verification diff --git a/docs/applications.rst b/docs/applications.rst index e27ff3d..985436d 100644 --- a/docs/applications.rst +++ b/docs/applications.rst @@ -67,6 +67,48 @@ and ``prod`` entries. Each entry contains a required ``settings`` object holding application-defined JSON values. The parser preserves these settings without interpreting them. +CMake build and installation +---------------------------- + +Application recipes inherit ``qt6-cmake`` and use its standard configure, +compile, and install tasks. The entire bundle is fetched locally, preserving its +directory structure. The recipe uses the declared ``license`` and an MD5 checksum +of ``license_file``. + +The recipe passes ``-DMOSAIC_APP_EXECUTABLE=`` to CMake. Apply this +variable to the executable target's ``OUTPUT_NAME`` and install the target into +``${CMAKE_INSTALL_BINDIR}``. Install required resources through CMake into +``${CMAKE_INSTALL_DATADIR}/mosaic/apps/`` so the generated package includes +them. For example, an application with ID ``example``, ``main.cpp``, and +``resources/`` can use: + +.. code-block:: cmake + + cmake_minimum_required(VERSION 3.16) + project(example LANGUAGES CXX) + include(GNUInstallDirs) + find_package(Qt6 REQUIRED COMPONENTS Quick) + + set(MOSAIC_APP_EXECUTABLE "example" CACHE STRING "Installed application name") + qt_add_executable(example_target main.cpp) + target_link_libraries(example_target PRIVATE Qt6::Quick) + set_target_properties(example_target PROPERTIES + OUTPUT_NAME "${MOSAIC_APP_EXECUTABLE}") + install(TARGETS example_target RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") + install(DIRECTORY resources/ + DESTINATION "${CMAKE_INSTALL_DATADIR}/mosaic/apps/example") + +After CMake installation, the recipe requires an executable file at +``${D}${bindir}/``. A missing or non-executable file fails the +build with guidance to check the naming and installation rules. + +The renderer adds ``qtbase``, ``qtdeclarative``, and ``qtdeclarative-native`` to +build dependencies. Runtime dependencies include ``qtbase-plugins``, +``qtdeclarative-qmlplugins``, and ``qtwayland-plugins``. Manifest declarations +extend these lists. Required service packages use the reserved names +``mosaic-service--``. Dependency lists are deduplicated and +sorted. Service-specific dependencies belong to the service recipes. + Example manifest ---------------- From 021a9ff48e5acc508baede8e0e546a8979f608c6 Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Thu, 8 Oct 2026 14:36:04 +0200 Subject: [PATCH 08/11] Add service recipe generation --- docs/app-generation.rst | 73 +++++-- docs/applications.rst | 205 +++++++++++++----- tools/app_layer.py | 86 ++++++-- tools/app_manifest.py | 81 ++++++- tools/tests/test_service_recipe.py | 332 +++++++++++++++++++++++++++++ 5 files changed, 688 insertions(+), 89 deletions(-) create mode 100644 tools/tests/test_service_recipe.py diff --git a/docs/app-generation.rst b/docs/app-generation.rst index 18cc8b5..d79c918 100644 --- a/docs/app-generation.rst +++ b/docs/app-generation.rst @@ -25,19 +25,29 @@ Recipe and image integration ---------------------------- The generator creates one recipe per application and one per bundled service. -Recipes use standard Yocto classes for Qt/CMake builds and systemd integration. -They reference the bundle's sources and license file. CMake and BitBake own compilation, -installation, and packaging. - -Applications and services explicitly declare their additional build and runtime -dependencies in the manifest. Build dependency recipe names map to ``DEPENDS``. -Runtime package names map to ``RDEPENDS``. The generator supplies the standard Qt -application dependencies rather than inferring dependencies from source code. - -Each application package depends on its required service packages. Service -recipes install the supplied systemd unit file or files and enable services -declared with ``start: boot``. Systemd runs services independently of application -selection and termination. +Application recipes inherit ``qt6-cmake``. Service recipes inherit ``cmake systemd`` +or ``cargo systemd`` according to the files in ``source_dir``. Each recipe fetches +the complete local bundle, sets ``S`` to its component's source directory, and +uses the bundle's version, license, and license-file checksum. The inherited +classes own compilation and installation, and BitBake owns packaging. + +Cargo lockfiles are parsed locally with ``tomllib``. The renderer adds sorted, +deduplicated crates.io URLs and named SHA-256 checksums to the recipe. Parsing and +rendering invoke neither Cargo nor the network. Yocto fetches the crates, and +Cargo retains its standard frozen build and executable installation tasks. + +Manifest build and runtime dependencies extend ``DEPENDS`` and ``RDEPENDS:${PN}``, +with duplicate names removed and lists sorted. Application build dependencies +include ``qtbase``, ``qtdeclarative``, and ``qtdeclarative-native``. Runtime +dependencies include ``qtbase-plugins``, ``qtdeclarative-qmlplugins``, and +``qtwayland-plugins``. Services receive no automatic Qt dependencies. + +Each application package depends on its required service packages, named +``mosaic-service--``. Service recipes append installation of +the declared units into ``${systemd_system_unitdir}``, preserving contents and +filenames with mode ``0644``. They include the units in the package and set +``SYSTEMD_SERVICE:${PN}`` and ``SYSTEMD_AUTO_ENABLE:${PN}`` to enable them. +Systemd runs services independently of application selection and termination. Layer configuration supplies ``MOSAIC_APPS_COMMON`` and ``MOSAIC_APPS_DEV_ONLY`` to the existing application packagegroup. Disabled @@ -48,7 +58,7 @@ Selecting an application also includes its required service packages. Each application recipe generates Qt Application Manager (QAM) metadata during installation at ``${datadir}/mosaic/apps//info.yaml`` (normally ``/usr/share/mosaic/apps//info.yaml``). The application package includes this -directory. See :doc:`applications` for the CMake contract and metadata fields. +directory. See :doc:`applications` for the application authoring contract. Shell integration must configure QAM discovery of this directory and display installed application names. Native QAM applications require Wayland support in the distro and compositor integration in the shell. @@ -71,12 +81,35 @@ file handling. It consists of three files under ``tools/``: Generates recipes and layer configuration from the validated data, and removes obsolete output. +Parser and renderer interfaces +------------------------------ + +``load_manifest(path)`` and ``load_manifests(apps_dir)`` return Pydantic models +with defaults filled in, paths resolved to absolute ``Path`` objects, and units +normalized to lists. Invalid input raises ``ManifestError`` with the manifest +path and affected field. All bundles are validated before selection. + +Service models expose derived ``build_system`` and ``crates`` properties. Crate +records contain ``name``, ``version``, and ``checksum``. These properties are not +manifest JSON fields. + +``render_application_files(manifest)`` and +``render_service_files(manifest, service)`` return lists of layer-relative +``(Path, str)`` pairs without writing files. Unsupported recipe literals raise +``ValueError`` identifying the field. + Verification ------------ -Verification covers an application with a service, multiple unit files, -dependency mapping, dev/prod selection, invalid manifests, repeatable generation, -and stale output removal. BitBake parsing and an image build validate the -generated recipes. -Launcher integration checks cover installed applications appearing by name and -starting on selection while their services run independently. +Install the development dependencies in the environment described in +:doc:`building`, then run the tooling tests from the repository root: + +.. code-block:: bash + + .venv-tools/bin/python -m pip install -r tools/requirements-dev.txt + .venv-tools/bin/python -m pytest + +Tests live under ``tools/tests/``. They cover manifest validation, recipe +rendering, native installation, and generated layer replacement. Yocto builds +and target checks are needed to verify cross-compilation, packaging, crate +fetching, and service startup. diff --git a/docs/applications.rst b/docs/applications.rst index 985436d..9e33af6 100644 --- a/docs/applications.rst +++ b/docs/applications.rst @@ -1,9 +1,15 @@ User applications ================= -Each application lives under ``apps/`` with a ``mosaic-app.json`` manifest, -Qt/CMake application code, and a license. You can optionally write services, -systemd units, and tests alongside the application. +To add an application: + +1. Create ``apps//`` with a ``mosaic-app.json`` manifest, Qt/CMake + application code, and a license file. Use ``apps/dummy-application/`` as an + example. +2. Set the manifest's application ID, display name, source directory, and + executable name. Add any dependencies and optional services. +3. Add the CMake installation rules shown below, then build your chosen image + using the commands in :doc:`building`. Build selection uses the manifest: @@ -11,14 +17,13 @@ Build selection uses the manifest: - ``include_in_prod``: also include the enabled bundle in PROD. Selection includes the bundle's required services. Applications start when -selected from the launcher after unlock; services run independently. +selected from the launcher after unlock. Services run independently. Manifest fields --------------- -The parser reads ``apps/*/mosaic-app.json`` in directory order and validates all -bundles, including disabled ones. Unknown fields and duplicate JSON keys are -errors. Each error identifies the manifest and affected field. +All manifests must be valid, including disabled bundles. Unknown fields and +duplicate JSON keys are errors. Errors identify the manifest and affected field. The following top-level fields are required: @@ -30,7 +35,8 @@ The following top-level fields are required: ``~``. - ``enabled`` and ``include_in_prod`` are booleans with the selection behavior described above. -- ``license`` is a nonempty Yocto license identifier or expression. +- ``license`` is a nonempty Yocto license identifier or expression, without + quotes, backslashes, or dollar signs. - ``license_file`` points to the bundle's license file. - ``app`` describes the application build and executable. @@ -45,42 +51,39 @@ Both ``app`` and each service accept optional ``build_dependencies`` and ``runtime_dependencies`` lists, defaulting to ``[]``. These contain Yocto recipe names and runtime package names respectively. Names start with a lowercase letter or digit and contain lowercase letters, digits, dots, hyphens, or ``+``. Entries -are plain names without version constraints or BitBake expressions. Provider -availability is checked by BitBake. +are plain names without version constraints or BitBake expressions and must be +available in your configured Yocto layers. The application's standard Qt Quick +dependencies are already included. Declare any additional libraries, and declare +service dependencies separately. ``services`` is an optional list, defaulting to ``[]``. Each service requires: - ``id``, following the application ID format and unique within the bundle. -- ``source_dir``, containing the service's ``CMakeLists.txt``. +- ``source_dir``, containing exactly one of ``CMakeLists.txt`` or ``Cargo.toml``. + Cargo services also require ``Cargo.lock`` in the same directory. - ``unit``, a path or nonempty list of paths to systemd unit files. Unit filenames must be unique across bundles and contain only letters, digits, dots, underscores, ``@``, ``:``, ``+``, or hyphens. - ``start``, currently accepting only ``"boot"``. All source, license, and unit paths are relative to the manifest's directory. -They must exist and stay within that directory after resolving symlinks. Text -fields other than arguments reject control characters. The parser checks file -presence, not CMake code, license validity, or systemd unit contents. +They must exist and stay within that directory after resolving symlinks, as must +service build files and lockfiles. Use ASCII letters, digits, underscores, dots, +slashes, ``+``, or hyphens in bundle, source, and license paths. Unit paths also +allow ``@`` and ``:``. Text fields other than arguments reject control characters. ``profiles`` is an optional object, defaulting to ``{}``, with optional ``dev`` and ``prod`` entries. Each entry contains a required ``settings`` object holding -application-defined JSON values. The parser preserves these settings without -interpreting them. +application-defined JSON values. CMake build and installation ---------------------------- -Application recipes inherit ``qt6-cmake`` and use its standard configure, -compile, and install tasks. The entire bundle is fetched locally, preserving its -directory structure. The recipe uses the declared ``license`` and an MD5 checksum -of ``license_file``. - -The recipe passes ``-DMOSAIC_APP_EXECUTABLE=`` to CMake. Apply this -variable to the executable target's ``OUTPUT_NAME`` and install the target into -``${CMAKE_INSTALL_BINDIR}``. Install required resources through CMake into -``${CMAKE_INSTALL_DATADIR}/mosaic/apps/`` so the generated package includes -them. For example, an application with ID ``example``, ``main.cpp``, and -``resources/`` can use: +Use ``MOSAIC_APP_EXECUTABLE`` as the executable target's ``OUTPUT_NAME``. Mosaic +sets it from ``app.executable``. Install the target into +``${CMAKE_INSTALL_BINDIR}`` and required resources into +``${CMAKE_INSTALL_DATADIR}/mosaic/apps/``. For example, an application with ID +``example``, ``main.cpp``, and ``resources/`` can use: .. code-block:: cmake @@ -98,16 +101,7 @@ them. For example, an application with ID ``example``, ``main.cpp``, and install(DIRECTORY resources/ DESTINATION "${CMAKE_INSTALL_DATADIR}/mosaic/apps/example") -After CMake installation, the recipe requires an executable file at -``${D}${bindir}/``. A missing or non-executable file fails the -build with guidance to check the naming and installation rules. - -The renderer adds ``qtbase``, ``qtdeclarative``, and ``qtdeclarative-native`` to -build dependencies. Runtime dependencies include ``qtbase-plugins``, -``qtdeclarative-qmlplugins``, and ``qtwayland-plugins``. Manifest declarations -extend these lists. Required service packages use the reserved names -``mosaic-service--``. Dependency lists are deduplicated and -sorted. Service-specific dependencies belong to the service recipes. +The installed executable must match ``app.executable`` or the build fails. Example manifest ---------------- @@ -115,26 +109,127 @@ Example manifest .. literalinclude:: ../apps/dummy-application/mosaic-app.json :language: json -Parser interface ----------------- +Adding services +--------------- + +Add each service to the manifest's ``services`` list. Provide a CMake or Cargo +project and systemd units with ``ExecStart`` paths matching the installed +executables. Include an appropriate ``[Install]`` section so systemd can start +the service at boot. Mosaic installs and enables the declared units for you. + +CMake service example +~~~~~~~~~~~~~~~~~~~~~ + +Inside an existing application bundle, add: + +.. code-block:: text + + services/collector/ + CMakeLists.txt + main.c + collector.service + +Add this entry to the manifest's ``services`` array: + +.. code-block:: json + + { + "id": "collector", + "source_dir": "services/collector", + "unit": "services/collector/collector.service", + "start": "boot", + "build_dependencies": ["zlib"], + "runtime_dependencies": [] + } + +``CMakeLists.txt`` owns executable naming and installation: + +.. code-block:: cmake + + cmake_minimum_required(VERSION 3.16) + project(collector LANGUAGES C) + include(GNUInstallDirs) + find_package(ZLIB REQUIRED) + add_executable(collector main.c) + target_link_libraries(collector PRIVATE ZLIB::ZLIB) + install(TARGETS collector RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") + +For a minimal ``main.c`` that prints the linked library version: + +.. code-block:: c + + #include + #include + int main(void) { puts(zlibVersion()); return 0; } + +Use this ``collector.service`` for the one-shot example: + +.. code-block:: ini + + [Unit] + Description=Collector example + + [Service] + Type=oneshot + ExecStart=/usr/bin/collector + + [Install] + WantedBy=multi-user.target + +Rust service example +~~~~~~~~~~~~~~~~~~~~ + +A standalone Cargo package can use this layout: + +.. code-block:: text + + services/reporter/ + Cargo.toml + Cargo.lock + src/main.rs + reporter.service + +Add this manifest entry: + +.. code-block:: json + + { + "id": "reporter", + "source_dir": "services/reporter", + "unit": "services/reporter/reporter.service", + "start": "boot", + "build_dependencies": [], + "runtime_dependencies": [] + } + +``Cargo.toml`` defines the executable through Cargo's normal naming rules: + +.. code-block:: toml + + [package] + name = "reporter" + version = "0.1.0" + edition = "2021" -``tools/app_manifest.py`` exposes ``load_manifest(path)`` for one bundle and -``load_manifests(apps_dir)`` for all bundles. They return typed Pydantic models with -optional defaults filled in, source and file paths resolved to absolute -``Path`` objects, and ``unit`` normalized to a list. Fields are accessed as -attributes, such as ``manifest.app.executable``. Each model exposes -``manifest_path`` for diagnostics. An empty application directory returns an -empty list. Invalid input raises ``ManifestError`` without writing files. +Use ``fn main() { println!("Reporter ready"); }`` in ``src/main.rs``. Generate +and commit ``Cargo.lock`` by running ``cargo generate-lockfile --offline`` in +``services/reporter``. Copy the unit example above to ``reporter.service``, +changing its description and ``ExecStart`` to ``/usr/bin/reporter``. Rust +executables are installed into ``/usr/bin`` in the standard image configuration. -Install the development dependencies once in the build tools environment -described in :doc:`building`, then run all Python tooling tests from the -repository root: +``source_dir`` may also identify a Cargo workspace root containing its lockfile. +Builds use Cargo's default package, target, and feature selection. Keep workspace +members and path dependencies inside the application bundle, including dependencies +outside ``source_dir``. -.. code-block:: bash +Declare Rust crate dependencies in ``Cargo.toml`` and update ``Cargo.lock`` with +Cargo before building. Native library recipe dependencies belong in the service's +``build_dependencies`` and additional runtime packages in ``runtime_dependencies``. +For example, a crate using system OpenSSL needs ``openssl`` in +``build_dependencies``. - .venv-tools/bin/python -m pip install -r tools/requirements-dev.txt - .venv-tools/bin/python -m pytest +Use crates.io dependencies or bundled path dependencies. Git dependencies and +alternative registries are not supported. Keep ``Cargo.lock`` committed and up +to date, including the checksums Cargo records for registry dependencies. -Pytest discovers tests under ``tools/tests/``. It does not scan upstream Yocto -sources. To run only the manifest tests, append -``tools/tests/test_app_manifest.py`` to the pytest command. +See :doc:`app-generation` for how Mosaic generates recipes and packages bundles. diff --git a/tools/app_layer.py b/tools/app_layer.py index 92ac63e..afe026a 100644 --- a/tools/app_layer.py +++ b/tools/app_layer.py @@ -8,9 +8,28 @@ import tempfile if __package__: - from .app_manifest import Manifest + from .app_manifest import Manifest, Service else: - from app_manifest import Manifest + from app_manifest import Manifest, Service + + +def _recipe_path(manifest: Manifest, field: str, path: Path, *, unit: bool = False) -> str: + pattern = r"[A-Za-z0-9_./@:+-]+" if unit else r"[A-Za-z0-9_./+-]+" + if not re.fullmatch(pattern, path.as_posix()): + raise ValueError( + f"{manifest.manifest_path}: {field}: cannot serialize path in BitBake " + "(use ASCII letters, digits, underscores, dots, slashes, + or -)" + ) + return path.as_posix() + + +def _recipe_license(manifest: Manifest) -> str: + if any(char in manifest.license for char in '\\"$'): + raise ValueError( + f"{manifest.manifest_path}: license: cannot serialize quotes, backslashes " + "or dollar signs in a BitBake literal" + ) + return hashlib.md5(manifest.license_file.read_bytes()).hexdigest() def render_application_files(manifest: Manifest) -> list[tuple[Path, str]]: @@ -20,20 +39,11 @@ def render_application_files(manifest: Manifest) -> list[tuple[Path, str]]: license_file = manifest.license_file.relative_to(bundle) for field, path in (("bundle", bundle), ("app.source_dir", source), ("license_file", license_file)): - if not re.fullmatch(r"[A-Za-z0-9_./+-]+", path.as_posix()): - raise ValueError( - f"{manifest.manifest_path}: {field}: cannot serialize path in BitBake " - "(use ASCII letters, digits, underscores, dots, slashes, + or -)" - ) - if any(char in manifest.license for char in '\\"$'): - raise ValueError( - f"{manifest.manifest_path}: license: cannot serialize quotes, backslashes " - "or dollar signs in a BitBake literal" - ) + _recipe_path(manifest, field, path) + checksum = _recipe_license(manifest) package = f"mosaic-app-{manifest.id}" unpacked = "${UNPACKDIR}/" + bundle.name - checksum = hashlib.md5(manifest.license_file.read_bytes()).hexdigest() build_dependencies = sorted({ "qtbase", "qtdeclarative", "qtdeclarative-native", *manifest.app.build_dependencies, @@ -98,6 +108,56 @@ def render_application_files(manifest: Manifest) -> list[tuple[Path, str]]: return [(Path(f"recipes-apps/{package}/{package}_{manifest.version}.bb"), recipe)] +def render_service_files(manifest: Manifest, service: Service) -> list[tuple[Path, str]]: + """Render one CMake or Cargo service recipe without writing files""" + bundle = manifest.manifest_path.parent + field = f"services[{service.id}]" + _recipe_path(manifest, "bundle", bundle) + source = _recipe_path(manifest, f"{field}.source_dir", service.source_dir.relative_to(bundle)) + license_file = _recipe_path(manifest, "license_file", manifest.license_file.relative_to(bundle)) + checksum = _recipe_license(manifest) + units = sorted(service.unit, key=lambda unit: unit.name) + unit_paths = [ + _recipe_path(manifest, f"{field}.unit", unit.relative_to(bundle), unit=True) + for unit in units + ] + package = f"mosaic-service-{manifest.id}-{service.id}" + unpacked = "${UNPACKDIR}/" + bundle.name + recipe = f'''SUMMARY = "Mosaic service {manifest.id}-{service.id}" +LICENSE = "{manifest.license}" +LIC_FILES_CHKSUM = "file://{unpacked}/{license_file};md5={checksum}" + +FILESEXTRAPATHS:prepend := "{bundle.parent.as_posix()}:" +SRC_URI = "file://{bundle.name};unpack=0" +S = "{unpacked}/{source}" + +inherit {service.build_system} systemd +''' + if service.crates: + recipe += '\nSRC_URI += "' + " ".join( + f"crate://crates.io/{crate.name}/{crate.version}" for crate in service.crates + ) + '"\n' + for crate in service.crates: + recipe += f'SRC_URI[{crate.name}-{crate.version}.sha256sum] = "{crate.checksum}"\n' + + for variable, dependencies in (("DEPENDS", service.build_dependencies), + ("RDEPENDS:${PN}", service.runtime_dependencies)): + if dependencies: + recipe += f'\n{variable} += "{" ".join(sorted(set(dependencies)))}"\n' + + recipe += '\ndo_install:append() {\n install -d "${D}${systemd_system_unitdir}"\n' + for unit, path in zip(units, unit_paths): + recipe += (f' install -m 0644 "{unpacked}/{path}" ' + f'"${{D}}${{systemd_system_unitdir}}/{unit.name}"\n') + recipe += '}\n\n' + recipe += f'SYSTEMD_SERVICE:${{PN}} = "{" ".join(unit.name for unit in units)}"\n' + recipe += 'SYSTEMD_AUTO_ENABLE:${PN} = "enable"\n' + recipe += 'FILES:${PN} += "' + " ".join( + "${systemd_system_unitdir}/" + unit.name for unit in units + ) + '"\n' + return [(Path(f"recipes-services/{package}/{package}_{manifest.version}.bb"), recipe)] + + def generate_layers(manifests: list[Manifest], output: Path) -> None: """Build the generated layer from validated manifests, rendering everything before writes""" enabled = sorted( diff --git a/tools/app_manifest.py b/tools/app_manifest.py index 63ddf1d..47c9382 100644 --- a/tools/app_manifest.py +++ b/tools/app_manifest.py @@ -2,6 +2,8 @@ import json import re +import tomllib +from dataclasses import dataclass from pathlib import Path from typing import Annotated, ClassVar, Literal, NoReturn, Self, cast @@ -67,7 +69,7 @@ def _unit_file(value: Path) -> Path: Dependency = Annotated[Text, Field(pattern=r"^[a-z0-9][a-z0-9.+-]*$")] Argument = Annotated[str, Field(pattern=r"^[^\x00]*$")] BundleFile = Annotated[FilePath, BeforeValidator(_bundle_path)] -SourceDirectory = Annotated[DirectoryPath, BeforeValidator(_bundle_path), AfterValidator(_cmake_directory)] +SourceDirectory = Annotated[DirectoryPath, BeforeValidator(_bundle_path)] UnitFile = Annotated[BundleFile, AfterValidator(_unit_file)] @@ -82,14 +84,91 @@ class Component(ManifestModel): class Application(Component): + source_dir: Annotated[SourceDirectory, AfterValidator(_cmake_directory)] executable: Annotated[Text, Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._+-]*$")] arguments: list[Argument] = Field(default_factory=list) +@dataclass(frozen=True, order=True) +class Crate: + name: str + version: str + checksum: str + + +def _cargo_crates(lockfile: Path) -> tuple[Crate, ...]: + with lockfile.open("rb") as stream: + lock = tomllib.load(stream) + packages = lock.get("package") + if not isinstance(packages, list) or not packages: + raise ValueError("Cargo.lock: expected a nonempty package array") + + records: dict[tuple[str, str, str | None], str | None] = {} + crates: set[Crate] = set() + for index, package in enumerate(packages): + field = f"Cargo.lock: package[{index}]" + if not isinstance(package, dict): + raise ValueError(f"{field}: expected a package table") + name, version = package.get("name"), package.get("version") + if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", name): + raise ValueError(f"{field}.name: invalid crate name for recipe serialization") + if not isinstance(version, str) or not re.fullmatch( + r"[0-9]+\.[0-9]+\.[0-9]+(?:-[A-Za-z0-9.-]+)?(?:\+[A-Za-z0-9.-]+)?", version + ): + raise ValueError(f"{field}.version: invalid crate version for recipe serialization") + source = package.get("source") + if "source" in package and source != "registry+https://github.com/rust-lang/crates.io-index": + raise ValueError(f"{field}.source: only crates.io registry dependencies are supported") + checksum = package.get("checksum") + if source is not None: + if not isinstance(checksum, str) or not re.fullmatch(r"[0-9a-fA-F]{64}", checksum): + raise ValueError(f"{field}.checksum: expected a SHA-256 checksum") + checksum = checksum.lower() + crates.add(Crate(name, version, checksum)) + elif "checksum" in package: + raise ValueError(f"{field}.checksum: local packages must not have a registry checksum") + key = (name, version, source) + if key in records and records[key] != checksum: + raise ValueError(f"{field}: conflicting duplicate package") + records[key] = checksum + return tuple(sorted(crates)) + + class Service(Component): id: Identifier unit: Annotated[list[UnitFile], Field(min_length=1)] start: Literal["boot"] + _build_system: Literal["cmake", "cargo"] = PrivateAttr() + _crates: tuple[Crate, ...] = PrivateAttr(default=()) + + @property + def build_system(self) -> Literal["cmake", "cargo"]: + return self._build_system + + @property + def crates(self) -> tuple[Crate, ...]: + return self._crates + + @model_validator(mode="after") + def detect_build_system(self, info: ValidationInfo) -> Self: + bundle = cast(Path, info.context).parent + cmake = self.source_dir / "CMakeLists.txt" + cargo = self.source_dir / "Cargo.toml" + try: + if cmake.is_file() == cargo.is_file(): + raise ValueError("require exactly one of CMakeLists.txt or Cargo.toml") + self._build_system = "cmake" if cmake.is_file() else "cargo" + files = [cmake] if self.build_system == "cmake" else [cargo, self.source_dir / "Cargo.lock"] + for path in files: + if not path.resolve().is_relative_to(bundle): + raise ValueError(f"{path.name}: path must stay inside the application bundle") + if not path.is_file(): + raise ValueError(f"missing {path.name}") + if self.build_system == "cargo": + self._crates = _cargo_crates(self.source_dir / "Cargo.lock") + except (OSError, RuntimeError, UnicodeError, ValueError) as error: + raise ValueError(f"source_dir: {error}") from error + return self @field_validator("unit", mode="before") @classmethod diff --git a/tools/tests/test_service_recipe.py b/tools/tests/test_service_recipe.py new file mode 100644 index 0000000..099d118 --- /dev/null +++ b/tools/tests/test_service_recipe.py @@ -0,0 +1,332 @@ +"""Service validation, recipe rendering, and native installation checks""" + +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import textwrap + +import pytest + +from tools.app_layer import render_service_files +from tools.app_manifest import Crate, Manifest, ManifestError, load_manifest, load_manifests + + +@pytest.fixture +def bundle(tmp_path: Path) -> Path: + bundle = tmp_path / "bundle" + (bundle / "app").mkdir(parents=True) + (bundle / "app/CMakeLists.txt").touch() + (bundle / "service").mkdir() + (bundle / "LICENSE").write_bytes(b"Test license\r\n") + (bundle / "worker.service").write_bytes( + b"[Service]\r\nExecStart=/usr/bin/worker\r\n[Install]\r\nWantedBy=multi-user.target\r\n" + ) + (bundle / "mosaic-app.json").write_text(json.dumps({ + "schema_version": 1, "id": "example", "name": "Example", "version": "1.2.3", + "enabled": True, "include_in_prod": False, "license": "MIT", "license_file": "LICENSE", + "app": {"source_dir": "app", "executable": "example"}, + "services": [{"id": "worker", "source_dir": "service", "unit": "worker.service", "start": "boot"}], + })) + return bundle + + +def update(bundle: Path, **fields: object) -> None: + path = bundle / "mosaic-app.json" + data = json.loads(path.read_text()) + data.update(fields) + path.write_text(json.dumps(data)) + + +def load(bundle: Path) -> Manifest: + return load_manifest(bundle / "mosaic-app.json") + + +def render(bundle: Path) -> str: + manifest = load(bundle) + return render_service_files(manifest, manifest.services[0])[0][1] + + +def cargo(bundle: Path, packages: list[dict] | None = None) -> None: + (bundle / "service/Cargo.toml").write_text('[package]\nname = "worker"\nversion = "1.0.0"\nedition = "2021"\n') + if packages is None: + packages = [{"name": "worker", "version": "1.0.0"}] + content = "version = 3\n" + for package in packages: + content += "\n[[package]]\n" + content += "".join(f"{key} = {json.dumps(value)}\n" for key, value in package.items()) + (bundle / "service/Cargo.lock").write_text(content) + + +def registry(name: str = "libc", version: str = "0.2.1", **fields: object) -> dict: + return {"name": name, "version": version, + "source": "registry+https://github.com/rust-lang/crates.io-index", + "checksum": "a" * 64, **fields} + + +@pytest.mark.parametrize("kind", ["cmake", "cargo"]) +def test_detection_and_derived_fields(bundle: Path, kind: str) -> None: + if kind == "cmake": + (bundle / "service/CMakeLists.txt").touch() + else: + cargo(bundle) + service = load(bundle).services[0] + assert service.build_system == kind + assert service.crates == () + assert "build_system" not in service.model_dump() + assert "crates" not in service.model_dump() + for field in ("build_system", "crates"): + data = json.loads((bundle / "mosaic-app.json").read_text()) + data["services"][0][field] = kind if field == "build_system" else [] + update(bundle, services=data["services"]) + with pytest.raises(ManifestError, match=field): + load(bundle) + data["services"][0].pop(field) + update(bundle, services=data["services"]) + + +@pytest.mark.parametrize("case", ["both", "neither", "nested", "missing-lock", "lock-directory"]) +def test_invalid_detection_even_when_disabled(bundle: Path, case: str) -> None: + if case in ("both", "missing-lock", "lock-directory"): + cargo(bundle) + if case == "both": + (bundle / "service/CMakeLists.txt").touch() + if case in ("missing-lock", "lock-directory"): + (bundle / "service/Cargo.lock").unlink() + if case == "lock-directory": + (bundle / "service/Cargo.lock").mkdir() + if case == "nested": + (bundle / "service/nested").mkdir() + (bundle / "service/nested/CMakeLists.txt").touch() + update(bundle, enabled=False) + with pytest.raises(ManifestError, match=r"services\[0\].*source_dir"): + load_manifests(bundle.parent) + + +@pytest.mark.parametrize("filename", ["CMakeLists.txt", "Cargo.toml", "Cargo.lock"]) +def test_build_files_cannot_escape_bundle(bundle: Path, filename: str) -> None: + if filename != "CMakeLists.txt": + cargo(bundle) + (bundle / "service" / filename).unlink() + outside = bundle.parent / filename + outside.write_text("outside") + (bundle / "service" / filename).symlink_to(outside) + with pytest.raises(ManifestError, match="stay inside the application bundle"): + load(bundle) + + +def test_application_still_requires_cmake(bundle: Path) -> None: + cargo(bundle) + (bundle / "app/CMakeLists.txt").unlink() + (bundle / "app/Cargo.toml").touch() + with pytest.raises(ManifestError, match=r"app.source_dir.*missing CMakeLists"): + load(bundle) + + +@pytest.mark.parametrize("content", ["not TOML {", "version = 3", "package = []", "package = [1]", "[package]\nname = 'x'", "[[package]]\nname = 'x'"]) +def test_malformed_lock(bundle: Path, content: str) -> None: + cargo(bundle) + (bundle / "service/Cargo.lock").write_text(content) + with pytest.raises(ManifestError, match=r"services\[0\].*source_dir"): + load(bundle) + + +@pytest.mark.parametrize("fields", [ + {"name": "bad/name"}, {"name": "${PN}"}, {"name": 1}, + {"version": "1.0.0;name=evil"}, {"version": "1.0.0\n"}, {"version": 1}, + {"checksum": "z" * 64}, {"checksum": "a" * 63}, {"checksum": 1}, + {"source": "git+https://example.com/repo#abc"}, + {"source": "registry+https://example.com/index"}, {"source": 1}, +]) +def test_invalid_registry_records(bundle: Path, fields: dict) -> None: + cargo(bundle, [registry(**fields)]) + with pytest.raises(ManifestError, match="Cargo.lock.*package"): + load(bundle) + + +def test_missing_checksum_and_conflicting_duplicates(bundle: Path) -> None: + package = registry() + del package["checksum"] + cargo(bundle, [package]) + with pytest.raises(ManifestError, match="checksum"): + load(bundle) + cargo(bundle, [registry(), registry(checksum="b" * 64)]) + with pytest.raises(ManifestError, match="conflicting duplicate"): + load(bundle) + + +def test_crates_are_sorted_deduplicated_and_local_packages_are_omitted(bundle: Path) -> None: + packages = [registry("zeta"), registry("libc", "0.2.2+build.1"), registry(), + registry(checksum="A" * 64), {"name": "worker", "version": "1.0.0"}, + {"name": "local-helper", "version": "0.1.0"}] + cargo(bundle, packages) + service = load(bundle).services[0] + assert service.crates == (Crate("libc", "0.2.1", "a" * 64), + Crate("libc", "0.2.2+build.1", "a" * 64), Crate("zeta", "0.2.1", "a" * 64)) + recipe = render(bundle) + assert 'inherit cargo systemd' in recipe + for crate in service.crates: + assert recipe.count(f"crate://crates.io/{crate.name}/{crate.version}") == 1 + assert f'SRC_URI[{crate.name}-{crate.version}.sha256sum] = "{crate.checksum}"' in recipe + assert "crate://crates.io/worker" not in recipe + assert "local-helper" not in recipe + assert "CARGO_BUILD_FLAGS" not in recipe + assert "CARGO_DISABLE_BITBAKE_VENDORING" not in recipe + assert not re.search(r"^do_(configure|compile|install)\(\)", recipe, re.M) + cargo(bundle, list(reversed(packages))) + assert render(bundle) == recipe + + +def stage_units(recipe: str, bundle: Path, stage: Path) -> None: + body = re.search(r"do_install:append\(\) \{\n(.*?)\n\}", recipe, re.S)[1] + for variable, value in {"UNPACKDIR": str(bundle.parent), "D": str(stage), + "systemd_system_unitdir": "/usr/lib/systemd/system"}.items(): + body = body.replace("${" + variable + "}", value) + subprocess.run(["sh", "-eu", "-c", body], check=True, capture_output=True, text=True) + + +def test_recipe_units_dependencies_and_no_writes(bundle: Path, tmp_path: Path) -> None: + (bundle / "service/CMakeLists.txt").touch() + directory = bundle / "units@bus:1" + directory.mkdir() + (directory / "a@bus:1.service").write_bytes(b"[Service]\nExecStart=/usr/bin/worker --label='${literal}'\n") + data = json.loads((bundle / "mosaic-app.json").read_text()) + service = data["services"][0] + service.update(unit=["worker.service", "units@bus:1/a@bus:1.service"], + build_dependencies=["zlib", "openssl", "zlib"], + runtime_dependencies=["bash", "zlib", "bash"]) + update(bundle, services=data["services"]) + before = {p: (p.read_bytes(), p.stat().st_mode) for p in bundle.rglob("*") if p.is_file()} + manifest = load(bundle) + files = render_service_files(manifest, manifest.services[0]) + assert len(files) == 1 + path, recipe = files[0] + assert path == Path("recipes-services/mosaic-service-example-worker/mosaic-service-example-worker_1.2.3.bb") + assert 'SRC_URI = "file://bundle;unpack=0"' in recipe + assert f'FILESEXTRAPATHS:prepend := "{bundle.parent}:"' in recipe + assert 'S = "${UNPACKDIR}/bundle/service"' in recipe + checksum = hashlib.md5((bundle / "LICENSE").read_bytes()).hexdigest() + assert f'LIC_FILES_CHKSUM = "file://${{UNPACKDIR}}/bundle/LICENSE;md5={checksum}"' in recipe + assert 'LICENSE = "MIT"' in recipe + assert 'inherit cmake systemd' in recipe + assert 'DEPENDS += "openssl zlib"' in recipe + assert 'RDEPENDS:${PN} += "bash zlib"' in recipe + assert 'SYSTEMD_SERVICE:${PN} = "a@bus:1.service worker.service"' in recipe + assert 'SYSTEMD_AUTO_ENABLE:${PN} = "enable"' in recipe + assert 'FILES:${PN} += "${systemd_system_unitdir}/a@bus:1.service ${systemd_system_unitdir}/worker.service"' in recipe + assert "qt" not in recipe + assert files == render_service_files(manifest, manifest.services[0]) + assert before == {p: (p.read_bytes(), p.stat().st_mode) for p in bundle.rglob("*") if p.is_file()} + assert list(tmp_path.iterdir()) == [bundle] + stage = tmp_path / "stage" + stage_units(recipe, bundle, stage) + for unit in manifest.services[0].unit: + installed = stage / "usr/lib/systemd/system" / unit.name + assert installed.read_bytes() == unit.read_bytes() + assert installed.stat().st_mode & 0o777 == 0o644 + service["unit"].reverse() + service["build_dependencies"].reverse() + update(bundle, services=data["services"]) + assert render(bundle) == recipe + + +def test_no_empty_or_automatic_dependencies(bundle: Path) -> None: + cargo(bundle) + recipe = render(bundle) + assert "DEPENDS" not in recipe + assert "crate://" not in recipe + + +@pytest.mark.parametrize("field", ["source_dir", "unit", "license_file", "bundle"]) +@pytest.mark.parametrize("unsafe", ["space here", '${PN}', 'quote"', 'back\\slash', 'semi;colon']) +def test_serialization_errors(bundle: Path, field: str, unsafe: str) -> None: + (bundle / "service/CMakeLists.txt").touch() + data = json.loads((bundle / "mosaic-app.json").read_text()) + if field == "bundle": + destination = bundle.with_name(unsafe) + bundle.rename(destination) + bundle = destination + elif field == "unit": + (bundle / unsafe).mkdir() + (bundle / "worker.service").rename(bundle / unsafe / "worker.service") + data["services"][0][field] = f"{unsafe}/worker.service" + elif field == "source_dir": + (bundle / "service").rename(bundle / unsafe) + data["services"][0][field] = unsafe + else: + (bundle / "LICENSE").rename(bundle / unsafe) + data[field] = unsafe + update(bundle, **data) + with pytest.raises(ValueError, match=field + ": cannot serialize"): + render(bundle) + + +@pytest.mark.skipif(shutil.which("cmake") is None, reason="CMake is not installed") +def test_native_cmake_install(bundle: Path, tmp_path: Path) -> None: + (bundle / "service/CMakeLists.txt").write_text(textwrap.dedent('''\ + cmake_minimum_required(VERSION 3.16) + project(worker LANGUAGES C) + include(GNUInstallDirs) + add_executable(worker main.c) + install(TARGETS worker RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") + ''')) + (bundle / "service/main.c").write_text("int main(void) { return 0; }\n") + build, stage = tmp_path / "build", tmp_path / "stage" + subprocess.run(["cmake", "-S", str(bundle / "service"), "-B", str(build), + "-DCMAKE_INSTALL_PREFIX=/usr", "-DCMAKE_INSTALL_BINDIR=bin"], + check=True, capture_output=True, text=True) + subprocess.run(["cmake", "--build", str(build)], check=True, capture_output=True, text=True) + subprocess.run(["cmake", "--install", str(build)], env={**os.environ, "DESTDIR": str(stage)}, + check=True, capture_output=True, text=True) + stage_units(render(bundle), bundle, stage) + subprocess.run([str(stage / "usr/bin/worker")], check=True) + + +@pytest.mark.skipif(shutil.which("cargo") is None, reason="Cargo is not installed") +@pytest.mark.parametrize("workspace", [False, True]) +def test_offline_cargo_with_bundled_path_dependency(bundle: Path, tmp_path: Path, workspace: bool) -> None: + cargo(bundle, [{"name": "worker", "version": "1.0.0"}, {"name": "helper", "version": "1.0.0"}]) + source = bundle / "service" + if workspace: + (source / "Cargo.toml").write_text('[workspace]\nmembers = ["worker", "helper"]\nresolver = "2"\n') + source = source / "worker" + source.mkdir() + (source / "Cargo.toml").write_text('[package]\nname = "worker"\nversion = "1.0.0"\nedition = "2021"\n' + '[dependencies]\nhelper = { path = "../helper" }\n') + (source / "src").mkdir() + (source / "src/main.rs").write_text('fn main() { println!("{}", helper::message()); }\n') + helper = source.parent / "helper" + (helper / "src").mkdir(parents=True) + (helper / "Cargo.toml").write_text('[package]\nname = "helper"\nversion = "1.0.0"\nedition = "2021"\n') + (helper / "src/lib.rs").write_text('pub fn message() -> &\'static str { "bundled helper" }\n') + environment = {**os.environ, "CARGO_HOME": str(tmp_path / "cargo-home")} + subprocess.run(["cargo", "generate-lockfile", "--offline", "--manifest-path", + str(bundle / "service/Cargo.toml")], env=environment, + check=True, capture_output=True, text=True) + original = (bundle / "service/Cargo.lock").read_bytes() + target = tmp_path / "target" + subprocess.run(["cargo", "build", "--frozen", "--offline", "--release", "--manifest-path", + str(bundle / "service/Cargo.toml"), "--target-dir", str(target)], + env=environment, + check=True, capture_output=True, text=True) + assert (bundle / "service/Cargo.lock").read_bytes() == original + assert load(bundle).services[0].crates == () + stage = tmp_path / "stage" + cargo_class = (Path(__file__).resolve().parents[2] + / "yocto/sources/openembedded-core/meta/classes-recipe/cargo.bbclass").read_text() + install = re.search(r"cargo_do_install \(\) \{\n(.*?)\n\}", cargo_class, re.S)[1] + for variable, value in {"B": str(tmp_path), "D": str(stage), "CARGO_TARGET_SUBDIR": "release", + "bindir": "/usr/bin", "rustlibdir": "/usr/lib/rust", + "CARGO_INSTALL_LIBRARIES": ""}.items(): + install = install.replace("${" + variable + "}", value) + subprocess.run(["sh", "-eu", "-c", "die() { echo \"$@\" >&2; exit 1; }\n" + + "cargo_install() {\n" + install + "\n}\ncargo_install"], + check=True, capture_output=True, text=True) + executable = stage / "usr/bin/worker" + assert executable.stat().st_mode & 0o777 == 0o755 + result = subprocess.run([str(executable)], check=True, capture_output=True, text=True) + assert result.stdout == "bundled helper\n" + stage_units(render(bundle), bundle, stage) From b1896484fa242b001e3b489a514354003bc0672b Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Thu, 8 Oct 2026 17:33:19 +0200 Subject: [PATCH 09/11] Render layer config --- tools/app_layer.py | 29 +++++++++++++++++ tools/tests/test_app_layer.py | 61 +++++++++++++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/tools/app_layer.py b/tools/app_layer.py index afe026a..74c88b5 100644 --- a/tools/app_layer.py +++ b/tools/app_layer.py @@ -158,6 +158,35 @@ def render_service_files(manifest: Manifest, service: Service) -> list[tuple[Pat return [(Path(f"recipes-services/{package}/{package}_{manifest.version}.bb"), recipe)] +def render_layer_config(manifests: list[Manifest]) -> str: + """Render layer registration and profile package lists without writing files""" + enabled = sorted( + (manifest for manifest in manifests if manifest.enabled), + key=lambda manifest: manifest.id, + ) + + common = " ".join(f"mosaic-app-{manifest.id}" + for manifest in enabled if manifest.include_in_prod) + dev_only = " ".join(f"mosaic-app-{manifest.id}" + for manifest in enabled if not manifest.include_in_prod) + + return f'''BBPATH .= ":${{LAYERDIR}}" + +BBFILES += "${{LAYERDIR}}/recipes-*/*/*.bb" + +BBFILE_COLLECTIONS += "mosaic-apps" +BBFILE_PATTERN_mosaic-apps := "^${{LAYERDIR}}/" +BBFILE_PRIORITY_mosaic-apps = "6" +BBFILE_PATTERN_IGNORE_EMPTY_mosaic-apps = "1" + +LAYERDEPENDS_mosaic-apps = "core qt6-layer" +LAYERSERIES_COMPAT_mosaic-apps = "wrynose" + +MOSAIC_APPS_COMMON = "{common}" +MOSAIC_APPS_DEV_ONLY = "{dev_only}" +''' + + def generate_layers(manifests: list[Manifest], output: Path) -> None: """Build the generated layer from validated manifests, rendering everything before writes""" enabled = sorted( diff --git a/tools/tests/test_app_layer.py b/tools/tests/test_app_layer.py index fc41454..afb4fce 100644 --- a/tools/tests/test_app_layer.py +++ b/tools/tests/test_app_layer.py @@ -1,10 +1,67 @@ -"""Generated layer replacement using isolated filesystem data""" +"""Layer configuration and replacement using isolated filesystem data""" +import itertools from pathlib import Path +import re import pytest -from tools.app_layer import replace_layer +from tools.app_layer import render_application_files, render_layer_config, render_service_files, replace_layer +from tools.app_manifest import Manifest, load_manifest + + +@pytest.fixture +def manifest() -> Manifest: + return load_manifest(Path(__file__).resolve().parents[2] / "apps/dummy-application/mosaic-app.json") + + +def test_layer_profile_selection(manifest: Manifest) -> None: + manifests = [ + manifest.model_copy(update={"id": name, "enabled": enabled, "include_in_prod": prod}) + for name, enabled, prod in [ + ("zulu", True, True), ("alpha", True, True), + ("tools", True, False), ("debug", True, False), + ("disabled-prod", False, True), ("disabled-dev", False, False), + ] + ] + config = render_layer_config(manifests) + assert 'MOSAIC_APPS_COMMON = "mosaic-app-alpha mosaic-app-zulu"' in config + assert 'MOSAIC_APPS_DEV_ONLY = "mosaic-app-debug mosaic-app-tools"' in config + assert "disabled" not in config + assert "mosaic-service-" not in config + for order in itertools.permutations(manifests): + assert render_layer_config(list(order)) == config + assert [item.id for item in manifests] == [ + "zulu", "alpha", "tools", "debug", "disabled-prod", "disabled-dev", + ] + + +def test_empty_layer_config(manifest: Manifest) -> None: + config = render_layer_config([]) + assert 'MOSAIC_APPS_COMMON = ""' in config + assert 'MOSAIC_APPS_DEV_ONLY = ""' in config + assert 'BBFILE_PATTERN_IGNORE_EMPTY_mosaic-apps = "1"' in config + assert render_layer_config([manifest.model_copy(update={"enabled": False})]) == config + + +def test_layer_registration_discovers_generated_recipes(manifest: Manifest, tmp_path: Path) -> None: + config = render_layer_config([manifest]) + assert list(tmp_path.iterdir()) == [] + assert 'BBPATH .= ":${LAYERDIR}"' in config + assert 'BBFILE_COLLECTIONS += "mosaic-apps"' in config + assert 'BBFILE_PRIORITY_mosaic-apps = "6"' in config + assert 'LAYERDEPENDS_mosaic-apps = "core qt6-layer"' in config + assert 'LAYERSERIES_COMPAT_mosaic-apps = "wrynose"' in config + files = render_application_files(manifest) + for service in manifest.services: + files.extend(render_service_files(manifest, service)) + layer = tmp_path / "meta-mosaic-apps" + replace_layer(layer, [(Path("conf/layer.conf"), config), *files]) + glob = re.search(r'^BBFILES \+= "(.*)"$', config, re.M)[1] + assert set(layer.glob(glob.removeprefix("${LAYERDIR}/"))) == {layer / path for path, _ in files} + pattern = re.search(r'^BBFILE_PATTERN_mosaic-apps := "(.*)"$', config, re.M)[1] + pattern = pattern.replace("${LAYERDIR}", str(layer)) + assert all(re.match(pattern, str(layer / path)) for path, _ in files) @pytest.fixture From 2ac1f5a7cff28ad316adbcc9a1f30b4d14af9b9d Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Thu, 8 Oct 2026 17:45:04 +0200 Subject: [PATCH 10/11] Add layer validation --- tools/app_layer.py | 24 ++++++++++++++++++++- tools/tests/test_app_layer.py | 39 ++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/tools/app_layer.py b/tools/app_layer.py index 74c88b5..33061b2 100644 --- a/tools/app_layer.py +++ b/tools/app_layer.py @@ -201,10 +201,32 @@ def generate_layers(manifests: list[Manifest], output: Path) -> None: files.extend(render_service_files(manifest, service)) files.append((Path("conf/layer.conf"), render_layer_config(enabled))) - validate_layer_output(files, output, manifests) + validate_layer_output(output, manifests) replace_layer(output, files) +def validate_layer_output(output: Path, manifests: list[Manifest]) -> None: + """Prevent package collisions and output overlapping source bundles""" + destination = output.resolve() + packages: set[str] = set() + + for manifest in manifests: + bundle = manifest.manifest_path.parent.resolve() + if destination.is_relative_to(bundle) or bundle.is_relative_to(destination): + raise ValueError(f"{output}: \ + generated layer must not overlap application bundle {bundle}") + if not manifest.enabled: + continue + + components = [f"mosaic-app-{manifest.id}"] + components.extend(f"mosaic-service-{manifest.id}-{service.id}" + for service in manifest.services) + for package in components: + if package in packages: + raise ValueError(f"{manifest.manifest_path}: duplicate generated package {package}") + packages.add(package) + + def replace_layer(output: Path, files: list[tuple[Path, str]]) -> None: """Replace a validated generated-only destination with UTF-8 files diff --git a/tools/tests/test_app_layer.py b/tools/tests/test_app_layer.py index afb4fce..1b2b604 100644 --- a/tools/tests/test_app_layer.py +++ b/tools/tests/test_app_layer.py @@ -6,7 +6,10 @@ import pytest -from tools.app_layer import render_application_files, render_layer_config, render_service_files, replace_layer +from tools.app_layer import ( + generate_layers, render_application_files, render_layer_config, + render_service_files, replace_layer, validate_layer_output, +) from tools.app_manifest import Manifest, load_manifest @@ -64,6 +67,40 @@ def test_layer_registration_discovers_generated_recipes(manifest: Manifest, tmp_ assert all(re.match(pattern, str(layer / path)) for path, _ in files) +def test_validate_layer_before_replacement(manifest: Manifest, tmp_path: Path) -> None: + output = tmp_path / "layer" + generate_layers([manifest], output) + original = {path.relative_to(output): path.read_text() + for path in output.rglob("*") if path.is_file()} + validate_layer_output(output, [manifest]) + + disabled = manifest.model_copy(update={"enabled": False}) + validate_layer_output(output, []) + validate_layer_output(output, [disabled]) + bundle = manifest.manifest_path.parent + alias = tmp_path / "bundle-alias" + alias.symlink_to(bundle, target_is_directory=True) + for destination in (bundle, bundle.parent, bundle / "generated", alias / "generated"): + with pytest.raises(ValueError, match="must not overlap"): + validate_layer_output(destination, [disabled]) + + first = manifest.model_copy(update={"id": "a-b", "services": [ + manifest.services[0].model_copy(update={"id": "c"}), + ]}) + second = manifest.model_copy(update={"id": "a", "version": "2.0.0", "services": [ + manifest.services[0].model_copy(update={"id": "b-c"}), + ]}) + with pytest.raises(ValueError, match="duplicate generated package mosaic-service-a-b-c"): + generate_layers([first, second], output) + assert original == {path.relative_to(output): path.read_text() + for path in output.rglob("*") if path.is_file()} + + generate_layers([], output) + assert {path.relative_to(output) for path in output.rglob("*") if path.is_file()} == { + Path("conf/layer.conf"), + } + + @pytest.fixture def files() -> list[tuple[Path, str]]: return [ From 47f2146fe93820dbfcd1e0916a7109575273e999 Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Thu, 8 Oct 2026 17:52:20 +0200 Subject: [PATCH 11/11] Inherit pkgconfig in generated app recipes --- tools/app_layer.py | 2 +- tools/tests/test_service_recipe.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/app_layer.py b/tools/app_layer.py index 33061b2..42c4d70 100644 --- a/tools/app_layer.py +++ b/tools/app_layer.py @@ -131,7 +131,7 @@ def render_service_files(manifest: Manifest, service: Service) -> list[tuple[Pat SRC_URI = "file://{bundle.name};unpack=0" S = "{unpacked}/{source}" -inherit {service.build_system} systemd +inherit {service.build_system} pkgconfig systemd ''' if service.crates: recipe += '\nSRC_URI += "' + " ".join( diff --git a/tools/tests/test_service_recipe.py b/tools/tests/test_service_recipe.py index 099d118..84a40a1 100644 --- a/tools/tests/test_service_recipe.py +++ b/tools/tests/test_service_recipe.py @@ -167,7 +167,7 @@ def test_crates_are_sorted_deduplicated_and_local_packages_are_omitted(bundle: P assert service.crates == (Crate("libc", "0.2.1", "a" * 64), Crate("libc", "0.2.2+build.1", "a" * 64), Crate("zeta", "0.2.1", "a" * 64)) recipe = render(bundle) - assert 'inherit cargo systemd' in recipe + assert 'inherit cargo pkgconfig systemd' in recipe for crate in service.crates: assert recipe.count(f"crate://crates.io/{crate.name}/{crate.version}") == 1 assert f'SRC_URI[{crate.name}-{crate.version}.sha256sum] = "{crate.checksum}"' in recipe @@ -188,8 +188,12 @@ def stage_units(recipe: str, bundle: Path, stage: Path) -> None: subprocess.run(["sh", "-eu", "-c", body], check=True, capture_output=True, text=True) -def test_recipe_units_dependencies_and_no_writes(bundle: Path, tmp_path: Path) -> None: - (bundle / "service/CMakeLists.txt").touch() +@pytest.mark.parametrize("kind", ["cmake", "cargo"]) +def test_recipe_units_dependencies_and_no_writes(bundle: Path, tmp_path: Path, kind: str) -> None: + if kind == "cmake": + (bundle / "service/CMakeLists.txt").touch() + else: + cargo(bundle) directory = bundle / "units@bus:1" directory.mkdir() (directory / "a@bus:1.service").write_bytes(b"[Service]\nExecStart=/usr/bin/worker --label='${literal}'\n") @@ -211,7 +215,7 @@ def test_recipe_units_dependencies_and_no_writes(bundle: Path, tmp_path: Path) - checksum = hashlib.md5((bundle / "LICENSE").read_bytes()).hexdigest() assert f'LIC_FILES_CHKSUM = "file://${{UNPACKDIR}}/bundle/LICENSE;md5={checksum}"' in recipe assert 'LICENSE = "MIT"' in recipe - assert 'inherit cmake systemd' in recipe + assert f'inherit {kind} pkgconfig systemd' in recipe assert 'DEPENDS += "openssl zlib"' in recipe assert 'RDEPENDS:${PN} += "bash zlib"' in recipe assert 'SYSTEMD_SERVICE:${PN} = "a@bus:1.service worker.service"' in recipe