From 73f404c89eba9b0c20dfd1b46fe0011eed9b3c18 Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Fri, 2 Oct 2026 16:27:36 +0200 Subject: [PATCH 01/10] Add start for python app parsing and generation --- mosaic-build | 10 ++++++++++ tools/generate-app-layer.py | 6 ++++++ .../conf/templates/mosaic/bblayers.conf.sample | 1 + 3 files changed, 17 insertions(+) create mode 100755 mosaic-build create mode 100644 tools/generate-app-layer.py diff --git a/mosaic-build b/mosaic-build new file mode 100755 index 0000000..ad97065 --- /dev/null +++ b/mosaic-build @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e + +mosaic_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" + +python3 "$mosaic_root/tools/generate-app-layer.py" \ + --apps-dir "$mosaic_root/apps" \ + --output "$mosaic_root/build/generated/meta-mosaic-apps" + +exec bitbake "$@" diff --git a/tools/generate-app-layer.py b/tools/generate-app-layer.py new file mode 100644 index 0000000..490cb16 --- /dev/null +++ b/tools/generate-app-layer.py @@ -0,0 +1,6 @@ +def main() -> None: + print("Hello") + + +if __name__ == "__main__": + main() diff --git a/yocto/layers/meta-mosaic-product/conf/templates/mosaic/bblayers.conf.sample b/yocto/layers/meta-mosaic-product/conf/templates/mosaic/bblayers.conf.sample index ab38aff..e92f36c 100644 --- a/yocto/layers/meta-mosaic-product/conf/templates/mosaic/bblayers.conf.sample +++ b/yocto/layers/meta-mosaic-product/conf/templates/mosaic/bblayers.conf.sample @@ -13,4 +13,5 @@ BBLAYERS ?= " \ ##OEROOT##/../meta-st-stm32mp \ ##OEROOT##/../../layers/meta-mosaic \ ##OEROOT##/../../layers/meta-mosaic-product \ + ${TOPDIR}/generated/meta-mosaic-apps \ " From 895ef774f597b158ce8d2896504d79ddc59f3a90 Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Sun, 4 Oct 2026 16:15:20 +0200 Subject: [PATCH 02/10] Add doc for user app generation --- docs/app-generation.rst | 81 +++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 82 insertions(+) create mode 100644 docs/app-generation.rst diff --git a/docs/app-generation.rst b/docs/app-generation.rst new file mode 100644 index 0000000..9aa8fda --- /dev/null +++ b/docs/app-generation.rst @@ -0,0 +1,81 @@ +Application layer generation +============================ + +Application bundles become part of a Mosaic image through a generated BitBake +layer. Developers supply a manifest, application code, and any required services +under ``apps/`` without editing shared shell source or writing BitBake recipes. + +Build flow +---------- + +``mosaic-build`` runs the generator before invoking BitBake: + +1. The generator parses ``apps/*/mosaic-app.json`` and validates identifiers, + required fields, dependency declarations, source paths, licenses, and systemd + unit files. + It validates all manifests before writing output. Invalid input reports the + manifest and failing field, and stops the build before BitBake runs. +2. It generates recipes and layer configuration in + ``build/generated/meta-mosaic-apps/``. Repeated generation produces consistent + output and removes obsolete generated files within this layer. +3. ``mosaic-build`` forwards the build arguments to BitBake, which builds the + selected applications, services, dependencies, and image in dependency order. + +Recipe and image integration +---------------------------- + +The generator creates one recipe per application and one per bundled service. +Recipes use standard Yocto classes for Qt/CMake builds and systemd integration. +They reference the bundle's sources and license file. CMake and BitBake own compilation, +installation, and packaging. + +Applications and services explicitly declare their additional build and runtime +dependencies in the manifest. Build dependency recipe names map to ``DEPENDS``. +Runtime package names map to ``RDEPENDS``. The generator supplies the standard Qt +application dependencies rather than inferring dependencies from source code. + +Each application package depends on its required service packages. Service +recipes install the supplied systemd unit file or files and enable services +declared with ``start: boot``. Systemd runs services independently of application +selection and termination. + +Layer configuration supplies ``MOSAIC_APPS_COMMON`` and +``MOSAIC_APPS_DEV_ONLY`` to the existing application packagegroup. Disabled +bundles are excluded. Enabled bundles with ``include_in_prod: true`` are included +in both profiles, while the remaining enabled bundles are included only in dev. +Selecting an application also includes its required service packages. + +Each application recipe installs a JSON launcher entry at +``/usr/share/mosaic/apps/.json`` containing its existing ``id``, ``name``, +``executable``, and ``arguments``. +The shell reads installed entries, displays each application's name, and starts +its executable with the listed arguments when selected after unlock. The launcher +owns discovery and startup of installed applications. + +Tool files +---------- + +The tool uses Python's standard library and consists of three files under +``tools/``: + +``generate-app-layer.py`` + Accepts ``--apps-dir`` and ``--output``, coordinates parsing and generation, + and reports errors with a nonzero exit status. + +``app_manifest.py`` + Reads and validates manifests, returning validated application and service + data without writing generated files. + +``app_layer.py`` + Generates recipes, launcher entries, and layer configuration from the + validated data, and removes obsolete output. + +Verification +------------ + +Verification covers an application with a service, multiple unit files, +dependency mapping, dev/prod selection, invalid manifests, repeatable generation, +and stale output removal. BitBake parsing and an image build validate the +generated recipes. +Launcher integration checks cover installed applications appearing by name and +starting on selection while their services run independently. diff --git a/docs/index.rst b/docs/index.rst index ceef3bd..7f363f0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,4 +12,5 @@ applications and background services without changing the shell. mosiac-structure building applications + app-generation documentation From cc84ce878b85d868c37178eaae2a23ce138d5f8c Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Sun, 4 Oct 2026 16:22:21 +0200 Subject: [PATCH 03/10] Write entrypoint and update name to snake case --- docs/app-generation.rst | 2 +- mosaic-build | 2 +- tools/app_layer.py | 0 tools/app_manifest.py | 0 tools/generate-app-layer.py | 6 ------ tools/generate_app_layer.py | 29 +++++++++++++++++++++++++++++ 6 files changed, 31 insertions(+), 8 deletions(-) create mode 100644 tools/app_layer.py create mode 100644 tools/app_manifest.py delete mode 100644 tools/generate-app-layer.py create mode 100644 tools/generate_app_layer.py diff --git a/docs/app-generation.rst b/docs/app-generation.rst index 9aa8fda..3e068a5 100644 --- a/docs/app-generation.rst +++ b/docs/app-generation.rst @@ -58,7 +58,7 @@ Tool files The tool uses Python's standard library and consists of three files under ``tools/``: -``generate-app-layer.py`` +``generate_app_layer.py`` Accepts ``--apps-dir`` and ``--output``, coordinates parsing and generation, and reports errors with a nonzero exit status. diff --git a/mosaic-build b/mosaic-build index ad97065..fe161a1 100755 --- a/mosaic-build +++ b/mosaic-build @@ -3,7 +3,7 @@ set -e mosaic_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" -python3 "$mosaic_root/tools/generate-app-layer.py" \ +python3 "$mosaic_root/tools/generate_app_layer.py" \ --apps-dir "$mosaic_root/apps" \ --output "$mosaic_root/build/generated/meta-mosaic-apps" diff --git a/tools/app_layer.py b/tools/app_layer.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/app_manifest.py b/tools/app_manifest.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/generate-app-layer.py b/tools/generate-app-layer.py deleted file mode 100644 index 490cb16..0000000 --- a/tools/generate-app-layer.py +++ /dev/null @@ -1,6 +0,0 @@ -def main() -> None: - print("Hello") - - -if __name__ == "__main__": - main() diff --git a/tools/generate_app_layer.py b/tools/generate_app_layer.py new file mode 100644 index 0000000..2f5a174 --- /dev/null +++ b/tools/generate_app_layer.py @@ -0,0 +1,29 @@ +"""Generate a Yocto application layer from user application manifests""" + +import argparse +from pathlib import Path + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--apps-dir", + type=Path, + required=True, + help="Directory containing application bundles", + ) + parser.add_argument( + "--output", + type=Path, + required=True, + help="Destination for the generated BitBake layer", + ) + parser.parse_args() + + # TODO: Read and validate manifests with app_manifest before writing output + # TODO: Generate the layer from the validated data with app_layer + parser.exit(1, "error: application layer generation is not implemented yet\n") + + +if __name__ == "__main__": + main() From ccdb7bf69d113e795bd4ea755f81084a661d8d85 Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Sun, 4 Oct 2026 23:11:33 +0200 Subject: [PATCH 04/10] Add manifest parsing --- .gitignore | 1 + docs/applications.rst | 76 +++++++++++ tools/app_manifest.py | 251 ++++++++++++++++++++++++++++++++++++ tools/generate_app_layer.py | 6 +- 4 files changed, 331 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index e8f5548..1750477 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ /docs/_build/ **/.DS_Store +/.idea/ diff --git a/docs/applications.rst b/docs/applications.rst index de2ed25..f08adae 100644 --- a/docs/applications.rst +++ b/docs/applications.rst @@ -12,3 +12,79 @@ Build selection uses the manifest: Selection includes the bundle's required services. Applications start when selected from the launcher after unlock; services run independently. + +Manifest fields +--------------- + +The parser reads ``apps/*/mosaic-app.json`` in directory order and validates all +bundles, including disabled ones. Unknown fields and duplicate JSON keys are +errors. Each error identifies the manifest and affected field. + +The following top-level fields are required: + +- ``schema_version`` is the integer ``1``. +- ``id`` is unique across bundles and contains lowercase letters, digits, and + hyphens, starting with a letter or digit. +- ``name`` is a nonempty display name. +- ``version`` starts with a digit and contains letters, digits, dots, ``+``, or + ``~``. +- ``enabled`` and ``include_in_prod`` are booleans with the selection behavior + described above. +- ``license`` is a nonempty Yocto license identifier or expression. +- ``license_file`` points to the bundle's license file. +- ``app`` describes the application build and executable. + +``app`` requires ``source_dir`` and ``executable``. The source directory contains +``CMakeLists.txt``. The executable is an installed program name, starting with +a letter or digit and containing only letters, digits, dots, underscores, +hyphens, or ``+``. Arguments are supplied separately in the optional +``arguments`` string list, which defaults to ``[]``. Empty argument strings are +allowed, while NUL characters are rejected. + +Both ``app`` and each service accept optional ``build_dependencies`` and +``runtime_dependencies`` lists, defaulting to ``[]``. These contain Yocto recipe +names and runtime package names respectively. Names start with a lowercase letter +or digit and contain lowercase letters, digits, dots, hyphens, or ``+``. Entries +are plain names without version constraints or BitBake expressions. Provider +availability is checked by BitBake. + +``services`` is an optional list, defaulting to ``[]``. Each service requires: + +- ``id``, following the application ID format and unique within the bundle. +- ``source_dir``, containing the service's ``CMakeLists.txt``. +- ``unit``, a path or nonempty list of paths to systemd unit files. Unit filenames + must be unique across bundles and contain only letters, digits, dots, + underscores, ``@``, ``:``, ``+``, or hyphens. +- ``start``, currently accepting only ``"boot"``. + +All source, license, and unit paths are relative to the manifest's directory. +They must exist and stay within that directory after resolving symlinks. Text +fields other than arguments reject control characters. The parser checks file +presence, not CMake code, license validity, or systemd unit contents. + +``profiles`` is an optional object, defaulting to ``{}``, with optional ``dev`` +and ``prod`` entries. Each entry contains a required ``settings`` object holding +application-defined JSON values. The parser preserves these settings without +interpreting them. + +Example manifest +---------------- + +.. literalinclude:: ../apps/dummy-application/mosaic-app.json + :language: json + +Parser interface +---------------- + +``tools/app_manifest.py`` exposes ``load_manifest(path)`` for one bundle and +``load_manifests(apps_dir)`` for all bundles. They return dictionaries with +optional defaults filled in, source and file paths resolved to absolute +``Path`` objects, and ``unit`` normalized to a list. Each dictionary includes +``manifest_path`` for diagnostics. An empty application directory returns an +empty list. Invalid input raises ``ManifestError`` without writing files. + +Run the parser checks from the repository root: + +.. code-block:: bash + + python3 -B -m tools.test_app_manifest diff --git a/tools/app_manifest.py b/tools/app_manifest.py index e69de29..81d02c5 100644 --- a/tools/app_manifest.py +++ b/tools/app_manifest.py @@ -0,0 +1,251 @@ +"""Read and validate application bundles without generating files""" + +import json +import re +from pathlib import Path +from typing import NoReturn, TypedDict, cast + + +class Component(TypedDict): + source_dir: Path + build_dependencies: list[str] + runtime_dependencies: list[str] + + +class Application(Component): + executable: str + arguments: list[str] + + +class Service(Component): + id: str + unit: list[Path] + start: str + + +class Profile(TypedDict): + settings: dict[str, object] + + +class Manifest(TypedDict): + schema_version: int + id: str + name: str + version: str + enabled: bool + include_in_prod: bool + license: str + license_file: Path + app: Application + services: list[Service] + profiles: dict[str, Profile] + manifest_path: Path + + +class ManifestError(ValueError): + """A manifest could not be read or contains an invalid field""" + + +def _unique_keys(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key {key!r}") + result[key] = value + return result + + +def _invalid_constant(value: str) -> NoReturn: + raise ValueError(f"invalid JSON value {value}") + + +def load_manifest(path: Path) -> Manifest: + """Return normalized data with absolute Paths and the manifest_path attached""" + path = Path(path) + + def fail(field: str, message: str) -> NoReturn: + raise ManifestError(f"{path}: {field}: {message}") + + def require(condition: bool, field: str, message: str) -> None: + if not condition: + fail(field, message) + + def object_fields(value: object, field: str) -> dict[str, object]: + if not isinstance(value, dict): + fail(field, "expected an object") + # JSON object keys are always strings + return cast(dict[str, object], value) + + def fields(value: object, field: str, required: set[str], optional: set[str]) -> dict[str, object]: + result = object_fields(value, field) + for key in sorted(required - result.keys()): + fail(f"{field}.{key}", "required field is missing") + for key in sorted(result.keys() - required - optional): + fail(f"{field}.{key}", "unknown field") + return result + + def text(value: object, field: str, pattern: str = "") -> str: + if not isinstance(value, str) or not value.strip(): + fail(field, "expected a nonempty string") + require(all(ord(char) >= 32 and ord(char) != 127 for char in value), + field, "control characters are not allowed") + if pattern: + require(re.fullmatch(pattern, value) is not None, field, + f"must match {pattern}") + return value + + def boolean(value: object, field: str) -> bool: + if not isinstance(value, bool): + fail(field, "expected a boolean") + return value + + def array(value: object, field: str) -> list[object]: + if not isinstance(value, list): + fail(field, "expected a list") + return cast(list[object], value) + + def strings(value: object, field: str) -> list[str]: + result: list[str] = [] + for index, item in enumerate(array(value, field)): + if not isinstance(item, str) or "\0" in item: + fail(f"{field}[{index}]", "expected a string without NUL characters") + result.append(item) + return result + + def bundle_path(value: object, field: str, directory: bool = False) -> Path: + relative = Path(text(value, field)) + require(not relative.is_absolute(), field, "expected a bundle-relative path") + try: + resolved = (path.parent / relative).resolve() + exists = resolved.is_dir() if directory else resolved.is_file() + except (OSError, RuntimeError, UnicodeError) as error: + raise ManifestError(f"{path}: {field}: {error}") from error + require(resolved.is_relative_to(path.parent), field, + "path must stay inside the application bundle") + require(exists, field, "directory does not exist" if directory else "file does not exist") + return resolved + + def component(value: dict[str, object], field: str) -> Component: + source_dir = bundle_path(value["source_dir"], f"{field}.source_dir", True) + require((source_dir / "CMakeLists.txt").is_file(), + f"{field}.source_dir", "missing CMakeLists.txt") + dependencies: dict[str, list[str]] = {} + for key in ("build_dependencies", "runtime_dependencies"): + dependencies[key] = strings(value.get(key, []), f"{field}.{key}") + for index, dependency in enumerate(dependencies[key]): + _ = text(dependency, f"{field}.{key}[{index}]", r"[a-z0-9][a-z0-9.+-]*") + return { + "source_dir": source_dir, + "build_dependencies": dependencies["build_dependencies"], + "runtime_dependencies": dependencies["runtime_dependencies"], + } + + try: + path = path.resolve() + raw = cast(object, json.loads(path.read_text(encoding="utf-8"), + object_pairs_hook=_unique_keys, parse_constant=_invalid_constant)) + except (OSError, RuntimeError, UnicodeError, ValueError) as error: + raise ManifestError(f"{path}: {error}") from error + + data = fields(raw, "manifest", + {"schema_version", "id", "name", "version", "enabled", "include_in_prod", + "license", "license_file", "app"}, {"services", "profiles"}) + require(isinstance(data["schema_version"], int) and data["schema_version"] == 1, + "schema_version", "expected integer 1") + app_id = text(data["id"], "id", r"[a-z0-9][a-z0-9-]*") + name = text(data["name"], "name") + version = text(data["version"], "version", r"[0-9][A-Za-z0-9.+~]*") + license_name = text(data["license"], "license") + enabled = boolean(data["enabled"], "enabled") + include_in_prod = boolean(data["include_in_prod"], "include_in_prod") + license_file = bundle_path(data["license_file"], "license_file") + + app_data = fields(data["app"], "app", {"source_dir", "executable"}, + {"arguments", "build_dependencies", "runtime_dependencies"}) + app: Application = { + **component(app_data, "app"), + "executable": text(app_data["executable"], "app.executable", r"[A-Za-z0-9][A-Za-z0-9._+-]*"), + "arguments": strings(app_data.get("arguments", []), "app.arguments"), + } + + services: list[Service] = [] + service_ids: set[str] = set() + unit_names: set[str] = set() + for index, value in enumerate(array(data.get("services", []), "services")): + field = f"services[{index}]" + service_data = fields(value, field, {"id", "source_dir", "unit", "start"}, + {"build_dependencies", "runtime_dependencies"}) + service_id = text(service_data["id"], f"{field}.id", r"[a-z0-9][a-z0-9-]*") + require(service_id not in service_ids, f"{field}.id", "duplicate service ID") + service_ids.add(service_id) + require(service_data["start"] == "boot", f"{field}.start", "expected 'boot'") + service: Service = { + **component(service_data, field), + "id": service_id, + "start": "boot", + "unit": [], + } + raw_units = service_data["unit"] + units = strings([raw_units] if isinstance(raw_units, str) else raw_units, f"{field}.unit") + require(bool(units), f"{field}.unit", "at least one unit file is required") + for unit_index, unit in enumerate(units): + unit_field = f"{field}.unit[{unit_index}]" + unit_path = bundle_path(unit, unit_field) + _ = text(unit_path.name, unit_field, r"[A-Za-z0-9_.@:+-]+") + require(unit_path.name not in unit_names, unit_field, "duplicate unit filename") + unit_names.add(unit_path.name) + service["unit"].append(unit_path) + services.append(service) + + profiles: dict[str, Profile] = {} + profile_data = fields(data.get("profiles", {}), "profiles", set(), {"dev", "prod"}) + for profile_name, value in profile_data.items(): + profile = fields(value, f"profiles.{profile_name}", {"settings"}, set()) + profiles[profile_name] = { + "settings": object_fields(profile["settings"], f"profiles.{profile_name}.settings"), + } + + return { + "schema_version": 1, + "id": app_id, + "name": name, + "version": version, + "enabled": enabled, + "include_in_prod": include_in_prod, + "license": license_name, + "license_file": license_file, + "app": app, + "services": services, + "profiles": profiles, + "manifest_path": path, + } + + +def load_manifests(apps_dir: Path) -> list[Manifest]: + """Load all bundles in directory order, including disabled bundles""" + try: + apps_dir = Path(apps_dir).resolve() + except (OSError, RuntimeError, UnicodeError) as error: + raise ManifestError(f"{apps_dir}: apps_dir: {error}") from error + if not apps_dir.is_dir(): + raise ManifestError(f"{apps_dir}: apps_dir: directory does not exist") + + manifests: list[Manifest] = [] + app_ids: dict[str, Path] = {} + unit_names: dict[str, Path] = {} + for path in sorted(apps_dir.glob("*/mosaic-app.json")): + manifest = load_manifest(path) + app_id = manifest["id"] + if app_id in app_ids: + raise ManifestError(f"{path}: id: {app_id!r} is already declared in {app_ids[app_id]}") + app_ids[app_id] = path + for index, service in enumerate(manifest["services"]): + for unit in service["unit"]: + if unit.name in unit_names: + raise ManifestError( + f"{path}: services[{index}].unit: {unit.name!r} " + + f"is already declared in {unit_names[unit.name]}" + ) + unit_names[unit.name] = path + manifests.append(manifest) + return manifests diff --git a/tools/generate_app_layer.py b/tools/generate_app_layer.py index 2f5a174..4e995bf 100644 --- a/tools/generate_app_layer.py +++ b/tools/generate_app_layer.py @@ -6,19 +6,19 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( + _ = parser.add_argument( "--apps-dir", type=Path, required=True, help="Directory containing application bundles", ) - parser.add_argument( + _ = parser.add_argument( "--output", type=Path, required=True, help="Destination for the generated BitBake layer", ) - parser.parse_args() + _ = parser.parse_args() # TODO: Read and validate manifests with app_manifest before writing output # TODO: Generate the layer from the validated data with app_layer From 66097da474a67c21641a6c7e62509ad788a41cda Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Sun, 4 Oct 2026 23:12:11 +0200 Subject: [PATCH 05/10] Add test for manifest parsing --- tools/test_app_manifest.py | 174 +++++++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 tools/test_app_manifest.py diff --git a/tools/test_app_manifest.py b/tools/test_app_manifest.py new file mode 100644 index 0000000..0a3f288 --- /dev/null +++ b/tools/test_app_manifest.py @@ -0,0 +1,174 @@ +"""Run with python3 -B -m tools.test_app_manifest.""" + +import copy +import json +import shutil +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import cast + +from tools.app_manifest import ManifestError, load_manifest, load_manifests + + +def get_value(value: object, keys: tuple[str | int, ...]) -> object: + for key in keys: + if isinstance(key, str): + value = cast(dict[str, object], value)[key] + else: + value = cast(list[object], value)[key] + return value + + +def set_value(data: dict[str, object], keys: tuple[str | int, ...], value: object) -> None: + target = get_value(data, keys[:-1]) + key = keys[-1] + if isinstance(key, str): + cast(dict[str, object], target)[key] = value + else: + cast(list[object], target)[key] = value + + +def main() -> None: + example = Path(__file__).resolve().parents[1] / "apps/dummy-application" + with TemporaryDirectory() as temporary: + apps = Path(temporary).resolve() / "apps" + bundle = apps / "first" + _ = shutil.copytree(example, bundle) + path = bundle / "mosaic-app.json" + original = path.read_text() + base = cast(dict[str, object], json.loads(original)) + + def write(data: dict[str, object]) -> None: + _ = path.write_text(json.dumps(data), encoding="utf-8") + + def rejected(field: str, multiple: bool = False) -> None: + try: + _ = load_manifests(apps) if multiple else load_manifest(path) + except ManifestError as error: + assert "mosaic-app.json" in str(error), str(error) + assert field in str(error), str(error) + else: + raise AssertionError(f"accepted invalid {field}") + + manifest = load_manifest(path) + assert manifest["id"] == "dummy-application" + assert manifest["app"]["source_dir"] == bundle / "app" + assert manifest["services"][0]["unit"][0].is_absolute() + assert manifest["app"]["build_dependencies"] == [] + assert manifest["profiles"] == base["profiles"] + assert path.read_text() == original + + valid = copy.deepcopy(base) + set_value(valid, ("app", "build_dependencies"), ["qtdeclarative-native", "qtbase"]) + set_value(valid, ("app", "runtime_dependencies"), ["qtbase-plugins"]) + set_value(valid, ("app", "arguments"), ["--label", "two words", ""]) + set_value(valid, ("services", 0, "runtime_dependencies"), ["bash"]) + timer = bundle / "telemetry.timer" + _ = timer.write_text("[Timer]\nOnBootSec=10\n", encoding="utf-8") + set_value(valid, ("services", 0, "unit"), [get_value(base, ("services", 0, "unit")), timer.name]) + write(valid) + manifest = load_manifest(path) + assert len(manifest["services"][0]["unit"]) == 2 + assert manifest["app"]["build_dependencies"] == ["qtdeclarative-native", "qtbase"] + assert manifest["app"]["arguments"] == ["--label", "two words", ""] + + cases: list[tuple[tuple[str | int, ...], object, str]] = [ + (("schema_version",), True, "schema_version"), + (("schema_version",), 2, "schema_version"), + (("enabled",), 1, "enabled"), + (("id",), "../app", "id"), + (("version",), "0.1/escape", "version"), + (("license_file",), "missing", "license_file"), + (("license_file",), "../outside", "license_file"), + (("app",), [], "app"), + (("app", "source_dir"), "missing", "app.source_dir"), + (("app", "source_dir"), "tests", "app.source_dir"), + (("app", "source_dir"), str(bundle / "app"), "app.source_dir"), + (("app", "executable"), "app --flag", "app.executable"), + (("app", "arguments"), "--flag", "app.arguments"), + (("app", "arguments"), [False], "app.arguments[0]"), + (("app", "build_dependencies"), ["${INJECT}"], "app.build_dependencies[0]"), + (("app", "runtime_dependencies"), "bash", "app.runtime_dependencies"), + (("services",), {}, "services"), + (("services", 0, "start"), "app", "services[0].start"), + (("services", 0, "unit"), [], "services[0].unit"), + (("services", 0, "unit"), "missing.service", "services[0].unit[0]"), + (("profiles", "dev", "settings"), [], "profiles.dev.settings"), + (("unknown",), True, "manifest.unknown"), + ] + for keys, value, field in cases: + data = copy.deepcopy(base) + set_value(data, keys, value) + write(data) + rejected(field) + + for content, field in [("{", "line"), ("[]", "manifest"), + ('{"id": 1, "id": 2}', "duplicate JSON key"), + ('{"value": NaN}', "invalid JSON value")]: + _ = path.write_text(content, encoding="utf-8") + rejected(field) + + data = copy.deepcopy(base) + del data["name"] + write(data) + rejected("manifest.name") + data = copy.deepcopy(base) + service = get_value(base, ("services", 0)) + data["services"] = [copy.deepcopy(service), copy.deepcopy(service)] + write(data) + rejected("duplicate service ID") + set_value(data, ("services", 1, "id"), "other") + write(data) + rejected("duplicate unit filename") + + (bundle / "outside").symlink_to(Path(temporary), target_is_directory=True) + data = copy.deepcopy(base) + set_value(data, ("app", "source_dir"), "outside") + write(data) + rejected("path must stay inside") + + (bundle / "loop").symlink_to("loop") + set_value(data, ("app", "source_dir"), "loop") + write(data) + rejected("app.source_dir") + + minimal = copy.deepcopy(base) + for key in ("services", "profiles"): + del minimal[key] + del cast(dict[str, object], minimal["app"])["arguments"] + minimal["enabled"] = False + write(minimal) + assert load_manifest(path)["services"] == [] + assert load_manifests(apps)[0]["enabled"] is False + set_value(minimal, ("app", "source_dir"), "missing") + write(minimal) + rejected("app.source_dir", multiple=True) + + write(base) + second = apps / "second" + _ = shutil.copytree(example, second) + rejected("already declared", multiple=True) + second_data = copy.deepcopy(base) + second_data["id"] = "second-app" + second_path = second / "mosaic-app.json" + _ = second_path.write_text(json.dumps(second_data), encoding="utf-8") + rejected("services[0].unit", multiple=True) + second_data["services"] = [] + _ = second_path.write_text(json.dumps(second_data), encoding="utf-8") + assert [item["id"] for item in load_manifests(apps)] == ["dummy-application", "second-app"] + + empty = Path(temporary) / "empty" + empty.mkdir() + assert load_manifests(empty) == [] + try: + _ = load_manifests(empty / "missing") + except ManifestError as error: + assert "apps_dir" in str(error) + else: + raise AssertionError("accepted missing apps directory") + + print("Manifest checks passed") + + +if __name__ == "__main__": + main() From ffae825f4dc81e5f9fdf23d9d5e009046047e00b Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Sun, 4 Oct 2026 23:20:35 +0200 Subject: [PATCH 06/10] Use pydantic as json parsing --- .dockerignore | 2 + .gitignore | 1 + Dockerfile | 9 +- docs/app-generation.rst | 4 +- docs/applications.rst | 8 +- docs/building.rst | 8 + tools/app_manifest.py | 303 +++++++++++++++---------------------- tools/requirements.txt | 1 + tools/test_app_manifest.py | 30 ++-- 9 files changed, 164 insertions(+), 202 deletions(-) create mode 100644 tools/requirements.txt diff --git a/.dockerignore b/.dockerignore index f27a865..cd79285 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ # Sources are mounted at runtime. The image only contains host build tools. ** !Dockerfile +!tools/ +!tools/requirements.txt diff --git a/.gitignore b/.gitignore index 1750477..47f1fb8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /.env /.venv-docs/ +/.venv-tools/ /docs/_build/ **/.DS_Store diff --git a/Dockerfile b/Dockerfile index 385827e..5fba745 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,11 +9,18 @@ RUN apt-get update \ build-essential ca-certificates chrpath cpio debianutils diffstat file \ gawk gcc git iputils-ping libacl1 libcrypt-dev locales openssh-client \ python3 python3-git python3-jinja2 python3-pexpect python3-pip \ - python3-subunit python3-websockets socat texinfo unzip wget xz-utils zstd \ + python3-subunit python3-venv python3-websockets socat texinfo unzip wget xz-utils zstd \ && rm -rf /var/lib/apt/lists/* \ && echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen \ && locale-gen +COPY tools/requirements.txt /tmp/mosaic-tools-requirements.txt +RUN python3 -m venv --system-site-packages /opt/mosaic-tools \ + && /opt/mosaic-tools/bin/python -m pip install --no-cache-dir \ + -r /tmp/mosaic-tools-requirements.txt + +ENV PATH="/opt/mosaic-tools/bin:${PATH}" + RUN if ! getent group "${HOST_GID}" > /dev/null; then \ groupadd --gid "${HOST_GID}" builder; \ fi \ diff --git a/docs/app-generation.rst b/docs/app-generation.rst index 3e068a5..e6f6248 100644 --- a/docs/app-generation.rst +++ b/docs/app-generation.rst @@ -55,8 +55,8 @@ owns discovery and startup of installed applications. Tool files ---------- -The tool uses Python's standard library and consists of three files under -``tools/``: +The tool uses Pydantic for manifest validation and Python's standard library for +file handling. It consists of three files under ``tools/``: ``generate_app_layer.py`` Accepts ``--apps-dir`` and ``--output``, coordinates parsing and generation, diff --git a/docs/applications.rst b/docs/applications.rst index f08adae..b4d4be0 100644 --- a/docs/applications.rst +++ b/docs/applications.rst @@ -77,13 +77,15 @@ Parser interface ---------------- ``tools/app_manifest.py`` exposes ``load_manifest(path)`` for one bundle and -``load_manifests(apps_dir)`` for all bundles. They return dictionaries with +``load_manifests(apps_dir)`` for all bundles. They return typed Pydantic models with optional defaults filled in, source and file paths resolved to absolute -``Path`` objects, and ``unit`` normalized to a list. Each dictionary includes +``Path`` objects, and ``unit`` normalized to a list. Fields are accessed as +attributes, such as ``manifest.app.executable``. Each model exposes ``manifest_path`` for diagnostics. An empty application directory returns an empty list. Invalid input raises ``ManifestError`` without writing files. -Run the parser checks from the repository root: +Run the parser checks from the repository root in the build tools environment +described in :doc:`building`: .. code-block:: bash diff --git a/docs/building.rst b/docs/building.rst index 5c04fae..fd79026 100644 --- a/docs/building.rst +++ b/docs/building.rst @@ -77,8 +77,16 @@ installed. From the repository root, in Bash or Zsh: .. code-block:: bash + python3 -m venv --system-site-packages .venv-tools + .venv-tools/bin/python -m pip install -r tools/requirements.txt + source .venv-tools/bin/activate source ./setup-build-env +The tools environment supplies Pydantic for manifest validation and retains +access to the host's Yocto Python dependencies. Activate it in each new build +shell before sourcing ``setup-build-env``. The Docker builder includes this +environment automatically. + The wrapper initializes submodules using ``git submodule update --init --recursive``, selects ``yocto/layers/meta-mosaic-product/conf/templates/mosaic``, and sources OpenEmbedded-Core's ``oe-init-build-env``. It creates ``build/`` at the repository diff --git a/tools/app_manifest.py b/tools/app_manifest.py index 81d02c5..02c98d2 100644 --- a/tools/app_manifest.py +++ b/tools/app_manifest.py @@ -3,47 +3,126 @@ import json import re from pathlib import Path -from typing import NoReturn, TypedDict, cast +from typing import Annotated, ClassVar, Literal, NoReturn, Self, cast +from pydantic import ( + AfterValidator, BaseModel, BeforeValidator, ConfigDict, DirectoryPath, Field, + FilePath, PrivateAttr, ValidationError, ValidationInfo, field_validator, + model_validator, +) -class Component(TypedDict): - source_dir: Path - build_dependencies: list[str] - runtime_dependencies: list[str] + +class ManifestError(ValueError): + """A manifest could not be read or contains an invalid field""" + + +def _text(value: str) -> str: + if not value.strip(): + raise ValueError("expected a nonempty string") + if any(ord(char) < 32 or ord(char) == 127 for char in value): + raise ValueError("control characters are not allowed") + return value + + +def _bundle_path(value: object, info: ValidationInfo) -> Path: + if not isinstance(value, str): + raise ValueError("expected a bundle-relative path string") + relative = Path(_text(value)) + if relative.is_absolute(): + raise ValueError("expected a bundle-relative path") + manifest_path = cast(Path, info.context) + try: + resolved = (manifest_path.parent / relative).resolve() + except (OSError, RuntimeError, UnicodeError) as error: + raise ValueError(str(error)) from error + if not resolved.is_relative_to(manifest_path.parent): + raise ValueError("path must stay inside the application bundle") + return resolved + + +def _cmake_directory(value: Path) -> Path: + if not (value / "CMakeLists.txt").is_file(): + raise ValueError("missing CMakeLists.txt") + return value + + +def _unit_file(value: Path) -> Path: + if not re.fullmatch(r"[A-Za-z0-9_.@:+-]+", value.name): + raise ValueError("invalid unit filename") + return value + + +Text = Annotated[str, AfterValidator(_text)] +Identifier = Annotated[Text, Field(pattern=r"^[a-z0-9][a-z0-9-]*$")] +Dependency = Annotated[Text, Field(pattern=r"^[a-z0-9][a-z0-9.+-]*$")] +Argument = Annotated[str, Field(pattern=r"^[^\x00]*$")] +BundleFile = Annotated[FilePath, BeforeValidator(_bundle_path)] +SourceDirectory = Annotated[DirectoryPath, BeforeValidator(_bundle_path), AfterValidator(_cmake_directory)] +UnitFile = Annotated[BundleFile, AfterValidator(_unit_file)] + + +class ManifestModel(BaseModel): + model_config: ClassVar[ConfigDict] = ConfigDict(strict=True, extra="forbid") + + +class Component(ManifestModel): + source_dir: SourceDirectory + build_dependencies: list[Dependency] = Field(default_factory=list) + runtime_dependencies: list[Dependency] = Field(default_factory=list) class Application(Component): - executable: str - arguments: list[str] + executable: Annotated[Text, Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._+-]*$")] + arguments: list[Argument] = Field(default_factory=list) class Service(Component): - id: str - unit: list[Path] - start: str + id: Identifier + unit: Annotated[list[UnitFile], Field(min_length=1)] + start: Literal["boot"] + + @field_validator("unit", mode="before") + @classmethod + def unit_list(cls, value: object) -> object: + return [value] if isinstance(value, str) else value -class Profile(TypedDict): +class Profile(ManifestModel): settings: dict[str, object] -class Manifest(TypedDict): - schema_version: int - id: str - name: str - version: str +class Manifest(ManifestModel): + schema_version: Annotated[int, Field(ge=1, le=1)] + id: Identifier + name: Text + version: Annotated[Text, Field(pattern=r"^[0-9][A-Za-z0-9.+~]*$")] enabled: bool include_in_prod: bool - license: str - license_file: Path + license: Text + license_file: BundleFile app: Application - services: list[Service] - profiles: dict[str, Profile] - manifest_path: Path - - -class ManifestError(ValueError): - """A manifest could not be read or contains an invalid field""" + services: list[Service] = Field(default_factory=list) + profiles: dict[Literal["dev", "prod"], Profile] = Field(default_factory=dict) + _manifest_path: Path = PrivateAttr() + + @property + def manifest_path(self) -> Path: + return self._manifest_path + + @model_validator(mode="after") + def check_services(self, info: ValidationInfo) -> Self: + self._manifest_path = cast(Path, info.context) + service_ids: set[str] = set() + unit_names: set[str] = set() + for index, service in enumerate(self.services): + if service.id in service_ids: + raise ValueError(f"services[{index}].id: duplicate service ID") + service_ids.add(service.id) + for unit in service.unit: + if unit.name in unit_names: + raise ValueError(f"services[{index}].unit: duplicate unit filename") + unit_names.add(unit.name) + return self def _unique_keys(pairs: list[tuple[str, object]]) -> dict[str, object]: @@ -60,171 +139,26 @@ def _invalid_constant(value: str) -> NoReturn: def load_manifest(path: Path) -> Manifest: - """Return normalized data with absolute Paths and the manifest_path attached""" - path = Path(path) - - def fail(field: str, message: str) -> NoReturn: - raise ManifestError(f"{path}: {field}: {message}") - - def require(condition: bool, field: str, message: str) -> None: - if not condition: - fail(field, message) - - def object_fields(value: object, field: str) -> dict[str, object]: - if not isinstance(value, dict): - fail(field, "expected an object") - # JSON object keys are always strings - return cast(dict[str, object], value) - - def fields(value: object, field: str, required: set[str], optional: set[str]) -> dict[str, object]: - result = object_fields(value, field) - for key in sorted(required - result.keys()): - fail(f"{field}.{key}", "required field is missing") - for key in sorted(result.keys() - required - optional): - fail(f"{field}.{key}", "unknown field") - return result - - def text(value: object, field: str, pattern: str = "") -> str: - if not isinstance(value, str) or not value.strip(): - fail(field, "expected a nonempty string") - require(all(ord(char) >= 32 and ord(char) != 127 for char in value), - field, "control characters are not allowed") - if pattern: - require(re.fullmatch(pattern, value) is not None, field, - f"must match {pattern}") - return value - - def boolean(value: object, field: str) -> bool: - if not isinstance(value, bool): - fail(field, "expected a boolean") - return value - - def array(value: object, field: str) -> list[object]: - if not isinstance(value, list): - fail(field, "expected a list") - return cast(list[object], value) - - def strings(value: object, field: str) -> list[str]: - result: list[str] = [] - for index, item in enumerate(array(value, field)): - if not isinstance(item, str) or "\0" in item: - fail(f"{field}[{index}]", "expected a string without NUL characters") - result.append(item) - return result - - def bundle_path(value: object, field: str, directory: bool = False) -> Path: - relative = Path(text(value, field)) - require(not relative.is_absolute(), field, "expected a bundle-relative path") - try: - resolved = (path.parent / relative).resolve() - exists = resolved.is_dir() if directory else resolved.is_file() - except (OSError, RuntimeError, UnicodeError) as error: - raise ManifestError(f"{path}: {field}: {error}") from error - require(resolved.is_relative_to(path.parent), field, - "path must stay inside the application bundle") - require(exists, field, "directory does not exist" if directory else "file does not exist") - return resolved - - def component(value: dict[str, object], field: str) -> Component: - source_dir = bundle_path(value["source_dir"], f"{field}.source_dir", True) - require((source_dir / "CMakeLists.txt").is_file(), - f"{field}.source_dir", "missing CMakeLists.txt") - dependencies: dict[str, list[str]] = {} - for key in ("build_dependencies", "runtime_dependencies"): - dependencies[key] = strings(value.get(key, []), f"{field}.{key}") - for index, dependency in enumerate(dependencies[key]): - _ = text(dependency, f"{field}.{key}[{index}]", r"[a-z0-9][a-z0-9.+-]*") - return { - "source_dir": source_dir, - "build_dependencies": dependencies["build_dependencies"], - "runtime_dependencies": dependencies["runtime_dependencies"], - } - + """Return a validated model with absolute paths and its manifest location""" try: path = path.resolve() raw = cast(object, json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=_unique_keys, parse_constant=_invalid_constant)) + return Manifest.model_validate(raw, context=path) + except ValidationError as error: + detail = error.errors(include_url=False, include_input=False, include_context=False)[0] + field = "manifest" + "".join( + f"[{part}]" if isinstance(part, int) else f".{part}" for part in detail["loc"] + ) + raise ManifestError(f"{path}: {field}: {detail['msg']}") from error except (OSError, RuntimeError, UnicodeError, ValueError) as error: raise ManifestError(f"{path}: {error}") from error - data = fields(raw, "manifest", - {"schema_version", "id", "name", "version", "enabled", "include_in_prod", - "license", "license_file", "app"}, {"services", "profiles"}) - require(isinstance(data["schema_version"], int) and data["schema_version"] == 1, - "schema_version", "expected integer 1") - app_id = text(data["id"], "id", r"[a-z0-9][a-z0-9-]*") - name = text(data["name"], "name") - version = text(data["version"], "version", r"[0-9][A-Za-z0-9.+~]*") - license_name = text(data["license"], "license") - enabled = boolean(data["enabled"], "enabled") - include_in_prod = boolean(data["include_in_prod"], "include_in_prod") - license_file = bundle_path(data["license_file"], "license_file") - - app_data = fields(data["app"], "app", {"source_dir", "executable"}, - {"arguments", "build_dependencies", "runtime_dependencies"}) - app: Application = { - **component(app_data, "app"), - "executable": text(app_data["executable"], "app.executable", r"[A-Za-z0-9][A-Za-z0-9._+-]*"), - "arguments": strings(app_data.get("arguments", []), "app.arguments"), - } - - services: list[Service] = [] - service_ids: set[str] = set() - unit_names: set[str] = set() - for index, value in enumerate(array(data.get("services", []), "services")): - field = f"services[{index}]" - service_data = fields(value, field, {"id", "source_dir", "unit", "start"}, - {"build_dependencies", "runtime_dependencies"}) - service_id = text(service_data["id"], f"{field}.id", r"[a-z0-9][a-z0-9-]*") - require(service_id not in service_ids, f"{field}.id", "duplicate service ID") - service_ids.add(service_id) - require(service_data["start"] == "boot", f"{field}.start", "expected 'boot'") - service: Service = { - **component(service_data, field), - "id": service_id, - "start": "boot", - "unit": [], - } - raw_units = service_data["unit"] - units = strings([raw_units] if isinstance(raw_units, str) else raw_units, f"{field}.unit") - require(bool(units), f"{field}.unit", "at least one unit file is required") - for unit_index, unit in enumerate(units): - unit_field = f"{field}.unit[{unit_index}]" - unit_path = bundle_path(unit, unit_field) - _ = text(unit_path.name, unit_field, r"[A-Za-z0-9_.@:+-]+") - require(unit_path.name not in unit_names, unit_field, "duplicate unit filename") - unit_names.add(unit_path.name) - service["unit"].append(unit_path) - services.append(service) - - profiles: dict[str, Profile] = {} - profile_data = fields(data.get("profiles", {}), "profiles", set(), {"dev", "prod"}) - for profile_name, value in profile_data.items(): - profile = fields(value, f"profiles.{profile_name}", {"settings"}, set()) - profiles[profile_name] = { - "settings": object_fields(profile["settings"], f"profiles.{profile_name}.settings"), - } - - return { - "schema_version": 1, - "id": app_id, - "name": name, - "version": version, - "enabled": enabled, - "include_in_prod": include_in_prod, - "license": license_name, - "license_file": license_file, - "app": app, - "services": services, - "profiles": profiles, - "manifest_path": path, - } - def load_manifests(apps_dir: Path) -> list[Manifest]: """Load all bundles in directory order, including disabled bundles""" try: - apps_dir = Path(apps_dir).resolve() + apps_dir = apps_dir.resolve() except (OSError, RuntimeError, UnicodeError) as error: raise ManifestError(f"{apps_dir}: apps_dir: {error}") from error if not apps_dir.is_dir(): @@ -235,12 +169,11 @@ def load_manifests(apps_dir: Path) -> list[Manifest]: unit_names: dict[str, Path] = {} for path in sorted(apps_dir.glob("*/mosaic-app.json")): manifest = load_manifest(path) - app_id = manifest["id"] - if app_id in app_ids: - raise ManifestError(f"{path}: id: {app_id!r} is already declared in {app_ids[app_id]}") - app_ids[app_id] = path - for index, service in enumerate(manifest["services"]): - for unit in service["unit"]: + if manifest.id in app_ids: + raise ManifestError(f"{path}: id: {manifest.id!r} is already declared in {app_ids[manifest.id]}") + app_ids[manifest.id] = path + for index, service in enumerate(manifest.services): + for unit in service.unit: if unit.name in unit_names: raise ManifestError( f"{path}: services[{index}].unit: {unit.name!r} " diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 0000000..a22eda9 --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1 @@ +pydantic==2.13.5 diff --git a/tools/test_app_manifest.py b/tools/test_app_manifest.py index 0a3f288..01d2397 100644 --- a/tools/test_app_manifest.py +++ b/tools/test_app_manifest.py @@ -51,11 +51,12 @@ def rejected(field: str, multiple: bool = False) -> None: raise AssertionError(f"accepted invalid {field}") manifest = load_manifest(path) - assert manifest["id"] == "dummy-application" - assert manifest["app"]["source_dir"] == bundle / "app" - assert manifest["services"][0]["unit"][0].is_absolute() - assert manifest["app"]["build_dependencies"] == [] - assert manifest["profiles"] == base["profiles"] + assert manifest.id == "dummy-application" + assert manifest.app.source_dir == bundle / "app" + assert manifest.services[0].unit[0].is_absolute() + assert manifest.app.build_dependencies == [] + assert manifest.manifest_path == path + assert {name: {"settings": profile.settings} for name, profile in manifest.profiles.items()} == base["profiles"] assert path.read_text() == original valid = copy.deepcopy(base) @@ -68,14 +69,17 @@ def rejected(field: str, multiple: bool = False) -> None: set_value(valid, ("services", 0, "unit"), [get_value(base, ("services", 0, "unit")), timer.name]) write(valid) manifest = load_manifest(path) - assert len(manifest["services"][0]["unit"]) == 2 - assert manifest["app"]["build_dependencies"] == ["qtdeclarative-native", "qtbase"] - assert manifest["app"]["arguments"] == ["--label", "two words", ""] + assert len(manifest.services[0].unit) == 2 + assert manifest.app.build_dependencies == ["qtdeclarative-native", "qtbase"] + assert manifest.app.arguments == ["--label", "two words", ""] cases: list[tuple[tuple[str | int, ...], object, str]] = [ (("schema_version",), True, "schema_version"), + (("schema_version",), 1.0, "schema_version"), + (("schema_version",), "1", "schema_version"), (("schema_version",), 2, "schema_version"), (("enabled",), 1, "enabled"), + (("enabled",), "true", "enabled"), (("id",), "../app", "id"), (("version",), "0.1/escape", "version"), (("license_file",), "missing", "license_file"), @@ -87,6 +91,8 @@ def rejected(field: str, multiple: bool = False) -> None: (("app", "executable"), "app --flag", "app.executable"), (("app", "arguments"), "--flag", "app.arguments"), (("app", "arguments"), [False], "app.arguments[0]"), + (("app", "arguments"), ["bad\0argument"], "app.arguments[0]"), + (("app", "unknown"), True, "app.unknown"), (("app", "build_dependencies"), ["${INJECT}"], "app.build_dependencies[0]"), (("app", "runtime_dependencies"), "bash", "app.runtime_dependencies"), (("services",), {}, "services"), @@ -94,6 +100,8 @@ def rejected(field: str, multiple: bool = False) -> None: (("services", 0, "unit"), [], "services[0].unit"), (("services", 0, "unit"), "missing.service", "services[0].unit[0]"), (("profiles", "dev", "settings"), [], "profiles.dev.settings"), + (("profiles", "unknown"), {}, "profiles.unknown"), + (("manifest_path",), "supplied", "manifest.manifest_path"), (("unknown",), True, "manifest.unknown"), ] for keys, value, field in cases: @@ -138,8 +146,8 @@ def rejected(field: str, multiple: bool = False) -> None: del cast(dict[str, object], minimal["app"])["arguments"] minimal["enabled"] = False write(minimal) - assert load_manifest(path)["services"] == [] - assert load_manifests(apps)[0]["enabled"] is False + assert load_manifest(path).services == [] + assert load_manifests(apps)[0].enabled is False set_value(minimal, ("app", "source_dir"), "missing") write(minimal) rejected("app.source_dir", multiple=True) @@ -155,7 +163,7 @@ def rejected(field: str, multiple: bool = False) -> None: rejected("services[0].unit", multiple=True) second_data["services"] = [] _ = second_path.write_text(json.dumps(second_data), encoding="utf-8") - assert [item["id"] for item in load_manifests(apps)] == ["dummy-application", "second-app"] + assert [item.id for item in load_manifests(apps)] == ["dummy-application", "second-app"] empty = Path(temporary) / "empty" empty.mkdir() From 4f4885eeb543208931e7d0ff129d469139c0c6c3 Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Sun, 4 Oct 2026 23:26:54 +0200 Subject: [PATCH 07/10] Move to using pytest --- .gitignore | 2 + docs/applications.rst | 12 +- pytest.ini | 3 + tools/requirements-dev.txt | 2 + tools/test_app_manifest.py | 182 ------------------------- tools/tests/test_app_manifest.py | 226 +++++++++++++++++++++++++++++++ 6 files changed, 242 insertions(+), 185 deletions(-) create mode 100644 pytest.ini create mode 100644 tools/requirements-dev.txt delete mode 100644 tools/test_app_manifest.py create mode 100644 tools/tests/test_app_manifest.py diff --git a/.gitignore b/.gitignore index 47f1fb8..055908a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ /.venv-docs/ /.venv-tools/ /docs/_build/ +/.pytest_cache/ +**/__pycache__/ **/.DS_Store /.idea/ diff --git a/docs/applications.rst b/docs/applications.rst index b4d4be0..e27ff3d 100644 --- a/docs/applications.rst +++ b/docs/applications.rst @@ -84,9 +84,15 @@ attributes, such as ``manifest.app.executable``. Each model exposes ``manifest_path`` for diagnostics. An empty application directory returns an empty list. Invalid input raises ``ManifestError`` without writing files. -Run the parser checks from the repository root in the build tools environment -described in :doc:`building`: +Install the development dependencies once in the build tools environment +described in :doc:`building`, then run all Python tooling tests from the +repository root: .. code-block:: bash - python3 -B -m tools.test_app_manifest + .venv-tools/bin/python -m pip install -r tools/requirements-dev.txt + .venv-tools/bin/python -m pytest + +Pytest discovers tests under ``tools/tests/``. It does not scan upstream Yocto +sources. To run only the manifest tests, append +``tools/tests/test_app_manifest.py`` to the pytest command. diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..b58eece --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +testpaths = tools/tests +addopts = -q diff --git a/tools/requirements-dev.txt b/tools/requirements-dev.txt new file mode 100644 index 0000000..1441c92 --- /dev/null +++ b/tools/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest==9.1.1 diff --git a/tools/test_app_manifest.py b/tools/test_app_manifest.py deleted file mode 100644 index 01d2397..0000000 --- a/tools/test_app_manifest.py +++ /dev/null @@ -1,182 +0,0 @@ -"""Run with python3 -B -m tools.test_app_manifest.""" - -import copy -import json -import shutil -from pathlib import Path -from tempfile import TemporaryDirectory -from typing import cast - -from tools.app_manifest import ManifestError, load_manifest, load_manifests - - -def get_value(value: object, keys: tuple[str | int, ...]) -> object: - for key in keys: - if isinstance(key, str): - value = cast(dict[str, object], value)[key] - else: - value = cast(list[object], value)[key] - return value - - -def set_value(data: dict[str, object], keys: tuple[str | int, ...], value: object) -> None: - target = get_value(data, keys[:-1]) - key = keys[-1] - if isinstance(key, str): - cast(dict[str, object], target)[key] = value - else: - cast(list[object], target)[key] = value - - -def main() -> None: - example = Path(__file__).resolve().parents[1] / "apps/dummy-application" - with TemporaryDirectory() as temporary: - apps = Path(temporary).resolve() / "apps" - bundle = apps / "first" - _ = shutil.copytree(example, bundle) - path = bundle / "mosaic-app.json" - original = path.read_text() - base = cast(dict[str, object], json.loads(original)) - - def write(data: dict[str, object]) -> None: - _ = path.write_text(json.dumps(data), encoding="utf-8") - - def rejected(field: str, multiple: bool = False) -> None: - try: - _ = load_manifests(apps) if multiple else load_manifest(path) - except ManifestError as error: - assert "mosaic-app.json" in str(error), str(error) - assert field in str(error), str(error) - else: - raise AssertionError(f"accepted invalid {field}") - - manifest = load_manifest(path) - assert manifest.id == "dummy-application" - assert manifest.app.source_dir == bundle / "app" - assert manifest.services[0].unit[0].is_absolute() - assert manifest.app.build_dependencies == [] - assert manifest.manifest_path == path - assert {name: {"settings": profile.settings} for name, profile in manifest.profiles.items()} == base["profiles"] - assert path.read_text() == original - - valid = copy.deepcopy(base) - set_value(valid, ("app", "build_dependencies"), ["qtdeclarative-native", "qtbase"]) - set_value(valid, ("app", "runtime_dependencies"), ["qtbase-plugins"]) - set_value(valid, ("app", "arguments"), ["--label", "two words", ""]) - set_value(valid, ("services", 0, "runtime_dependencies"), ["bash"]) - timer = bundle / "telemetry.timer" - _ = timer.write_text("[Timer]\nOnBootSec=10\n", encoding="utf-8") - set_value(valid, ("services", 0, "unit"), [get_value(base, ("services", 0, "unit")), timer.name]) - write(valid) - manifest = load_manifest(path) - assert len(manifest.services[0].unit) == 2 - assert manifest.app.build_dependencies == ["qtdeclarative-native", "qtbase"] - assert manifest.app.arguments == ["--label", "two words", ""] - - cases: list[tuple[tuple[str | int, ...], object, str]] = [ - (("schema_version",), True, "schema_version"), - (("schema_version",), 1.0, "schema_version"), - (("schema_version",), "1", "schema_version"), - (("schema_version",), 2, "schema_version"), - (("enabled",), 1, "enabled"), - (("enabled",), "true", "enabled"), - (("id",), "../app", "id"), - (("version",), "0.1/escape", "version"), - (("license_file",), "missing", "license_file"), - (("license_file",), "../outside", "license_file"), - (("app",), [], "app"), - (("app", "source_dir"), "missing", "app.source_dir"), - (("app", "source_dir"), "tests", "app.source_dir"), - (("app", "source_dir"), str(bundle / "app"), "app.source_dir"), - (("app", "executable"), "app --flag", "app.executable"), - (("app", "arguments"), "--flag", "app.arguments"), - (("app", "arguments"), [False], "app.arguments[0]"), - (("app", "arguments"), ["bad\0argument"], "app.arguments[0]"), - (("app", "unknown"), True, "app.unknown"), - (("app", "build_dependencies"), ["${INJECT}"], "app.build_dependencies[0]"), - (("app", "runtime_dependencies"), "bash", "app.runtime_dependencies"), - (("services",), {}, "services"), - (("services", 0, "start"), "app", "services[0].start"), - (("services", 0, "unit"), [], "services[0].unit"), - (("services", 0, "unit"), "missing.service", "services[0].unit[0]"), - (("profiles", "dev", "settings"), [], "profiles.dev.settings"), - (("profiles", "unknown"), {}, "profiles.unknown"), - (("manifest_path",), "supplied", "manifest.manifest_path"), - (("unknown",), True, "manifest.unknown"), - ] - for keys, value, field in cases: - data = copy.deepcopy(base) - set_value(data, keys, value) - write(data) - rejected(field) - - for content, field in [("{", "line"), ("[]", "manifest"), - ('{"id": 1, "id": 2}', "duplicate JSON key"), - ('{"value": NaN}', "invalid JSON value")]: - _ = path.write_text(content, encoding="utf-8") - rejected(field) - - data = copy.deepcopy(base) - del data["name"] - write(data) - rejected("manifest.name") - data = copy.deepcopy(base) - service = get_value(base, ("services", 0)) - data["services"] = [copy.deepcopy(service), copy.deepcopy(service)] - write(data) - rejected("duplicate service ID") - set_value(data, ("services", 1, "id"), "other") - write(data) - rejected("duplicate unit filename") - - (bundle / "outside").symlink_to(Path(temporary), target_is_directory=True) - data = copy.deepcopy(base) - set_value(data, ("app", "source_dir"), "outside") - write(data) - rejected("path must stay inside") - - (bundle / "loop").symlink_to("loop") - set_value(data, ("app", "source_dir"), "loop") - write(data) - rejected("app.source_dir") - - minimal = copy.deepcopy(base) - for key in ("services", "profiles"): - del minimal[key] - del cast(dict[str, object], minimal["app"])["arguments"] - minimal["enabled"] = False - write(minimal) - assert load_manifest(path).services == [] - assert load_manifests(apps)[0].enabled is False - set_value(minimal, ("app", "source_dir"), "missing") - write(minimal) - rejected("app.source_dir", multiple=True) - - write(base) - second = apps / "second" - _ = shutil.copytree(example, second) - rejected("already declared", multiple=True) - second_data = copy.deepcopy(base) - second_data["id"] = "second-app" - second_path = second / "mosaic-app.json" - _ = second_path.write_text(json.dumps(second_data), encoding="utf-8") - rejected("services[0].unit", multiple=True) - second_data["services"] = [] - _ = second_path.write_text(json.dumps(second_data), encoding="utf-8") - assert [item.id for item in load_manifests(apps)] == ["dummy-application", "second-app"] - - empty = Path(temporary) / "empty" - empty.mkdir() - assert load_manifests(empty) == [] - try: - _ = load_manifests(empty / "missing") - except ManifestError as error: - assert "apps_dir" in str(error) - else: - raise AssertionError("accepted missing apps directory") - - print("Manifest checks passed") - - -if __name__ == "__main__": - main() diff --git a/tools/tests/test_app_manifest.py b/tools/tests/test_app_manifest.py new file mode 100644 index 0000000..f4127ea --- /dev/null +++ b/tools/tests/test_app_manifest.py @@ -0,0 +1,226 @@ +"""Manifest parsing, normalization, and validation checks""" + +import copy +import json +import re +import shutil +from pathlib import Path +from typing import cast + +import pytest + +from tools.app_manifest import ManifestError, load_manifest, load_manifests + + +@pytest.fixture +def manifest_path(tmp_path: Path) -> Path: + example = Path(__file__).resolve().parents[2] / "apps/dummy-application" + bundle = tmp_path.resolve() / "apps/first" + _ = shutil.copytree(example, bundle) + return bundle / "mosaic-app.json" + + +@pytest.fixture +def data(manifest_path: Path) -> dict[str, object]: + return cast(dict[str, object], json.loads(manifest_path.read_text(encoding="utf-8"))) + + +def get_value(value: object, keys: tuple[str | int, ...]) -> object: + for key in keys: + if isinstance(key, str): + value = cast(dict[str, object], value)[key] + else: + value = cast(list[object], value)[key] + return value + + +def set_value(data: dict[str, object], keys: tuple[str | int, ...], value: object) -> None: + target = get_value(data, keys[:-1]) + key = keys[-1] + if isinstance(key, str): + cast(dict[str, object], target)[key] = value + else: + cast(list[object], target)[key] = value + + +def write_manifest(path: Path, data: dict[str, object]) -> None: + _ = path.write_text(json.dumps(data), encoding="utf-8") + + +def test_load_example(manifest_path: Path, data: dict[str, object]) -> None: + original = manifest_path.read_text() + manifest = load_manifest(manifest_path) + assert manifest.id == "dummy-application" + assert manifest.app.source_dir == manifest_path.parent / "app" + assert manifest.services[0].unit[0].is_absolute() + assert manifest.app.build_dependencies == [] + assert manifest.manifest_path == manifest_path + assert {name: {"settings": profile.settings} for name, profile in manifest.profiles.items()} == data["profiles"] + assert manifest_path.read_text() == original + + +def test_dependencies_arguments_and_units(manifest_path: Path, data: dict[str, object]) -> None: + set_value(data, ("app", "build_dependencies"), ["qtdeclarative-native", "qtbase"]) + set_value(data, ("app", "runtime_dependencies"), ["qtbase-plugins"]) + set_value(data, ("app", "arguments"), ["--label", "two words", ""]) + set_value(data, ("services", 0, "runtime_dependencies"), ["bash"]) + timer = manifest_path.parent / "telemetry.timer" + _ = timer.write_text("[Timer]\nOnBootSec=10\n", encoding="utf-8") + set_value(data, ("services", 0, "unit"), [get_value(data, ("services", 0, "unit")), timer.name]) + write_manifest(manifest_path, data) + + manifest = load_manifest(manifest_path) + assert len(manifest.services[0].unit) == 2 + assert manifest.services[0].unit[1] == timer + assert manifest.services[0].runtime_dependencies == ["bash"] + assert manifest.app.build_dependencies == ["qtdeclarative-native", "qtbase"] + assert manifest.app.runtime_dependencies == ["qtbase-plugins"] + assert manifest.app.arguments == ["--label", "two words", ""] + + +@pytest.mark.parametrize(("keys", "value", "field"), [ + (("schema_version",), True, "schema_version"), + (("schema_version",), 1.0, "schema_version"), + (("schema_version",), "1", "schema_version"), + (("schema_version",), 2, "schema_version"), + (("enabled",), 1, "enabled"), + (("enabled",), "true", "enabled"), + (("id",), "../app", "id"), + (("version",), "0.1/escape", "version"), + (("license_file",), "missing", "license_file"), + (("license_file",), "../outside", "license_file"), + (("app",), [], "app"), + (("app", "source_dir"), "missing", "app.source_dir"), + (("app", "source_dir"), "tests", "app.source_dir"), + (("app", "executable"), "app --flag", "app.executable"), + (("app", "arguments"), "--flag", "app.arguments"), + (("app", "arguments"), [False], "app.arguments[0]"), + (("app", "arguments"), ["bad\0argument"], "app.arguments[0]"), + (("app", "unknown"), True, "app.unknown"), + (("app", "build_dependencies"), ["${INJECT}"], "app.build_dependencies[0]"), + (("app", "runtime_dependencies"), "bash", "app.runtime_dependencies"), + (("services",), {}, "services"), + (("services", 0, "start"), "app", "services[0].start"), + (("services", 0, "unit"), [], "services[0].unit"), + (("services", 0, "unit"), "missing.service", "services[0].unit[0]"), + (("profiles", "dev", "settings"), [], "profiles.dev.settings"), + (("profiles", "unknown"), {}, "profiles.unknown"), + (("manifest_path",), "supplied", "manifest.manifest_path"), + (("unknown",), True, "manifest.unknown"), +]) +def test_invalid_field( + manifest_path: Path, data: dict[str, object], + keys: tuple[str | int, ...], value: object, field: str, +) -> None: + set_value(data, keys, value) + write_manifest(manifest_path, data) + with pytest.raises(ManifestError, match=re.escape(field)) as error: + _ = load_manifest(manifest_path) + assert str(manifest_path) in str(error.value) + + +@pytest.mark.parametrize(("content", "message"), [ + ("{", "line"), + ("[]", "manifest"), + ('{"id": 1, "id": 2}', "duplicate JSON key"), + ('{"value": NaN}', "invalid JSON value"), +]) +def test_invalid_json(manifest_path: Path, content: str, message: str) -> None: + _ = manifest_path.write_text(content, encoding="utf-8") + with pytest.raises(ManifestError, match=message) as error: + _ = load_manifest(manifest_path) + assert str(manifest_path) in str(error.value) + + +def test_missing_required_field(manifest_path: Path, data: dict[str, object]) -> None: + del data["name"] + write_manifest(manifest_path, data) + with pytest.raises(ManifestError, match=r"manifest\.name"): + _ = load_manifest(manifest_path) + + +@pytest.mark.parametrize(("same_id", "message"), [ + (True, "duplicate service ID"), + (False, "duplicate unit filename"), +]) +def test_duplicate_services( + manifest_path: Path, data: dict[str, object], same_id: bool, message: str, +) -> None: + service = get_value(data, ("services", 0)) + data["services"] = [copy.deepcopy(service), copy.deepcopy(service)] + if not same_id: + set_value(data, ("services", 1, "id"), "other") + write_manifest(manifest_path, data) + with pytest.raises(ManifestError, match=message): + _ = load_manifest(manifest_path) + + +@pytest.mark.parametrize("link", ["outside", "loop"]) +def test_invalid_source_symlink(manifest_path: Path, data: dict[str, object], link: str) -> None: + target = str(manifest_path.parent.parent) if link == "outside" else link + (manifest_path.parent / link).symlink_to(target, target_is_directory=True) + set_value(data, ("app", "source_dir"), link) + write_manifest(manifest_path, data) + with pytest.raises(ManifestError, match=r"app\.source_dir"): + _ = load_manifest(manifest_path) + + +def test_absolute_source_path(manifest_path: Path, data: dict[str, object]) -> None: + set_value(data, ("app", "source_dir"), str(manifest_path.parent / "app")) + write_manifest(manifest_path, data) + with pytest.raises(ManifestError, match="bundle-relative path"): + _ = load_manifest(manifest_path) + + +def test_defaults_and_disabled_selection(manifest_path: Path, data: dict[str, object]) -> None: + del data["services"] + del data["profiles"] + del cast(dict[str, object], data["app"])["arguments"] + data["enabled"] = False + write_manifest(manifest_path, data) + manifest = load_manifest(manifest_path) + assert manifest.services == [] + assert manifest.profiles == {} + assert manifest.app.arguments == [] + assert load_manifests(manifest_path.parent.parent)[0].enabled is False + + +def test_disabled_bundle_is_validated(manifest_path: Path, data: dict[str, object]) -> None: + data["enabled"] = False + set_value(data, ("app", "source_dir"), "missing") + write_manifest(manifest_path, data) + with pytest.raises(ManifestError, match=r"app\.source_dir"): + _ = load_manifests(manifest_path.parent.parent) + + +@pytest.mark.parametrize("same_id", [True, False]) +def test_duplicates_across_bundles(manifest_path: Path, data: dict[str, object], same_id: bool) -> None: + apps = manifest_path.parent.parent + second = apps / "second" + _ = shutil.copytree(manifest_path.parent, second) + if not same_id: + data["id"] = "second-app" + write_manifest(second / "mosaic-app.json", data) + with pytest.raises(ManifestError, match="already declared") as error: + _ = load_manifests(apps) + assert str(second / "mosaic-app.json") in str(error.value) + assert ("id:" if same_id else "services[0].unit:") in str(error.value) + + +def test_directory_order(manifest_path: Path, data: dict[str, object]) -> None: + apps = manifest_path.parent.parent + second = apps / "00-second" + _ = shutil.copytree(manifest_path.parent, second) + data["id"] = "second-app" + data["services"] = [] + write_manifest(second / "mosaic-app.json", data) + assert [item.id for item in load_manifests(apps)] == ["second-app", "dummy-application"] + + +def test_empty_apps_directory(tmp_path: Path) -> None: + assert load_manifests(tmp_path) == [] + + +def test_missing_apps_directory(tmp_path: Path) -> None: + with pytest.raises(ManifestError, match="apps_dir"): + _ = load_manifests(tmp_path / "missing") From 6db94168b830093054f5d8857266452ab3cd0704 Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Sun, 4 Oct 2026 23:47:12 +0200 Subject: [PATCH 08/10] Configure python in setup-build-env --- Dockerfile | 4 +++- docs/building.rst | 15 ++++++++------- setup-build-env | 34 +++++++++++++++++++++++++++++++--- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5fba745..5b30d12 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,8 +17,10 @@ RUN apt-get update \ COPY tools/requirements.txt /tmp/mosaic-tools-requirements.txt RUN python3 -m venv --system-site-packages /opt/mosaic-tools \ && /opt/mosaic-tools/bin/python -m pip install --no-cache-dir \ - -r /tmp/mosaic-tools-requirements.txt + -r /tmp/mosaic-tools-requirements.txt \ + && cp /tmp/mosaic-tools-requirements.txt /opt/mosaic-tools/.mosaic-requirements.txt +ENV MOSAIC_PREBUILT_TOOLS=/opt/mosaic-tools ENV PATH="/opt/mosaic-tools/bin:${PATH}" RUN if ! getent group "${HOST_GID}" > /dev/null; then \ diff --git a/docs/building.rst b/docs/building.rst index fd79026..56c6450 100644 --- a/docs/building.rst +++ b/docs/building.rst @@ -77,15 +77,16 @@ installed. From the repository root, in Bash or Zsh: .. code-block:: bash - python3 -m venv --system-site-packages .venv-tools - .venv-tools/bin/python -m pip install -r tools/requirements.txt - source .venv-tools/bin/activate source ./setup-build-env -The tools environment supplies Pydantic for manifest validation and retains -access to the host's Yocto Python dependencies. Activate it in each new build -shell before sourcing ``setup-build-env``. The Docker builder includes this -environment automatically. +The wrapper creates and activates ``.venv-tools``, retaining access to the host's +Yocto Python dependencies. It installs ``tools/requirements.txt`` on first setup +and when that file changes. Unchanged environments skip package installation. +The first install may need network access. Pytest remains an optional development +dependency, installed as described in :doc:`applications`. + +Docker uses its prepared Python environment. If its dependencies no longer match +the checkout, setup asks you to rebuild it with ``docker compose build``. The wrapper initializes submodules using ``git submodule update --init --recursive``, selects ``yocto/layers/meta-mosaic-product/conf/templates/mosaic``, and sources diff --git a/setup-build-env b/setup-build-env index 946c565..ef8384d 100644 --- a/setup-build-env +++ b/setup-build-env @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Source this file so BitBake's environment remains in the current shell. +# Source this file so BitBake's environment remains in the current shell if [ -n "${BASH_VERSION:-}" ]; then if [ "${BASH_SOURCE[0]}" = "$0" ]; then @@ -23,11 +23,39 @@ fi _mosaic_setup_build() { local mosaic_script=$1 local mosaic_root + local mosaic_venv + local mosaic_requirements local TEMPLATECONF mosaic_root=$(cd -- "$(dirname -- "$mosaic_script")" && pwd -P) || return 1 - # Git retains the revisions pinned by the parent repository. + mosaic_requirements="$mosaic_root/tools/requirements.txt" + mosaic_venv=${MOSAIC_PREBUILT_TOOLS:-"$mosaic_root/.venv-tools"} + + if [ -n "${MOSAIC_PREBUILT_TOOLS:-}" ]; then + if [ ! -x "$mosaic_venv/bin/python" ] || [ ! -f "$mosaic_venv/bin/activate" ] || + ! cmp -s "$mosaic_requirements" "$mosaic_venv/.mosaic-requirements.txt"; then + printf '%s\n' 'Builder Python dependencies are outdated. Run: docker compose build' >&2 + return 1 + fi + else + if [ ! -x "$mosaic_venv/bin/python" ] || [ ! -f "$mosaic_venv/bin/activate" ]; then + if ! python3 -m venv --system-site-packages "$mosaic_venv"; then + printf '%s\n' 'Could not create .venv-tools. Check Python venv support is installed.' >&2 + return 1 + fi + fi + if ! cmp -s "$mosaic_requirements" "$mosaic_venv/.mosaic-requirements.txt"; then + if ! "$mosaic_venv/bin/python" -m pip install -r "$mosaic_requirements"; then + printf '%s\n' 'Could not install Mosaic Python dependencies. Source setup-build-env to retry.' >&2 + return 1 + fi + cp "$mosaic_requirements" "$mosaic_venv/.mosaic-requirements.txt" || return 1 + fi + fi + source "$mosaic_venv/bin/activate" || return 1 + + # Git retains the revisions pinned by the parent repository git -C "$mosaic_root" submodule update --init --recursive || return 1 export TEMPLATECONF="$mosaic_root/yocto/layers/meta-mosaic-product/conf/templates/mosaic" @@ -38,7 +66,7 @@ _mosaic_setup_build() { return 1 fi - # Explicit paths keep setup independent of the caller's working directory. + # Explicit paths keep setup independent of the caller's working directory source "$mosaic_root/yocto/sources/openembedded-core/oe-init-build-env" \ "$mosaic_root/build" "$mosaic_root/yocto/sources/bitbake" || return 1 } From 504460bba17686ef337cf8e3c7e1fb25b08b3b70 Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Mon, 5 Oct 2026 00:00:32 +0200 Subject: [PATCH 09/10] Add python self test --- tools/tests/test_setup_build_env.py | 106 ++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 tools/tests/test_setup_build_env.py diff --git a/tools/tests/test_setup_build_env.py b/tools/tests/test_setup_build_env.py new file mode 100644 index 0000000..92da9e9 --- /dev/null +++ b/tools/tests/test_setup_build_env.py @@ -0,0 +1,106 @@ +"""Check setup and environment reuse without downloads or a Yocto build""" + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + + +pytestmark = pytest.mark.parametrize("shell", ["bash", "zsh"]) + + +@pytest.fixture +def checkout(tmp_path: Path) -> Path: + root = tmp_path.resolve() / "mosaic checkout" + root.mkdir() + source = Path(__file__).resolve().parents[2] / "setup-build-env" + _ = shutil.copyfile(source, root / "setup-build-env") + (root / "tools").mkdir() + _ = (root / "tools/requirements.txt").write_text("example==1\n") + templates = root / "yocto/layers/meta-mosaic-product/conf/templates/mosaic" + templates.mkdir(parents=True) + for name in ("local.conf.sample", "bblayers.conf.sample"): + (templates / name).touch() + oe = root / "yocto/sources/openembedded-core" + oe.mkdir(parents=True) + _ = (oe / "oe-init-build-env").write_text('mkdir -p "$1"\ncd "$1" || return 1\n') + + # Use real virtual environments, replacing only pip's network operation + (root / "stubs").mkdir() + _ = (root / "stubs/pip.py").write_text(''' + import os + import sys + from pathlib import Path + + root = Path(os.environ["TEST_ROOT"]) + with (root / "installs").open("a") as calls: + calls.write("install\\n") + sys.exit(int((root / "fail-install").exists())) + ''') + return root + + +def run_setup( + root: Path, shell: str, *, repeat: bool = False, prebuilt: Path | None = None, +) -> subprocess.CompletedProcess[str]: + shell_path = shutil.which(shell) + if shell_path is None: + pytest.skip(f"{shell} is not installed") + env = { + **os.environ, + "PATH": f"{Path(sys.executable).parent}:{os.environ['PATH']}", + "PYTHONPATH": str(root / "stubs"), + "TEST_ROOT": str(root), + "MOSAIC_PREBUILT_TOOLS": str(prebuilt) if prebuilt else "", + } + command = 'git() { :; }\nsource "$TEST_ROOT/setup-build-env" || exit $?\n' + if repeat: + command += 'source "$TEST_ROOT/setup-build-env" || exit $?\n' + command += 'printf "%s\\n" "$VIRTUAL_ENV" "$PWD" "$(command -v python3)"\n' + return subprocess.run([shell_path, "-c", command], cwd=root.parent, env=env, text=True, capture_output=True) + + +def test_first_setup_and_reentry(checkout: Path, shell: str) -> None: + result = run_setup(checkout, shell, repeat=True) + assert result.returncode == 0, result.stderr + assert result.stdout.splitlines() == [ + str(checkout / ".venv-tools"), str(checkout / "build"), str(checkout / ".venv-tools/bin/python3"), + ] + assert (checkout / "installs").read_text() == "install\n" + assert (checkout / ".venv-tools/.mosaic-requirements.txt").read_text() == "example==1\n" + + +def test_failed_install_can_retry(checkout: Path, shell: str) -> None: + failure = checkout / "fail-install" + failure.touch() + result = run_setup(checkout, shell) + assert result.returncode != 0 + assert "Could not install" in result.stderr + assert not (checkout / "build").exists() + assert not (checkout / ".venv-tools/.mosaic-requirements.txt").exists() + + failure.unlink() + result = run_setup(checkout, shell) + assert result.returncode == 0, result.stderr + assert (checkout / "installs").read_text() == "install\ninstall\n" + + +def test_prebuilt_environment(checkout: Path, shell: str) -> None: + prebuilt = checkout.parent / "prebuilt tools" + _ = subprocess.run([sys.executable, "-m", "venv", "--without-pip", str(prebuilt)], check=True) + requirements = checkout / "tools/requirements.txt" + _ = shutil.copyfile(requirements, prebuilt / ".mosaic-requirements.txt") + result = run_setup(checkout, shell, prebuilt=prebuilt) + assert result.returncode == 0, result.stderr + assert result.stdout.splitlines()[0] == str(prebuilt) + assert not (checkout / ".venv-tools").exists() + assert not (checkout / "installs").exists() + + _ = requirements.write_text("example==2\n") + result = run_setup(checkout, shell, prebuilt=prebuilt) + assert result.returncode != 0 + assert "docker compose build" in result.stderr + assert not (checkout / "installs").exists() From 1f229e3eb7dfdc2332a7860fc69974b0aa9d94ae Mon Sep 17 00:00:00 2001 From: Tmpecho Date: Mon, 5 Oct 2026 00:07:01 +0200 Subject: [PATCH 10/10] Add mosiac-build script to PATH on setup --- docs/building.rst | 17 +++++++++-------- setup-build-env | 5 +++++ tools/tests/test_setup_build_env.py | 14 ++++++++++---- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/docs/building.rst b/docs/building.rst index 56c6450..bd227d2 100644 --- a/docs/building.rst +++ b/docs/building.rst @@ -43,7 +43,7 @@ The container automatically sources ``setup-build-env``, then starts a shell in .. code-block:: bash - docker compose run --rm builder bitbake mc:qemu-dev:mosaic-image + docker compose run --rm builder mosaic-build mc:qemu-dev:mosaic-image The repository is mounted at ``/workspace``, and ``/workspace/build`` is a persistent Docker volume. This keeps temporary build files, downloads, and sstate on a @@ -52,7 +52,8 @@ directory inside the container. Docker build outputs are stored in the volume an not in the host checkout. Removing a container with ``--rm`` preserves that volume. To copy the image and related build outputs for the QEMU development profile -from the Docker volume to ``build/artifacts/`` in your local checkout: +from the Docker volume to ``build/artifacts/`` in your local checkout, from outside of +Docker run: .. code-block:: bash @@ -91,8 +92,9 @@ the checkout, setup asks you to rebuild it with ``docker compose build``. The wrapper initializes submodules using ``git submodule update --init --recursive``, selects ``yocto/layers/meta-mosaic-product/conf/templates/mosaic``, and sources OpenEmbedded-Core's ``oe-init-build-env``. It creates ``build/`` at the repository -root and changes the current directory there, with BitBake available in the -current shell. The first run needs network access if submodules are missing. +root and changes the current directory there, with BitBake and ``mosaic-build`` +available on ``PATH`` in the current shell. The first run needs network access +if submodules are missing. Always source the wrapper. Executing it in a child shell cannot initialize your current shell. You can also source it by absolute path from another directory: @@ -107,13 +109,12 @@ are preserved. Build dev and prod images ------------------------- -After setup you can check bitbake works, and you can build an image: +From the ``build/`` directory entered by setup, run ``mosaic-build`` to generate +the application layer and then invoke BitBake: .. code-block:: bash - bitbake-getvar DISTRO - - bitbake mc:qemu-dev:mosaic-image + mosaic-build mc:qemu-dev:mosaic-image The available profiles for the targets are: diff --git a/setup-build-env b/setup-build-env index ef8384d..adb9461 100644 --- a/setup-build-env +++ b/setup-build-env @@ -69,6 +69,11 @@ _mosaic_setup_build() { # Explicit paths keep setup independent of the caller's working directory source "$mosaic_root/yocto/sources/openembedded-core/oe-init-build-env" \ "$mosaic_root/build" "$mosaic_root/yocto/sources/bitbake" || return 1 + + case ":$PATH:" in + *":$mosaic_root:"*) ;; + *) export PATH="$mosaic_root:$PATH" ;; + esac } if _mosaic_setup_build "$_mosaic_setup_script"; then diff --git a/tools/tests/test_setup_build_env.py b/tools/tests/test_setup_build_env.py index 92da9e9..b4d28ed 100644 --- a/tools/tests/test_setup_build_env.py +++ b/tools/tests/test_setup_build_env.py @@ -5,6 +5,7 @@ import subprocess import sys from pathlib import Path +from textwrap import dedent import pytest @@ -18,6 +19,7 @@ def checkout(tmp_path: Path) -> Path: root.mkdir() source = Path(__file__).resolve().parents[2] / "setup-build-env" _ = shutil.copyfile(source, root / "setup-build-env") + _ = shutil.copy2(source.parent / "mosaic-build", root / "mosaic-build") (root / "tools").mkdir() _ = (root / "tools/requirements.txt").write_text("example==1\n") templates = root / "yocto/layers/meta-mosaic-product/conf/templates/mosaic" @@ -30,7 +32,7 @@ def checkout(tmp_path: Path) -> Path: # Use real virtual environments, replacing only pip's network operation (root / "stubs").mkdir() - _ = (root / "stubs/pip.py").write_text(''' + _ = (root / "stubs/pip.py").write_text(dedent(''' import os import sys from pathlib import Path @@ -39,7 +41,7 @@ def checkout(tmp_path: Path) -> Path: with (root / "installs").open("a") as calls: calls.write("install\\n") sys.exit(int((root / "fail-install").exists())) - ''') + ''')) return root @@ -59,16 +61,19 @@ def run_setup( command = 'git() { :; }\nsource "$TEST_ROOT/setup-build-env" || exit $?\n' if repeat: command += 'source "$TEST_ROOT/setup-build-env" || exit $?\n' - command += 'printf "%s\\n" "$VIRTUAL_ENV" "$PWD" "$(command -v python3)"\n' + command += 'printf "%s\\n" "$VIRTUAL_ENV" "$PWD" "$(command -v python3)" "$(command -v mosaic-build)" "$PATH"\n' return subprocess.run([shell_path, "-c", command], cwd=root.parent, env=env, text=True, capture_output=True) def test_first_setup_and_reentry(checkout: Path, shell: str) -> None: result = run_setup(checkout, shell, repeat=True) assert result.returncode == 0, result.stderr - assert result.stdout.splitlines() == [ + lines = result.stdout.splitlines() + assert lines[:4] == [ str(checkout / ".venv-tools"), str(checkout / "build"), str(checkout / ".venv-tools/bin/python3"), + str(checkout / "mosaic-build"), ] + assert lines[4].split(os.pathsep).count(str(checkout)) == 1 assert (checkout / "installs").read_text() == "install\n" assert (checkout / ".venv-tools/.mosaic-requirements.txt").read_text() == "example==1\n" @@ -96,6 +101,7 @@ def test_prebuilt_environment(checkout: Path, shell: str) -> None: result = run_setup(checkout, shell, prebuilt=prebuilt) assert result.returncode == 0, result.stderr assert result.stdout.splitlines()[0] == str(prebuilt) + assert result.stdout.splitlines()[3] == str(checkout / "mosaic-build") assert not (checkout / ".venv-tools").exists() assert not (checkout / "installs").exists()