diff --git a/.dockerignore b/.dockerignore index f27a865..cd79285 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ # Sources are mounted at runtime. The image only contains host build tools. ** !Dockerfile +!tools/ +!tools/requirements.txt diff --git a/.gitignore b/.gitignore index e8f5548..055908a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,10 @@ /.env /.venv-docs/ +/.venv-tools/ /docs/_build/ +/.pytest_cache/ +**/__pycache__/ **/.DS_Store +/.idea/ diff --git a/Dockerfile b/Dockerfile index 385827e..5b30d12 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,11 +9,20 @@ RUN apt-get update \ build-essential ca-certificates chrpath cpio debianutils diffstat file \ gawk gcc git iputils-ping libacl1 libcrypt-dev locales openssh-client \ python3 python3-git python3-jinja2 python3-pexpect python3-pip \ - python3-subunit python3-websockets socat texinfo unzip wget xz-utils zstd \ + python3-subunit python3-venv python3-websockets socat texinfo unzip wget xz-utils zstd \ && rm -rf /var/lib/apt/lists/* \ && echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen \ && locale-gen +COPY tools/requirements.txt /tmp/mosaic-tools-requirements.txt +RUN python3 -m venv --system-site-packages /opt/mosaic-tools \ + && /opt/mosaic-tools/bin/python -m pip install --no-cache-dir \ + -r /tmp/mosaic-tools-requirements.txt \ + && cp /tmp/mosaic-tools-requirements.txt /opt/mosaic-tools/.mosaic-requirements.txt + +ENV MOSAIC_PREBUILT_TOOLS=/opt/mosaic-tools +ENV PATH="/opt/mosaic-tools/bin:${PATH}" + RUN if ! getent group "${HOST_GID}" > /dev/null; then \ groupadd --gid "${HOST_GID}" builder; \ fi \ diff --git a/docs/app-generation.rst b/docs/app-generation.rst new file mode 100644 index 0000000..e6f6248 --- /dev/null +++ b/docs/app-generation.rst @@ -0,0 +1,81 @@ +Application layer generation +============================ + +Application bundles become part of a Mosaic image through a generated BitBake +layer. Developers supply a manifest, application code, and any required services +under ``apps/`` without editing shared shell source or writing BitBake recipes. + +Build flow +---------- + +``mosaic-build`` runs the generator before invoking BitBake: + +1. The generator parses ``apps/*/mosaic-app.json`` and validates identifiers, + required fields, dependency declarations, source paths, licenses, and systemd + unit files. + It validates all manifests before writing output. Invalid input reports the + manifest and failing field, and stops the build before BitBake runs. +2. It generates recipes and layer configuration in + ``build/generated/meta-mosaic-apps/``. Repeated generation produces consistent + output and removes obsolete generated files within this layer. +3. ``mosaic-build`` forwards the build arguments to BitBake, which builds the + selected applications, services, dependencies, and image in dependency order. + +Recipe and image integration +---------------------------- + +The generator creates one recipe per application and one per bundled service. +Recipes use standard Yocto classes for Qt/CMake builds and systemd integration. +They reference the bundle's sources and license file. CMake and BitBake own compilation, +installation, and packaging. + +Applications and services explicitly declare their additional build and runtime +dependencies in the manifest. Build dependency recipe names map to ``DEPENDS``. +Runtime package names map to ``RDEPENDS``. The generator supplies the standard Qt +application dependencies rather than inferring dependencies from source code. + +Each application package depends on its required service packages. Service +recipes install the supplied systemd unit file or files and enable services +declared with ``start: boot``. Systemd runs services independently of application +selection and termination. + +Layer configuration supplies ``MOSAIC_APPS_COMMON`` and +``MOSAIC_APPS_DEV_ONLY`` to the existing application packagegroup. Disabled +bundles are excluded. Enabled bundles with ``include_in_prod: true`` are included +in both profiles, while the remaining enabled bundles are included only in dev. +Selecting an application also includes its required service packages. + +Each application recipe installs a JSON launcher entry at +``/usr/share/mosaic/apps/.json`` containing its existing ``id``, ``name``, +``executable``, and ``arguments``. +The shell reads installed entries, displays each application's name, and starts +its executable with the listed arguments when selected after unlock. The launcher +owns discovery and startup of installed applications. + +Tool files +---------- + +The tool uses Pydantic for manifest validation and Python's standard library for +file handling. It consists of three files under ``tools/``: + +``generate_app_layer.py`` + Accepts ``--apps-dir`` and ``--output``, coordinates parsing and generation, + and reports errors with a nonzero exit status. + +``app_manifest.py`` + Reads and validates manifests, returning validated application and service + data without writing generated files. + +``app_layer.py`` + Generates recipes, launcher entries, and layer configuration from the + validated data, and removes obsolete output. + +Verification +------------ + +Verification covers an application with a service, multiple unit files, +dependency mapping, dev/prod selection, invalid manifests, repeatable generation, +and stale output removal. BitBake parsing and an image build validate the +generated recipes. +Launcher integration checks cover installed applications appearing by name and +starting on selection while their services run independently. diff --git a/docs/applications.rst b/docs/applications.rst index de2ed25..e27ff3d 100644 --- a/docs/applications.rst +++ b/docs/applications.rst @@ -12,3 +12,87 @@ Build selection uses the manifest: Selection includes the bundle's required services. Applications start when selected from the launcher after unlock; services run independently. + +Manifest fields +--------------- + +The parser reads ``apps/*/mosaic-app.json`` in directory order and validates all +bundles, including disabled ones. Unknown fields and duplicate JSON keys are +errors. Each error identifies the manifest and affected field. + +The following top-level fields are required: + +- ``schema_version`` is the integer ``1``. +- ``id`` is unique across bundles and contains lowercase letters, digits, and + hyphens, starting with a letter or digit. +- ``name`` is a nonempty display name. +- ``version`` starts with a digit and contains letters, digits, dots, ``+``, or + ``~``. +- ``enabled`` and ``include_in_prod`` are booleans with the selection behavior + described above. +- ``license`` is a nonempty Yocto license identifier or expression. +- ``license_file`` points to the bundle's license file. +- ``app`` describes the application build and executable. + +``app`` requires ``source_dir`` and ``executable``. The source directory contains +``CMakeLists.txt``. The executable is an installed program name, starting with +a letter or digit and containing only letters, digits, dots, underscores, +hyphens, or ``+``. Arguments are supplied separately in the optional +``arguments`` string list, which defaults to ``[]``. Empty argument strings are +allowed, while NUL characters are rejected. + +Both ``app`` and each service accept optional ``build_dependencies`` and +``runtime_dependencies`` lists, defaulting to ``[]``. These contain Yocto recipe +names and runtime package names respectively. Names start with a lowercase letter +or digit and contain lowercase letters, digits, dots, hyphens, or ``+``. Entries +are plain names without version constraints or BitBake expressions. Provider +availability is checked by BitBake. + +``services`` is an optional list, defaulting to ``[]``. Each service requires: + +- ``id``, following the application ID format and unique within the bundle. +- ``source_dir``, containing the service's ``CMakeLists.txt``. +- ``unit``, a path or nonempty list of paths to systemd unit files. Unit filenames + must be unique across bundles and contain only letters, digits, dots, + underscores, ``@``, ``:``, ``+``, or hyphens. +- ``start``, currently accepting only ``"boot"``. + +All source, license, and unit paths are relative to the manifest's directory. +They must exist and stay within that directory after resolving symlinks. Text +fields other than arguments reject control characters. The parser checks file +presence, not CMake code, license validity, or systemd unit contents. + +``profiles`` is an optional object, defaulting to ``{}``, with optional ``dev`` +and ``prod`` entries. Each entry contains a required ``settings`` object holding +application-defined JSON values. The parser preserves these settings without +interpreting them. + +Example manifest +---------------- + +.. literalinclude:: ../apps/dummy-application/mosaic-app.json + :language: json + +Parser interface +---------------- + +``tools/app_manifest.py`` exposes ``load_manifest(path)`` for one bundle and +``load_manifests(apps_dir)`` for all bundles. They return typed Pydantic models with +optional defaults filled in, source and file paths resolved to absolute +``Path`` objects, and ``unit`` normalized to a list. Fields are accessed as +attributes, such as ``manifest.app.executable``. Each model exposes +``manifest_path`` for diagnostics. An empty application directory returns an +empty list. Invalid input raises ``ManifestError`` without writing files. + +Install the development dependencies once in the build tools environment +described in :doc:`building`, then run all Python tooling tests from the +repository root: + +.. code-block:: bash + + .venv-tools/bin/python -m pip install -r tools/requirements-dev.txt + .venv-tools/bin/python -m pytest + +Pytest discovers tests under ``tools/tests/``. It does not scan upstream Yocto +sources. To run only the manifest tests, append +``tools/tests/test_app_manifest.py`` to the pytest command. diff --git a/docs/building.rst b/docs/building.rst index 5c04fae..bd227d2 100644 --- a/docs/building.rst +++ b/docs/building.rst @@ -43,7 +43,7 @@ The container automatically sources ``setup-build-env``, then starts a shell in .. code-block:: bash - docker compose run --rm builder bitbake mc:qemu-dev:mosaic-image + docker compose run --rm builder mosaic-build mc:qemu-dev:mosaic-image The repository is mounted at ``/workspace``, and ``/workspace/build`` is a persistent Docker volume. This keeps temporary build files, downloads, and sstate on a @@ -52,7 +52,8 @@ directory inside the container. Docker build outputs are stored in the volume an not in the host checkout. Removing a container with ``--rm`` preserves that volume. To copy the image and related build outputs for the QEMU development profile -from the Docker volume to ``build/artifacts/`` in your local checkout: +from the Docker volume to ``build/artifacts/`` in your local checkout, from outside of +Docker run: .. code-block:: bash @@ -79,11 +80,21 @@ installed. From the repository root, in Bash or Zsh: source ./setup-build-env +The wrapper creates and activates ``.venv-tools``, retaining access to the host's +Yocto Python dependencies. It installs ``tools/requirements.txt`` on first setup +and when that file changes. Unchanged environments skip package installation. +The first install may need network access. Pytest remains an optional development +dependency, installed as described in :doc:`applications`. + +Docker uses its prepared Python environment. If its dependencies no longer match +the checkout, setup asks you to rebuild it with ``docker compose build``. + The wrapper initializes submodules using ``git submodule update --init --recursive``, selects ``yocto/layers/meta-mosaic-product/conf/templates/mosaic``, and sources OpenEmbedded-Core's ``oe-init-build-env``. It creates ``build/`` at the repository -root and changes the current directory there, with BitBake available in the -current shell. The first run needs network access if submodules are missing. +root and changes the current directory there, with BitBake and ``mosaic-build`` +available on ``PATH`` in the current shell. The first run needs network access +if submodules are missing. Always source the wrapper. Executing it in a child shell cannot initialize your current shell. You can also source it by absolute path from another directory: @@ -98,13 +109,12 @@ are preserved. Build dev and prod images ------------------------- -After setup you can check bitbake works, and you can build an image: +From the ``build/`` directory entered by setup, run ``mosaic-build`` to generate +the application layer and then invoke BitBake: .. code-block:: bash - bitbake-getvar DISTRO - - bitbake mc:qemu-dev:mosaic-image + mosaic-build mc:qemu-dev:mosaic-image The available profiles for the targets are: diff --git a/docs/index.rst b/docs/index.rst index ceef3bd..7f363f0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,4 +12,5 @@ applications and background services without changing the shell. mosiac-structure building applications + app-generation documentation diff --git a/mosaic-build b/mosaic-build new file mode 100755 index 0000000..fe161a1 --- /dev/null +++ b/mosaic-build @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e + +mosaic_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" + +python3 "$mosaic_root/tools/generate_app_layer.py" \ + --apps-dir "$mosaic_root/apps" \ + --output "$mosaic_root/build/generated/meta-mosaic-apps" + +exec bitbake "$@" diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..b58eece --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +testpaths = tools/tests +addopts = -q diff --git a/setup-build-env b/setup-build-env index 946c565..adb9461 100644 --- a/setup-build-env +++ b/setup-build-env @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Source this file so BitBake's environment remains in the current shell. +# Source this file so BitBake's environment remains in the current shell if [ -n "${BASH_VERSION:-}" ]; then if [ "${BASH_SOURCE[0]}" = "$0" ]; then @@ -23,11 +23,39 @@ fi _mosaic_setup_build() { local mosaic_script=$1 local mosaic_root + local mosaic_venv + local mosaic_requirements local TEMPLATECONF mosaic_root=$(cd -- "$(dirname -- "$mosaic_script")" && pwd -P) || return 1 - # Git retains the revisions pinned by the parent repository. + mosaic_requirements="$mosaic_root/tools/requirements.txt" + mosaic_venv=${MOSAIC_PREBUILT_TOOLS:-"$mosaic_root/.venv-tools"} + + if [ -n "${MOSAIC_PREBUILT_TOOLS:-}" ]; then + if [ ! -x "$mosaic_venv/bin/python" ] || [ ! -f "$mosaic_venv/bin/activate" ] || + ! cmp -s "$mosaic_requirements" "$mosaic_venv/.mosaic-requirements.txt"; then + printf '%s\n' 'Builder Python dependencies are outdated. Run: docker compose build' >&2 + return 1 + fi + else + if [ ! -x "$mosaic_venv/bin/python" ] || [ ! -f "$mosaic_venv/bin/activate" ]; then + if ! python3 -m venv --system-site-packages "$mosaic_venv"; then + printf '%s\n' 'Could not create .venv-tools. Check Python venv support is installed.' >&2 + return 1 + fi + fi + if ! cmp -s "$mosaic_requirements" "$mosaic_venv/.mosaic-requirements.txt"; then + if ! "$mosaic_venv/bin/python" -m pip install -r "$mosaic_requirements"; then + printf '%s\n' 'Could not install Mosaic Python dependencies. Source setup-build-env to retry.' >&2 + return 1 + fi + cp "$mosaic_requirements" "$mosaic_venv/.mosaic-requirements.txt" || return 1 + fi + fi + source "$mosaic_venv/bin/activate" || return 1 + + # Git retains the revisions pinned by the parent repository git -C "$mosaic_root" submodule update --init --recursive || return 1 export TEMPLATECONF="$mosaic_root/yocto/layers/meta-mosaic-product/conf/templates/mosaic" @@ -38,9 +66,14 @@ _mosaic_setup_build() { return 1 fi - # Explicit paths keep setup independent of the caller's working directory. + # Explicit paths keep setup independent of the caller's working directory source "$mosaic_root/yocto/sources/openembedded-core/oe-init-build-env" \ "$mosaic_root/build" "$mosaic_root/yocto/sources/bitbake" || return 1 + + case ":$PATH:" in + *":$mosaic_root:"*) ;; + *) export PATH="$mosaic_root:$PATH" ;; + esac } if _mosaic_setup_build "$_mosaic_setup_script"; then diff --git a/tools/app_layer.py b/tools/app_layer.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/app_manifest.py b/tools/app_manifest.py new file mode 100644 index 0000000..02c98d2 --- /dev/null +++ b/tools/app_manifest.py @@ -0,0 +1,184 @@ +"""Read and validate application bundles without generating files""" + +import json +import re +from pathlib import Path +from typing import Annotated, ClassVar, Literal, NoReturn, Self, cast + +from pydantic import ( + AfterValidator, BaseModel, BeforeValidator, ConfigDict, DirectoryPath, Field, + FilePath, PrivateAttr, ValidationError, ValidationInfo, field_validator, + model_validator, +) + + +class ManifestError(ValueError): + """A manifest could not be read or contains an invalid field""" + + +def _text(value: str) -> str: + if not value.strip(): + raise ValueError("expected a nonempty string") + if any(ord(char) < 32 or ord(char) == 127 for char in value): + raise ValueError("control characters are not allowed") + return value + + +def _bundle_path(value: object, info: ValidationInfo) -> Path: + if not isinstance(value, str): + raise ValueError("expected a bundle-relative path string") + relative = Path(_text(value)) + if relative.is_absolute(): + raise ValueError("expected a bundle-relative path") + manifest_path = cast(Path, info.context) + try: + resolved = (manifest_path.parent / relative).resolve() + except (OSError, RuntimeError, UnicodeError) as error: + raise ValueError(str(error)) from error + if not resolved.is_relative_to(manifest_path.parent): + raise ValueError("path must stay inside the application bundle") + return resolved + + +def _cmake_directory(value: Path) -> Path: + if not (value / "CMakeLists.txt").is_file(): + raise ValueError("missing CMakeLists.txt") + return value + + +def _unit_file(value: Path) -> Path: + if not re.fullmatch(r"[A-Za-z0-9_.@:+-]+", value.name): + raise ValueError("invalid unit filename") + return value + + +Text = Annotated[str, AfterValidator(_text)] +Identifier = Annotated[Text, Field(pattern=r"^[a-z0-9][a-z0-9-]*$")] +Dependency = Annotated[Text, Field(pattern=r"^[a-z0-9][a-z0-9.+-]*$")] +Argument = Annotated[str, Field(pattern=r"^[^\x00]*$")] +BundleFile = Annotated[FilePath, BeforeValidator(_bundle_path)] +SourceDirectory = Annotated[DirectoryPath, BeforeValidator(_bundle_path), AfterValidator(_cmake_directory)] +UnitFile = Annotated[BundleFile, AfterValidator(_unit_file)] + + +class ManifestModel(BaseModel): + model_config: ClassVar[ConfigDict] = ConfigDict(strict=True, extra="forbid") + + +class Component(ManifestModel): + source_dir: SourceDirectory + build_dependencies: list[Dependency] = Field(default_factory=list) + runtime_dependencies: list[Dependency] = Field(default_factory=list) + + +class Application(Component): + executable: Annotated[Text, Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._+-]*$")] + arguments: list[Argument] = Field(default_factory=list) + + +class Service(Component): + id: Identifier + unit: Annotated[list[UnitFile], Field(min_length=1)] + start: Literal["boot"] + + @field_validator("unit", mode="before") + @classmethod + def unit_list(cls, value: object) -> object: + return [value] if isinstance(value, str) else value + + +class Profile(ManifestModel): + settings: dict[str, object] + + +class Manifest(ManifestModel): + schema_version: Annotated[int, Field(ge=1, le=1)] + id: Identifier + name: Text + version: Annotated[Text, Field(pattern=r"^[0-9][A-Za-z0-9.+~]*$")] + enabled: bool + include_in_prod: bool + license: Text + license_file: BundleFile + app: Application + services: list[Service] = Field(default_factory=list) + profiles: dict[Literal["dev", "prod"], Profile] = Field(default_factory=dict) + _manifest_path: Path = PrivateAttr() + + @property + def manifest_path(self) -> Path: + return self._manifest_path + + @model_validator(mode="after") + def check_services(self, info: ValidationInfo) -> Self: + self._manifest_path = cast(Path, info.context) + service_ids: set[str] = set() + unit_names: set[str] = set() + for index, service in enumerate(self.services): + if service.id in service_ids: + raise ValueError(f"services[{index}].id: duplicate service ID") + service_ids.add(service.id) + for unit in service.unit: + if unit.name in unit_names: + raise ValueError(f"services[{index}].unit: duplicate unit filename") + unit_names.add(unit.name) + return self + + +def _unique_keys(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key {key!r}") + result[key] = value + return result + + +def _invalid_constant(value: str) -> NoReturn: + raise ValueError(f"invalid JSON value {value}") + + +def load_manifest(path: Path) -> Manifest: + """Return a validated model with absolute paths and its manifest location""" + try: + path = path.resolve() + raw = cast(object, json.loads(path.read_text(encoding="utf-8"), + object_pairs_hook=_unique_keys, parse_constant=_invalid_constant)) + return Manifest.model_validate(raw, context=path) + except ValidationError as error: + detail = error.errors(include_url=False, include_input=False, include_context=False)[0] + field = "manifest" + "".join( + f"[{part}]" if isinstance(part, int) else f".{part}" for part in detail["loc"] + ) + raise ManifestError(f"{path}: {field}: {detail['msg']}") from error + except (OSError, RuntimeError, UnicodeError, ValueError) as error: + raise ManifestError(f"{path}: {error}") from error + + +def load_manifests(apps_dir: Path) -> list[Manifest]: + """Load all bundles in directory order, including disabled bundles""" + try: + apps_dir = apps_dir.resolve() + except (OSError, RuntimeError, UnicodeError) as error: + raise ManifestError(f"{apps_dir}: apps_dir: {error}") from error + if not apps_dir.is_dir(): + raise ManifestError(f"{apps_dir}: apps_dir: directory does not exist") + + manifests: list[Manifest] = [] + app_ids: dict[str, Path] = {} + unit_names: dict[str, Path] = {} + for path in sorted(apps_dir.glob("*/mosaic-app.json")): + manifest = load_manifest(path) + if manifest.id in app_ids: + raise ManifestError(f"{path}: id: {manifest.id!r} is already declared in {app_ids[manifest.id]}") + app_ids[manifest.id] = path + for index, service in enumerate(manifest.services): + for unit in service.unit: + if unit.name in unit_names: + raise ManifestError( + f"{path}: services[{index}].unit: {unit.name!r} " + + f"is already declared in {unit_names[unit.name]}" + ) + unit_names[unit.name] = path + manifests.append(manifest) + return manifests diff --git a/tools/generate_app_layer.py b/tools/generate_app_layer.py new file mode 100644 index 0000000..4e995bf --- /dev/null +++ b/tools/generate_app_layer.py @@ -0,0 +1,29 @@ +"""Generate a Yocto application layer from user application manifests""" + +import argparse +from pathlib import Path + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + _ = parser.add_argument( + "--apps-dir", + type=Path, + required=True, + help="Directory containing application bundles", + ) + _ = parser.add_argument( + "--output", + type=Path, + required=True, + help="Destination for the generated BitBake layer", + ) + _ = parser.parse_args() + + # TODO: Read and validate manifests with app_manifest before writing output + # TODO: Generate the layer from the validated data with app_layer + parser.exit(1, "error: application layer generation is not implemented yet\n") + + +if __name__ == "__main__": + main() diff --git a/tools/requirements-dev.txt b/tools/requirements-dev.txt new file mode 100644 index 0000000..1441c92 --- /dev/null +++ b/tools/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest==9.1.1 diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 0000000..a22eda9 --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1 @@ +pydantic==2.13.5 diff --git a/tools/tests/test_app_manifest.py b/tools/tests/test_app_manifest.py new file mode 100644 index 0000000..f4127ea --- /dev/null +++ b/tools/tests/test_app_manifest.py @@ -0,0 +1,226 @@ +"""Manifest parsing, normalization, and validation checks""" + +import copy +import json +import re +import shutil +from pathlib import Path +from typing import cast + +import pytest + +from tools.app_manifest import ManifestError, load_manifest, load_manifests + + +@pytest.fixture +def manifest_path(tmp_path: Path) -> Path: + example = Path(__file__).resolve().parents[2] / "apps/dummy-application" + bundle = tmp_path.resolve() / "apps/first" + _ = shutil.copytree(example, bundle) + return bundle / "mosaic-app.json" + + +@pytest.fixture +def data(manifest_path: Path) -> dict[str, object]: + return cast(dict[str, object], json.loads(manifest_path.read_text(encoding="utf-8"))) + + +def get_value(value: object, keys: tuple[str | int, ...]) -> object: + for key in keys: + if isinstance(key, str): + value = cast(dict[str, object], value)[key] + else: + value = cast(list[object], value)[key] + return value + + +def set_value(data: dict[str, object], keys: tuple[str | int, ...], value: object) -> None: + target = get_value(data, keys[:-1]) + key = keys[-1] + if isinstance(key, str): + cast(dict[str, object], target)[key] = value + else: + cast(list[object], target)[key] = value + + +def write_manifest(path: Path, data: dict[str, object]) -> None: + _ = path.write_text(json.dumps(data), encoding="utf-8") + + +def test_load_example(manifest_path: Path, data: dict[str, object]) -> None: + original = manifest_path.read_text() + manifest = load_manifest(manifest_path) + assert manifest.id == "dummy-application" + assert manifest.app.source_dir == manifest_path.parent / "app" + assert manifest.services[0].unit[0].is_absolute() + assert manifest.app.build_dependencies == [] + assert manifest.manifest_path == manifest_path + assert {name: {"settings": profile.settings} for name, profile in manifest.profiles.items()} == data["profiles"] + assert manifest_path.read_text() == original + + +def test_dependencies_arguments_and_units(manifest_path: Path, data: dict[str, object]) -> None: + set_value(data, ("app", "build_dependencies"), ["qtdeclarative-native", "qtbase"]) + set_value(data, ("app", "runtime_dependencies"), ["qtbase-plugins"]) + set_value(data, ("app", "arguments"), ["--label", "two words", ""]) + set_value(data, ("services", 0, "runtime_dependencies"), ["bash"]) + timer = manifest_path.parent / "telemetry.timer" + _ = timer.write_text("[Timer]\nOnBootSec=10\n", encoding="utf-8") + set_value(data, ("services", 0, "unit"), [get_value(data, ("services", 0, "unit")), timer.name]) + write_manifest(manifest_path, data) + + manifest = load_manifest(manifest_path) + assert len(manifest.services[0].unit) == 2 + assert manifest.services[0].unit[1] == timer + assert manifest.services[0].runtime_dependencies == ["bash"] + assert manifest.app.build_dependencies == ["qtdeclarative-native", "qtbase"] + assert manifest.app.runtime_dependencies == ["qtbase-plugins"] + assert manifest.app.arguments == ["--label", "two words", ""] + + +@pytest.mark.parametrize(("keys", "value", "field"), [ + (("schema_version",), True, "schema_version"), + (("schema_version",), 1.0, "schema_version"), + (("schema_version",), "1", "schema_version"), + (("schema_version",), 2, "schema_version"), + (("enabled",), 1, "enabled"), + (("enabled",), "true", "enabled"), + (("id",), "../app", "id"), + (("version",), "0.1/escape", "version"), + (("license_file",), "missing", "license_file"), + (("license_file",), "../outside", "license_file"), + (("app",), [], "app"), + (("app", "source_dir"), "missing", "app.source_dir"), + (("app", "source_dir"), "tests", "app.source_dir"), + (("app", "executable"), "app --flag", "app.executable"), + (("app", "arguments"), "--flag", "app.arguments"), + (("app", "arguments"), [False], "app.arguments[0]"), + (("app", "arguments"), ["bad\0argument"], "app.arguments[0]"), + (("app", "unknown"), True, "app.unknown"), + (("app", "build_dependencies"), ["${INJECT}"], "app.build_dependencies[0]"), + (("app", "runtime_dependencies"), "bash", "app.runtime_dependencies"), + (("services",), {}, "services"), + (("services", 0, "start"), "app", "services[0].start"), + (("services", 0, "unit"), [], "services[0].unit"), + (("services", 0, "unit"), "missing.service", "services[0].unit[0]"), + (("profiles", "dev", "settings"), [], "profiles.dev.settings"), + (("profiles", "unknown"), {}, "profiles.unknown"), + (("manifest_path",), "supplied", "manifest.manifest_path"), + (("unknown",), True, "manifest.unknown"), +]) +def test_invalid_field( + manifest_path: Path, data: dict[str, object], + keys: tuple[str | int, ...], value: object, field: str, +) -> None: + set_value(data, keys, value) + write_manifest(manifest_path, data) + with pytest.raises(ManifestError, match=re.escape(field)) as error: + _ = load_manifest(manifest_path) + assert str(manifest_path) in str(error.value) + + +@pytest.mark.parametrize(("content", "message"), [ + ("{", "line"), + ("[]", "manifest"), + ('{"id": 1, "id": 2}', "duplicate JSON key"), + ('{"value": NaN}', "invalid JSON value"), +]) +def test_invalid_json(manifest_path: Path, content: str, message: str) -> None: + _ = manifest_path.write_text(content, encoding="utf-8") + with pytest.raises(ManifestError, match=message) as error: + _ = load_manifest(manifest_path) + assert str(manifest_path) in str(error.value) + + +def test_missing_required_field(manifest_path: Path, data: dict[str, object]) -> None: + del data["name"] + write_manifest(manifest_path, data) + with pytest.raises(ManifestError, match=r"manifest\.name"): + _ = load_manifest(manifest_path) + + +@pytest.mark.parametrize(("same_id", "message"), [ + (True, "duplicate service ID"), + (False, "duplicate unit filename"), +]) +def test_duplicate_services( + manifest_path: Path, data: dict[str, object], same_id: bool, message: str, +) -> None: + service = get_value(data, ("services", 0)) + data["services"] = [copy.deepcopy(service), copy.deepcopy(service)] + if not same_id: + set_value(data, ("services", 1, "id"), "other") + write_manifest(manifest_path, data) + with pytest.raises(ManifestError, match=message): + _ = load_manifest(manifest_path) + + +@pytest.mark.parametrize("link", ["outside", "loop"]) +def test_invalid_source_symlink(manifest_path: Path, data: dict[str, object], link: str) -> None: + target = str(manifest_path.parent.parent) if link == "outside" else link + (manifest_path.parent / link).symlink_to(target, target_is_directory=True) + set_value(data, ("app", "source_dir"), link) + write_manifest(manifest_path, data) + with pytest.raises(ManifestError, match=r"app\.source_dir"): + _ = load_manifest(manifest_path) + + +def test_absolute_source_path(manifest_path: Path, data: dict[str, object]) -> None: + set_value(data, ("app", "source_dir"), str(manifest_path.parent / "app")) + write_manifest(manifest_path, data) + with pytest.raises(ManifestError, match="bundle-relative path"): + _ = load_manifest(manifest_path) + + +def test_defaults_and_disabled_selection(manifest_path: Path, data: dict[str, object]) -> None: + del data["services"] + del data["profiles"] + del cast(dict[str, object], data["app"])["arguments"] + data["enabled"] = False + write_manifest(manifest_path, data) + manifest = load_manifest(manifest_path) + assert manifest.services == [] + assert manifest.profiles == {} + assert manifest.app.arguments == [] + assert load_manifests(manifest_path.parent.parent)[0].enabled is False + + +def test_disabled_bundle_is_validated(manifest_path: Path, data: dict[str, object]) -> None: + data["enabled"] = False + set_value(data, ("app", "source_dir"), "missing") + write_manifest(manifest_path, data) + with pytest.raises(ManifestError, match=r"app\.source_dir"): + _ = load_manifests(manifest_path.parent.parent) + + +@pytest.mark.parametrize("same_id", [True, False]) +def test_duplicates_across_bundles(manifest_path: Path, data: dict[str, object], same_id: bool) -> None: + apps = manifest_path.parent.parent + second = apps / "second" + _ = shutil.copytree(manifest_path.parent, second) + if not same_id: + data["id"] = "second-app" + write_manifest(second / "mosaic-app.json", data) + with pytest.raises(ManifestError, match="already declared") as error: + _ = load_manifests(apps) + assert str(second / "mosaic-app.json") in str(error.value) + assert ("id:" if same_id else "services[0].unit:") in str(error.value) + + +def test_directory_order(manifest_path: Path, data: dict[str, object]) -> None: + apps = manifest_path.parent.parent + second = apps / "00-second" + _ = shutil.copytree(manifest_path.parent, second) + data["id"] = "second-app" + data["services"] = [] + write_manifest(second / "mosaic-app.json", data) + assert [item.id for item in load_manifests(apps)] == ["second-app", "dummy-application"] + + +def test_empty_apps_directory(tmp_path: Path) -> None: + assert load_manifests(tmp_path) == [] + + +def test_missing_apps_directory(tmp_path: Path) -> None: + with pytest.raises(ManifestError, match="apps_dir"): + _ = load_manifests(tmp_path / "missing") diff --git a/tools/tests/test_setup_build_env.py b/tools/tests/test_setup_build_env.py new file mode 100644 index 0000000..b4d28ed --- /dev/null +++ b/tools/tests/test_setup_build_env.py @@ -0,0 +1,112 @@ +"""Check setup and environment reuse without downloads or a Yocto build""" + +import os +import shutil +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import pytest + + +pytestmark = pytest.mark.parametrize("shell", ["bash", "zsh"]) + + +@pytest.fixture +def checkout(tmp_path: Path) -> Path: + root = tmp_path.resolve() / "mosaic checkout" + root.mkdir() + source = Path(__file__).resolve().parents[2] / "setup-build-env" + _ = shutil.copyfile(source, root / "setup-build-env") + _ = shutil.copy2(source.parent / "mosaic-build", root / "mosaic-build") + (root / "tools").mkdir() + _ = (root / "tools/requirements.txt").write_text("example==1\n") + templates = root / "yocto/layers/meta-mosaic-product/conf/templates/mosaic" + templates.mkdir(parents=True) + for name in ("local.conf.sample", "bblayers.conf.sample"): + (templates / name).touch() + oe = root / "yocto/sources/openembedded-core" + oe.mkdir(parents=True) + _ = (oe / "oe-init-build-env").write_text('mkdir -p "$1"\ncd "$1" || return 1\n') + + # Use real virtual environments, replacing only pip's network operation + (root / "stubs").mkdir() + _ = (root / "stubs/pip.py").write_text(dedent(''' + import os + import sys + from pathlib import Path + + root = Path(os.environ["TEST_ROOT"]) + with (root / "installs").open("a") as calls: + calls.write("install\\n") + sys.exit(int((root / "fail-install").exists())) + ''')) + return root + + +def run_setup( + root: Path, shell: str, *, repeat: bool = False, prebuilt: Path | None = None, +) -> subprocess.CompletedProcess[str]: + shell_path = shutil.which(shell) + if shell_path is None: + pytest.skip(f"{shell} is not installed") + env = { + **os.environ, + "PATH": f"{Path(sys.executable).parent}:{os.environ['PATH']}", + "PYTHONPATH": str(root / "stubs"), + "TEST_ROOT": str(root), + "MOSAIC_PREBUILT_TOOLS": str(prebuilt) if prebuilt else "", + } + command = 'git() { :; }\nsource "$TEST_ROOT/setup-build-env" || exit $?\n' + if repeat: + command += 'source "$TEST_ROOT/setup-build-env" || exit $?\n' + command += 'printf "%s\\n" "$VIRTUAL_ENV" "$PWD" "$(command -v python3)" "$(command -v mosaic-build)" "$PATH"\n' + return subprocess.run([shell_path, "-c", command], cwd=root.parent, env=env, text=True, capture_output=True) + + +def test_first_setup_and_reentry(checkout: Path, shell: str) -> None: + result = run_setup(checkout, shell, repeat=True) + assert result.returncode == 0, result.stderr + lines = result.stdout.splitlines() + assert lines[:4] == [ + str(checkout / ".venv-tools"), str(checkout / "build"), str(checkout / ".venv-tools/bin/python3"), + str(checkout / "mosaic-build"), + ] + assert lines[4].split(os.pathsep).count(str(checkout)) == 1 + assert (checkout / "installs").read_text() == "install\n" + assert (checkout / ".venv-tools/.mosaic-requirements.txt").read_text() == "example==1\n" + + +def test_failed_install_can_retry(checkout: Path, shell: str) -> None: + failure = checkout / "fail-install" + failure.touch() + result = run_setup(checkout, shell) + assert result.returncode != 0 + assert "Could not install" in result.stderr + assert not (checkout / "build").exists() + assert not (checkout / ".venv-tools/.mosaic-requirements.txt").exists() + + failure.unlink() + result = run_setup(checkout, shell) + assert result.returncode == 0, result.stderr + assert (checkout / "installs").read_text() == "install\ninstall\n" + + +def test_prebuilt_environment(checkout: Path, shell: str) -> None: + prebuilt = checkout.parent / "prebuilt tools" + _ = subprocess.run([sys.executable, "-m", "venv", "--without-pip", str(prebuilt)], check=True) + requirements = checkout / "tools/requirements.txt" + _ = shutil.copyfile(requirements, prebuilt / ".mosaic-requirements.txt") + result = run_setup(checkout, shell, prebuilt=prebuilt) + assert result.returncode == 0, result.stderr + assert result.stdout.splitlines()[0] == str(prebuilt) + assert result.stdout.splitlines()[3] == str(checkout / "mosaic-build") + assert not (checkout / ".venv-tools").exists() + assert not (checkout / "installs").exists() + + _ = requirements.write_text("example==2\n") + result = run_setup(checkout, shell, prebuilt=prebuilt) + assert result.returncode != 0 + assert "docker compose build" in result.stderr + assert not (checkout / "installs").exists() diff --git a/yocto/layers/meta-mosaic-product/conf/templates/mosaic/bblayers.conf.sample b/yocto/layers/meta-mosaic-product/conf/templates/mosaic/bblayers.conf.sample index ab38aff..e92f36c 100644 --- a/yocto/layers/meta-mosaic-product/conf/templates/mosaic/bblayers.conf.sample +++ b/yocto/layers/meta-mosaic-product/conf/templates/mosaic/bblayers.conf.sample @@ -13,4 +13,5 @@ BBLAYERS ?= " \ ##OEROOT##/../meta-st-stm32mp \ ##OEROOT##/../../layers/meta-mosaic \ ##OEROOT##/../../layers/meta-mosaic-product \ + ${TOPDIR}/generated/meta-mosaic-apps \ "