feat: start python parser

.dockerignore

@@ -1,3 +1,5 @@
 # Sources are mounted at runtime. The image only contains host build tools.
 **
 !Dockerfile
+!tools/
+!tools/requirements.txt

.gitignore

@@ -2,6 +2,10 @@
 /.env
 
 /.venv-docs/
+/.venv-tools/
 /docs/_build/
+/.pytest_cache/
+**/__pycache__/
 
 **/.DS_Store
+/.idea/

Dockerfile

@@ -9,11 +9,20 @@ RUN apt-get update \
         build-essential ca-certificates chrpath cpio debianutils diffstat file \
         gawk gcc git iputils-ping libacl1 libcrypt-dev locales openssh-client \
         python3 python3-git python3-jinja2 python3-pexpect python3-pip \
-        python3-subunit python3-websockets socat texinfo unzip wget xz-utils zstd \
+        python3-subunit python3-venv python3-websockets socat texinfo unzip wget xz-utils zstd \
     && rm -rf /var/lib/apt/lists/* \
     && echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen \
     && locale-gen
 
+COPY tools/requirements.txt /tmp/mosaic-tools-requirements.txt
+RUN python3 -m venv --system-site-packages /opt/mosaic-tools \
+    && /opt/mosaic-tools/bin/python -m pip install --no-cache-dir \
+        -r /tmp/mosaic-tools-requirements.txt \
+    && cp /tmp/mosaic-tools-requirements.txt /opt/mosaic-tools/.mosaic-requirements.txt
+
+ENV MOSAIC_PREBUILT_TOOLS=/opt/mosaic-tools
+ENV PATH="/opt/mosaic-tools/bin:${PATH}"
+
 RUN if ! getent group "${HOST_GID}" > /dev/null; then \
         groupadd --gid "${HOST_GID}" builder; \
     fi \

docs/app-generation.rst

@@ -0,0 +1,81 @@
+Application layer generation
+============================
+
+Application bundles become part of a Mosaic image through a generated BitBake
+layer. Developers supply a manifest, application code, and any required services
+under ``apps/`` without editing shared shell source or writing BitBake recipes.
+
+Build flow
+----------
+
+``mosaic-build`` runs the generator before invoking BitBake:
+
+1. The generator parses ``apps/*/mosaic-app.json`` and validates identifiers,
+   required fields, dependency declarations, source paths, licenses, and systemd
+   unit files.
+   It validates all manifests before writing output. Invalid input reports the
+   manifest and failing field, and stops the build before BitBake runs.
+2. It generates recipes and layer configuration in
+   ``build/generated/meta-mosaic-apps/``. Repeated generation produces consistent
+   output and removes obsolete generated files within this layer.
+3. ``mosaic-build`` forwards the build arguments to BitBake, which builds the
+   selected applications, services, dependencies, and image in dependency order.
+
+Recipe and image integration
+----------------------------
+
+The generator creates one recipe per application and one per bundled service.
+Recipes use standard Yocto classes for Qt/CMake builds and systemd integration.
+They reference the bundle's sources and license file. CMake and BitBake own compilation,
+installation, and packaging.
+
+Applications and services explicitly declare their additional build and runtime
+dependencies in the manifest. Build dependency recipe names map to ``DEPENDS``.
+Runtime package names map to ``RDEPENDS``. The generator supplies the standard Qt
+application dependencies rather than inferring dependencies from source code.
+
+Each application package depends on its required service packages. Service
+recipes install the supplied systemd unit file or files and enable services
+declared with ``start: boot``. Systemd runs services independently of application
+selection and termination.
+
+Layer configuration supplies ``MOSAIC_APPS_COMMON`` and
+``MOSAIC_APPS_DEV_ONLY`` to the existing application packagegroup. Disabled
+bundles are excluded. Enabled bundles with ``include_in_prod: true`` are included
+in both profiles, while the remaining enabled bundles are included only in dev.
+Selecting an application also includes its required service packages.
+
+Each application recipe installs a JSON launcher entry at
+``/usr/share/mosaic/apps/<id>.json`` containing its existing ``id``, ``name``,
+``executable``, and ``arguments``. 
+The shell reads installed entries, displays each application's name, and starts
+its executable with the listed arguments when selected after unlock. The launcher
+owns discovery and startup of installed applications.
+
+Tool files
+----------
+
+The tool uses Pydantic for manifest validation and Python's standard library for
+file handling. It consists of three files under ``tools/``:
+
+``generate_app_layer.py``
+    Accepts ``--apps-dir`` and ``--output``, coordinates parsing and generation,
+    and reports errors with a nonzero exit status.
+
+``app_manifest.py``
+    Reads and validates manifests, returning validated application and service
+    data without writing generated files.
+
+``app_layer.py``
+    Generates recipes, launcher entries, and layer configuration from the
+    validated data, and removes obsolete output.
+
+Verification
+------------
+
+Verification covers an application with a service, multiple unit files,
+dependency mapping, dev/prod selection, invalid manifests, repeatable generation,
+and stale output removal. BitBake parsing and an image build validate the
+generated recipes.
+Launcher integration checks cover installed applications appearing by name and
+starting on selection while their services run independently.

docs/applications.rst

@@ -12,3 +12,87 @@ Build selection uses the manifest:
 
 Selection includes the bundle's required services. Applications start when
 selected from the launcher after unlock; services run independently.
+
+Manifest fields
+---------------
+
+The parser reads ``apps/*/mosaic-app.json`` in directory order and validates all
+bundles, including disabled ones. Unknown fields and duplicate JSON keys are
+errors. Each error identifies the manifest and affected field.
+
+The following top-level fields are required:
+
+- ``schema_version`` is the integer ``1``.
+- ``id`` is unique across bundles and contains lowercase letters, digits, and
+  hyphens, starting with a letter or digit.
+- ``name`` is a nonempty display name.
+- ``version`` starts with a digit and contains letters, digits, dots, ``+``, or
+  ``~``.
+- ``enabled`` and ``include_in_prod`` are booleans with the selection behavior
+  described above.
+- ``license`` is a nonempty Yocto license identifier or expression.
+- ``license_file`` points to the bundle's license file.
+- ``app`` describes the application build and executable.
+
+``app`` requires ``source_dir`` and ``executable``. The source directory contains
+``CMakeLists.txt``. The executable is an installed program name, starting with
+a letter or digit and containing only letters, digits, dots, underscores,
+hyphens, or ``+``. Arguments are supplied separately in the optional
+``arguments`` string list, which defaults to ``[]``. Empty argument strings are
+allowed, while NUL characters are rejected.
+
+Both ``app`` and each service accept optional ``build_dependencies`` and
+``runtime_dependencies`` lists, defaulting to ``[]``. These contain Yocto recipe
+names and runtime package names respectively. Names start with a lowercase letter
+or digit and contain lowercase letters, digits, dots, hyphens, or ``+``. Entries
+are plain names without version constraints or BitBake expressions. Provider
+availability is checked by BitBake.
+
+``services`` is an optional list, defaulting to ``[]``. Each service requires:
+
+- ``id``, following the application ID format and unique within the bundle.
+- ``source_dir``, containing the service's ``CMakeLists.txt``.
+- ``unit``, a path or nonempty list of paths to systemd unit files. Unit filenames
+  must be unique across bundles and contain only letters, digits, dots,
+  underscores, ``@``, ``:``, ``+``, or hyphens.
+- ``start``, currently accepting only ``"boot"``.
+
+All source, license, and unit paths are relative to the manifest's directory.
+They must exist and stay within that directory after resolving symlinks. Text
+fields other than arguments reject control characters. The parser checks file
+presence, not CMake code, license validity, or systemd unit contents.
+
+``profiles`` is an optional object, defaulting to ``{}``, with optional ``dev``
+and ``prod`` entries. Each entry contains a required ``settings`` object holding
+application-defined JSON values. The parser preserves these settings without
+interpreting them.
+
+Example manifest
+----------------
+
+.. literalinclude:: ../apps/dummy-application/mosaic-app.json
+   :language: json
+
+Parser interface
+----------------
+
+``tools/app_manifest.py`` exposes ``load_manifest(path)`` for one bundle and
+``load_manifests(apps_dir)`` for all bundles. They return typed Pydantic models with
+optional defaults filled in, source and file paths resolved to absolute
+``Path`` objects, and ``unit`` normalized to a list. Fields are accessed as
+attributes, such as ``manifest.app.executable``. Each model exposes
+``manifest_path`` for diagnostics. An empty application directory returns an
+empty list. Invalid input raises ``ManifestError`` without writing files.
+
+Install the development dependencies once in the build tools environment
+described in :doc:`building`, then run all Python tooling tests from the
+repository root:
+
+.. code-block:: bash
+
+   .venv-tools/bin/python -m pip install -r tools/requirements-dev.txt
+   .venv-tools/bin/python -m pytest
+
+Pytest discovers tests under ``tools/tests/``. It does not scan upstream Yocto
+sources. To run only the manifest tests, append
+``tools/tests/test_app_manifest.py`` to the pytest command.

docs/building.rst

@@ -43,7 +43,7 @@ The container automatically sources ``setup-build-env``, then starts a shell in
 
 .. code-block:: bash
 
-   docker compose run --rm builder bitbake mc:qemu-dev:mosaic-image
+   docker compose run --rm builder mosaic-build mc:qemu-dev:mosaic-image
 
 The repository is mounted at ``/workspace``, and ``/workspace/build`` is a persistent
 Docker volume. This keeps temporary build files, downloads, and sstate on a
@@ -52,7 +52,8 @@ directory inside the container. Docker build outputs are stored in the volume an
 not in the host checkout. Removing a container with ``--rm`` preserves that volume.
 
 To copy the image and related build outputs for the QEMU development profile
-from the Docker volume to ``build/artifacts/`` in your local checkout:
+from the Docker volume to ``build/artifacts/`` in your local checkout, from outside of
+Docker run:
 
 .. code-block:: bash
 
@@ -79,11 +80,21 @@ installed. From the repository root, in Bash or Zsh:
 
    source ./setup-build-env
 
+The wrapper creates and activates ``.venv-tools``, retaining access to the host's
+Yocto Python dependencies. It installs ``tools/requirements.txt`` on first setup
+and when that file changes. Unchanged environments skip package installation.
+The first install may need network access. Pytest remains an optional development
+dependency, installed as described in :doc:`applications`.
+
+Docker uses its prepared Python environment. If its dependencies no longer match
+the checkout, setup asks you to rebuild it with ``docker compose build``.
+
 The wrapper initializes submodules using ``git submodule update --init --recursive``,
 selects ``yocto/layers/meta-mosaic-product/conf/templates/mosaic``, and sources
 OpenEmbedded-Core's ``oe-init-build-env``. It creates ``build/`` at the repository
-root and changes the current directory there, with BitBake available in the
-current shell. The first run needs network access if submodules are missing.
+root and changes the current directory there, with BitBake and ``mosaic-build``
+available on ``PATH`` in the current shell. The first run needs network access
+if submodules are missing.
 
 Always source the wrapper. Executing it in a child shell cannot initialize your
 current shell. You can also source it by absolute path from another directory:
@@ -98,13 +109,12 @@ are preserved.
 Build dev and prod images
 -------------------------
 
-After setup you can check bitbake works, and you can build an image:
+From the ``build/`` directory entered by setup, run ``mosaic-build`` to generate
+the application layer and then invoke BitBake:
 
 .. code-block:: bash
 
-   bitbake-getvar DISTRO
-
-   bitbake mc:qemu-dev:mosaic-image
+   mosaic-build mc:qemu-dev:mosaic-image
 
 The available profiles for the targets are:
 

docs/index.rst

@@ -12,4 +12,5 @@ applications and background services without changing the shell.
    mosiac-structure
    building
    applications
+   app-generation
    documentation

mosaic-build

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -e
+
+mosaic_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+
+python3 "$mosaic_root/tools/generate_app_layer.py" \
+    --apps-dir "$mosaic_root/apps" \
+    --output "$mosaic_root/build/generated/meta-mosaic-apps"
+
+exec bitbake "$@"

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+testpaths = tools/tests
+addopts = -q

setup-build-env

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Source this file so BitBake's environment remains in the current shell.
+# Source this file so BitBake's environment remains in the current shell
 
 if [ -n "${BASH_VERSION:-}" ]; then
     if [ "${BASH_SOURCE[0]}" = "$0" ]; then
@@ -23,11 +23,39 @@ fi
 _mosaic_setup_build() {
     local mosaic_script=$1
     local mosaic_root
+    local mosaic_venv
+    local mosaic_requirements
     local TEMPLATECONF
 
     mosaic_root=$(cd -- "$(dirname -- "$mosaic_script")" && pwd -P) || return 1
 
-    # Git retains the revisions pinned by the parent repository.
+    mosaic_requirements="$mosaic_root/tools/requirements.txt"
+    mosaic_venv=${MOSAIC_PREBUILT_TOOLS:-"$mosaic_root/.venv-tools"}
+
+    if [ -n "${MOSAIC_PREBUILT_TOOLS:-}" ]; then
+        if [ ! -x "$mosaic_venv/bin/python" ] || [ ! -f "$mosaic_venv/bin/activate" ] ||
+           ! cmp -s "$mosaic_requirements" "$mosaic_venv/.mosaic-requirements.txt"; then
+            printf '%s\n' 'Builder Python dependencies are outdated. Run: docker compose build' >&2
+            return 1
+        fi
+    else
+        if [ ! -x "$mosaic_venv/bin/python" ] || [ ! -f "$mosaic_venv/bin/activate" ]; then
+            if ! python3 -m venv --system-site-packages "$mosaic_venv"; then
+                printf '%s\n' 'Could not create .venv-tools. Check Python venv support is installed.' >&2
+                return 1
+            fi
+        fi
+        if ! cmp -s "$mosaic_requirements" "$mosaic_venv/.mosaic-requirements.txt"; then
+            if ! "$mosaic_venv/bin/python" -m pip install -r "$mosaic_requirements"; then
+                printf '%s\n' 'Could not install Mosaic Python dependencies. Source setup-build-env to retry.' >&2
+                return 1
+            fi
+            cp "$mosaic_requirements" "$mosaic_venv/.mosaic-requirements.txt" || return 1
+        fi
+    fi
+    source "$mosaic_venv/bin/activate" || return 1
+
+    # Git retains the revisions pinned by the parent repository
     git -C "$mosaic_root" submodule update --init --recursive || return 1
 
     export TEMPLATECONF="$mosaic_root/yocto/layers/meta-mosaic-product/conf/templates/mosaic"
@@ -38,9 +66,14 @@ _mosaic_setup_build() {
         return 1
     fi
 
-    # Explicit paths keep setup independent of the caller's working directory.
+    # Explicit paths keep setup independent of the caller's working directory
     source "$mosaic_root/yocto/sources/openembedded-core/oe-init-build-env" \
         "$mosaic_root/build" "$mosaic_root/yocto/sources/bitbake" || return 1
+
+    case ":$PATH:" in
+        *":$mosaic_root:"*) ;;
+        *) export PATH="$mosaic_root:$PATH" ;;
+    esac
 }
 
 if _mosaic_setup_build "$_mosaic_setup_script"; then