From 4b8f55556ad1d65598bb8ddeb37a52e9ef6f1af1 Mon Sep 17 00:00:00 2001 From: 3alpha <15694175+3alpha@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:03:21 +0100 Subject: [PATCH] Validate manifest references and all upstream build arguments --- doc/package-spec-audit-2026-11.md | 33 +++++ doc/package-spec.md | 88 ++++++++----- src/files/compose/types.ts | 11 +- src/spec/checkPackageSpec.ts | 82 ++++++++---- src/spec/manifestReferences.ts | 85 ++++++++++++ src/spec/rules.ts | 12 +- test/spec/checkPackageSpec.test.ts | 205 ++++++++++++++++++++++++++++- 7 files changed, 441 insertions(+), 75 deletions(-) create mode 100644 doc/package-spec-audit-2026-11.md create mode 100644 src/spec/manifestReferences.ts diff --git a/doc/package-spec-audit-2026-11.md b/doc/package-spec-audit-2026-11.md new file mode 100644 index 00000000..25b8a8bf --- /dev/null +++ b/doc/package-spec-audit-2026-11.md @@ -0,0 +1,33 @@ +# Package-spec audit — November 2026 baseline + +This is the historical baseline used when the package spec was introduced. It predates subsequent package migrations and is not a pending-work list or a current scan. The normative rules are in [package-spec.md](package-spec.md). New scans should record their date, repository revisions and scan scope separately. + +Counts are from all 113 non-archived `DAppNodePackage*` and `DNP_*` repositories in November 2026; "active" means a push in the last 12 months (87 repositories). 58 active repositories currently have at least one finding, mostly SPEC-M03; 37 repositories are already clean. + +| Rule | Baseline observation | +|---|---| +| SPEC-M01 | 0 active repositories. | +| SPEC-M02 | 2 inactive (Zosma-Nethermind, Zosma-OE). | +| SPEC-M03 | 44 active repositories still use them, including DNP_BIND, DNP_IPFS, Hopr, NEAR, Hermes-agent and Honcho. This is the largest migration and removes a second code path from the SDK bumper, tropibot's discovery and edit code, and the dashboards. | +| SPEC-M04 | 9 active (SafeStake-testnet, SplitPro, avalanche, graph, monero, rocketpool, rocketpool-testnet, supabase, web3signer-kiln). | +| SPEC-M05 | 4 active (GnosisVPNExitNode, Hopr-testnet, Tornado-Cash-Relayer, aztec-testnet). | +| SPEC-M06 | 0 repositories. | +| SPEC-M07 | 1 active (owncloud: `minimumDappnodeVersion`), 5 inactive with legacy `avatar`/`image`. | +| SPEC-M08 | 8 active point elsewhere: renamed repositories (DMS, anchor-generic), sibling packages (Erigon-Sepolia-Archival, lukso-erigon, NEAR-testnet), a fork (avalanche) and upstream projects (SplitPro, supabase). | +| SPEC-M09 | 2 active (op-erigon, op-geth). | +| SPEC-T01 | 5 active (juno-generic, jupyter-generic, nethermind-generic, ollama-generic, pathfinder-generic) and madara-generic. | +| SPEC-T02 | 0 repositories. | +| SPEC-T03 | 0 repositories. | +| SPEC-T04 | 1 active (erigon-generic: a stale root `prometheus-targets.json`). | +| SPEC-T05 | 0 repositories. | +| SPEC-C01 | 0 repositories use the flag. | +| SPEC-C02 | 6 active (Hermes-agent, Nextcloud, graph, jupyter-generic, ollama-generic, openclaw). | +| SPEC-C03 | 2 active (graph: `postgres`; owncloud: `webhippie/mariadb:latest`, `webhippie/redis:latest`). | +| SPEC-C04 | 2 active (alephium, supabase) and madara-generic. | +| SPEC-C05 | 3 active (DNP_NOTIFICATIONS, pi-hole, portainer). | +| SPEC-F01 | 5 active: NEAR `setup-wizard-ui.yml`, Tornado-Cash-Relayer `avatar-default.jpg`, package-harness `avatar-harness.svg`, pathfinder-generic `getting-started.yml` (must be `.md`) and reth-gnosis `notifications.yml` (must be `.yaml`). | +| SPEC-F02 | 2 active (Lido-CSM-generic avatar 123 KB, lukso-erigon avatar 245 KB). | +| SPEC-F03 | 1 active (DMS notifications). | +| SPEC-B01 | no workflow uses it. | + +Other observations from the same review: 109 repositories declared Compose version, 25 omitted architectures, 6 inactive repositories used unpinned Dockerfile bases, and 20 active repositories used remote ADD without a checksum (19 through the staking-scripts pattern). Tropibot workflows covered 46 repositories and the SDK bumper covered 5. diff --git a/doc/package-spec.md b/doc/package-spec.md index ff4748f5..e1f5cd43 100644 --- a/doc/package-spec.md +++ b/doc/package-spec.md @@ -17,55 +17,75 @@ The SDK checks every package against these rules during `build` and `publish`. ## Rules -Counts are from all 113 non-archived `DAppNodePackage*` and `DNP_*` repositories in November 2026; "active" means a push in the last 12 months (87 repositories). 58 active repositories currently have at least one finding, mostly SPEC-M03; 37 repositories are already clean. +Repository counts and migration observations belong to dated audit reports; see the [November 2026 baseline](package-spec-audit-2026-11.md). They are not part of the package contract. ### Manifest -**SPEC-M01: One manifest named `dappnode_package.json`.** Each package directory (root and every variant) contains exactly one manifest, in JSON, named `dappnode_package.json`. `sdk-publish`, the ISO installer and tropibot read that exact name, and when several files match `dappnode_package*` the SDK picks one arbitrarily. Today: 0 active repositories. +**SPEC-M01: One manifest named `dappnode_package.json`.** Each package directory (root and every variant) contains exactly one manifest, in JSON, named `dappnode_package.json`. `sdk-publish`, the ISO installer and tropibot read that exact name, and when several files match `dappnode_package*` the SDK picks one arbitrarily. -**SPEC-M02: Package name format.** `name` matches `^[a-z0-9]+(-[a-z0-9]+)*\.(dnp|public)\.dappnode\.eth$`. Today: 2 inactive (Zosma-Nethermind, Zosma-OE). +**SPEC-M02: Package name format.** `name` matches `^[a-z0-9]+(-[a-z0-9]+)*\.(dnp|public)\.dappnode\.eth$`. -**SPEC-M03: Upstream declared with `upstream[]`.** Upstream software is declared as `"upstream": [{ "repo", "version", "arg" }]`. The legacy `upstreamRepo`, `upstreamVersion` and `upstreamArg` fields are deprecated. Today: 44 active repositories still use them, including DNP_BIND, DNP_IPFS, Hopr, NEAR, Hermes-agent and Honcho. This is the largest migration and removes a second code path from the SDK bumper, tropibot's discovery and edit code, and the dashboards. +**SPEC-M03: Upstream declared with `upstream[]`.** Upstream software is declared as `"upstream": [{ "repo", "version", "arg" }]`. The legacy `upstreamRepo`, `upstreamVersion` and `upstreamArg` fields are deprecated. -**SPEC-M04: The upstream `arg` is a compose build arg.** Every upstream entry names an `arg` that some compose service declares in `build.args`. Packages using the legacy fields often rely on the implicit default `UPSTREAM_VERSION`. Today: 9 active (SafeStake-testnet, SplitPro, avalanche, graph, monero, rocketpool, rocketpool-testnet, supabase, web3signer-kiln). +**SPEC-M04: The upstream `arg` is a compose build arg.** Every upstream entry names an `arg` that some compose service declares in `build.args`. Packages using the legacy fields often rely on the implicit default `UPSTREAM_VERSION`. -**SPEC-M05: The upstream `version` equals the build arg value.** The manifest version and the compose build arg are the same string, so tools never translate between `v1.2.3` and `1.2.3`. A digest-pinned arg (`1.2.3@sha256:…`) counts as equal. Today: 4 active (GnosisVPNExitNode, Hopr-testnet, Tornado-Cash-Relayer, aztec-testnet). +**SPEC-M05: The upstream `version` equals every matching build arg value.** Every compose service declaring the upstream `arg` must use the same string as `upstream[].version`. Matching one service does not excuse a conflicting value in another. A value `@sha256:` also matches when the digest contains exactly 64 hexadecimal characters. Findings identify the conflicting service and the compose fragment that supplies its value. -**SPEC-M06: A variant `upstream[]` keeps the root's positions.** Root and variant manifests are merged by array position. A variant may list fewer or more entries than the root, but an entry at an existing position must refer to the same `repo`. Today: 0 repositories. +**SPEC-M06: A variant `upstream[]` keeps the root's positions.** Root and variant manifests are merged by array position. A variant may list fewer or more entries than the root, but an entry at an existing position must refer to the same `repo`. -**SPEC-M07: Only fields from the manifest schema.** Fields outside the published manifest schema (plus `upnpDisable`, supported by DAPPMANAGER but not yet published) are ignored or stripped at install. Today: 1 active (owncloud: `minimumDappnodeVersion`), 5 inactive with legacy `avatar`/`image`. +**SPEC-M07: Only fields from the manifest schema.** Fields outside the published manifest schema (plus `upnpDisable`, supported by DAPPMANAGER but not yet published) are ignored or stripped at install. -**SPEC-M08: `repository.url` points to the package repository.** When a manifest declares `repository`, its URL is the package's own GitHub repository (`https://` or `git+https://`, with or without `.git`). Today: 8 active point elsewhere: renamed repositories (DMS, anchor-generic), sibling packages (Erigon-Sepolia-Archival, lukso-erigon, NEAR-testnet), a fork (avalanche) and upstream projects (SplitPro, supabase). +**SPEC-M08: `repository.url` points to the package repository.** When a manifest declares `repository`, its URL is the package's own GitHub repository (`https://` or `git+https://`, with or without `.git`). -**SPEC-M09: Exposable services exist in the compose.** `exposable[].serviceName` names a service in the compose. Today: 2 active (op-erigon, op-geth). +**SPEC-M09: Exposable services exist in the compose.** `exposable[].serviceName` names a service in the compose. + +**SPEC-M10: Manifest service references exist in the resolved compose.** Explicit `mainService`, `chain.serviceName`, `backup[].service` and `globalEnvs[].services[]` references name services present after root and variant files are merged. Omitted optional references retain their runtime defaults; the spec does not choose a service implicitly. Findings identify the manifest field, its defining source fragment and the affected variant. + +**SPEC-M11: Global environment names match the runtime catalogue.** Entries in `globalEnvs[].envs` use short names from the `GLOBAL_ENVS` catalogue bundled with `@dappnode/types`, for example `PUBLIC_IP`, `HOSTNAME` or `ENVS_ACTIVE`. DAPPMANAGER adds `_DAPPNODE_GLOBAL_` at install, so `_DAPPNODE_GLOBAL_PUBLIC_IP` in the manifest would be prefixed twice. Catalogue keys are not aliases: the key `ACTIVE` maps to `ENVS_ACTIVE`, which is the valid manifest value. Catalogue updates must be released through shared types and consumed by the SDK before packages adopt newly supported names. + +### Manifest semantics + +The source manifest describes package metadata and installation instructions. A template root and its variant manifests are source fragments; the merged result is the installable manifest and is validated against the full shared schema. Root fragments need not repeat each variant’s required identity fields. + +- `version` is the package’s registry version in numeric `major.minor.patch` form. It is independent of `upstream[].version`, whose string may include a leading `v` or an upstream-specific prefix. The registry ABI stores package-version components as unsigned 16-bit integers. Full canonical/range checking and the initial-publication rule are tracked in the later-phase work below; static builds should not require a chain connection. +- `upstream[].version` is the version string expected by its compose build argument, before an optional digest suffix. The spec never strips prefixes automatically. Mapping a source release tag to a different build value requires a separately defined upstream-provider contract. The SDK bumper currently requires unique upstream repositories and argument names; a shared duplicate-entry policy is still needed. +- Explicit global-env lists use `{ "envs": ["PUBLIC_IP"], "services": ["app"] }` entries. The runtime also supports legacy `{ "all": true }` to inject its global env file into every service; `{ "all": false }` is a no-op. Empty lists are allowed. Schema corrections for this legacy form and required `envs`/`services` fields must be published before all consumers agree. +- Minimum runtime versions belong under `requirements.minimumDappnodeVersion` and `requirements.minimumDockerVersion`, not at the manifest top level. +- SPEC-M07 currently checks top-level field names. Nested-field typo checks remain later-phase work, including the treatment of deliberately open maps such as `links`. + +### Variant inheritance + +The current manifest merge recursively combines objects, replaces scalar values and merges arrays by index. A variant entry at index 0 overrides the root entry at index 0; it does not replace the whole array. Omitted object fields and trailing array entries are inherited. An empty array does not clear an inherited array, and omitting a field does not delete it. Schema-invalid values such as `null` are not a deletion mechanism. + +For example, if the root upstream list is `[app, scripts]` and a variant supplies only a compatible `[appOverride]`, the resolved list is `[mergedApp, scripts]`. SPEC-M06 requires the root and variant to retain the same repository at every overlapping position. Checks of argument values and service references run on this resolved result. A move to whole-array replacement or merging by a stable key requires a coordinated migration across the SDK and consumers. ### Templates (multi-variant packages) -**SPEC-T01: The template root manifest has no `name` or `version`.** When a package has variants, each variant manifest declares its own `name` and `version` and the root holds only shared fields. A named root becomes a phantom package for pin cleanup, dashboards and on-chain sync. Today: 5 active (juno-generic, jupyter-generic, nethermind-generic, ollama-generic, pathfinder-generic) and madara-generic. +**SPEC-T01: The template root manifest has no `name` or `version`.** When a package has variants, each variant manifest declares its own `name` and `version` and the root holds only shared fields. A named root becomes a phantom package for pin cleanup, dashboards and on-chain sync. -**SPEC-T02: Variants live in `package_variants/`.** A custom `--variants_dir_name` is reported. Workflows, tropibot and the staking tests only read `package_variants/`. Today: 0 repositories. +**SPEC-T02: Variants live in `package_variants/`.** A custom `--variants_dir_name` is reported. Workflows, tropibot and the staking tests only read `package_variants/`. -**SPEC-T03: Variant directory names.** Variant names match `^[a-z0-9]+(-[a-z0-9]+)*$`; they appear in git tags (`@v`) and ENS names. Today: 0 repositories. +**SPEC-T03: Variant directory names.** Variant names match `^[a-z0-9]+(-[a-z0-9]+)*$`; they appear in git tags (`@v`) and ENS names. -**SPEC-T04: A single release file is in the root or the variant, not both.** For avatar, Prometheus targets, disclaimer, getting-started and the legacy setup files, the variant copy replaces the root copy, so a root copy next to variant copies is dead and goes stale. Setup wizards and notifications are merged by design, and Grafana dashboards are collected from both, so they are exempt. Today: 1 active (erigon-generic: a stale root `prometheus-targets.json`). +**SPEC-T04: A single release file is in the root or the variant, not both.** For avatar, Prometheus targets, disclaimer, getting-started and the legacy setup files, the variant copy replaces the root copy, so a root copy next to variant copies is dead and goes stale. Setup wizards and notifications are merged by design, and Grafana dashboards are collected from both, so they are exempt. -**SPEC-T05: Every variant directory has a manifest.** Today: 0 repositories. +**SPEC-T05: Every variant directory has a manifest.** ### Compose -**SPEC-C01: The compose file is `docker-compose.yml`.** A custom `--compose_file_name` is reported. DAPPMANAGER only recognises `compose*.yml` in a release, so `docker-compose.yaml` or `production.yml` produce a release that cannot be installed. Today: 0 repositories use the flag. +**SPEC-C01: The compose file is `docker-compose.yml`.** A custom `--compose_file_name` is reported. DAPPMANAGER only recognises `compose*.yml` in a release, so `docker-compose.yaml` or `production.yml` produce a release that cannot be installed. -**SPEC-C02: No `container_name`.** DAPPMANAGER replaces it at install, so it has no effect. Core packages (`type: dncore`) are exempt: the installer starts them directly and addresses them by name. Today: 6 active (Hermes-agent, Nextcloud, graph, jupyter-generic, ollama-generic, openclaw). +**SPEC-C02: No `container_name`.** DAPPMANAGER replaces it at install, so it has no effect. Core packages (`type: dncore`) are exempt: the installer starts them directly and addresses them by name. -**SPEC-C03: External images are pinned.** A service without `build` uses an image with an explicit, non-`latest` tag or a digest. Today: 2 active (graph: `postgres`; owncloud: `webhippie/mariadb:latest`, `webhippie/redis:latest`). +**SPEC-C03: External images are pinned.** A service without `build` uses an image with an explicit, non-`latest` tag or a digest. -**SPEC-C04: No `${VAR}` interpolation.** Compose values are written out; interpolation depends on the environment of whoever runs compose. Today: 2 active (alephium, supabase) and madara-generic. +**SPEC-C04: No `${VAR}` interpolation.** Compose values are written out; interpolation depends on the environment of whoever runs compose. -**SPEC-C05: `build.args` is a mapping.** `args: { KEY: value }`, not a list of `KEY=value` strings, so every tool that edits upstream versions needs one parser. Today: 3 active (DNP_NOTIFICATIONS, pi-hole, portainer). +**SPEC-C05: `build.args` is a mapping.** `args: { KEY: value }`, not a list of `KEY=value` strings, so every tool that edits upstream versions needs one parser. ### Release files -**SPEC-F01: Release files use the recognised names.** A file that looks like a release file but does not match its pattern is silently left out of the release. Today: 5 active: NEAR `setup-wizard-ui.yml`, Tornado-Cash-Relayer `avatar-default.jpg`, package-harness `avatar-harness.svg`, pathfinder-generic `getting-started.yml` (must be `.md`) and reth-gnosis `notifications.yml` (must be `.yaml`). +**SPEC-F01: Release files use the recognised names.** A file that looks like a release file but does not match its pattern is silently left out of the release. | Type | Recognised names | |---|---| @@ -79,13 +99,13 @@ Counts are from all 113 non-archived `DAppNodePackage*` and `DNP_*` repositories | Disclaimer | `disclaimer.md` | | Getting started | `getting-started.md` | -**SPEC-F02: Release files stay within the size limits** from `@dappnode/types` `releaseFiles` (avatar 100 KB, compose 10 KB, setup wizard 100 KB, notifications 10 KB, Prometheus targets 10 KB, dashboards 10 MB). Today: 2 active (Lido-CSM-generic avatar 123 KB, lukso-erigon avatar 245 KB). +**SPEC-F02: Release files stay within the size limits** from `@dappnode/types` `releaseFiles` (avatar 100 KB, compose 10 KB, setup wizard 100 KB, notifications 10 KB, Prometheus targets 10 KB, dashboards 10 MB). -**SPEC-F03: Values already have the schema type.** Validation coerces values in place (for example `isBanner: "false"` becomes `false`), and the release ships the coerced value. Today: 1 active (DMS notifications). +**SPEC-F03: Values already have the schema type.** Validation coerces values in place (for example `isBanner: "false"` becomes `false`), and the release ships the coerced value. ### Build inputs -**SPEC-B01: Compose validation is not skipped.** DAPPMANAGER validates the compose again at install, so `--skip_compose_validation` only defers the failure to users. Today: no workflow uses it. +**SPEC-B01: Compose validation is not skipped.** DAPPMANAGER validates the compose again at install, so `--skip_compose_validation` only defers the failure to users. **SPEC-B02: No `UPSTREAM_VERSION` environment override.** The upstream version comes from the package files, never from the build environment. @@ -93,12 +113,15 @@ Counts are from all 113 non-archived `DAppNodePackage*` and `DNP_*` repositories These are not checked yet, either because they need consumer changes first or because they are too noisy today. -- **Compose `version:`** is obsolete (109 repositories still set it). Rather than warn, the SDK should strip it from releases by default once versionless composes are verified with DAPPMANAGER (`--strip_compose_version` exists today as an opt-in). -- **Dockerfiles:** no `FROM …:latest` or untagged base images (6 inactive repositories), and no remote `ADD` without `--checksum` (20 active repositories; 19 are the staking `ADD ${STAKER_SCRIPTS_URL}/*.sh` pattern, which needs a shared replacement first). +- **Package-version validation:** canonical numeric versions without leading zeros, registry component limits, and early first-publication checks before upload/transaction generation (#212). Keep chain-dependent publish checks separate from offline manifest checks. +- **Nested manifest fields:** report unknown keys with full paths, while preserving intentional open maps. Use shared schema metadata and distinguish source fragments from resolved releases. +- **Upstream identity and inheritance:** define duplicate-entry behavior, a stable identity for overrides, and any migration away from positional array merging before changing either consumer. +- **Compose `version:`** is obsolete. Rather than warn, the SDK should strip it from releases by default once versionless composes are verified with DAPPMANAGER (`--strip_compose_version` exists today as an opt-in). +- **Dockerfiles:** no `FROM …:latest` or untagged base images, and no remote `ADD` without `--checksum`. The staking `ADD ${STAKER_SCRIPTS_URL}/*.sh` pattern needs a shared replacement first. - **Declarative upstream metadata:** optional `upstream[].provider`, `tagPattern` and `imageTag` fields would let tropibot drop its built-in provider table and tag-to-image-tag inference. This needs a manifest schema change. - **ENS names of new variants:** `-..dappnode.eth` for new variants, with an allowlist of existing on-chain names (`hoodi-geth`, `nethermind-xdai`, …). -- **Explicit `architectures`** (25 repositories rely on the implicit `linux/amd64`). -- **Setup wizard targets:** every wizard target refers to a service and environment variable that exist in the compose (7 repositories reference variables the compose does not declare). +- **Explicit `architectures`**, replacing reliance on the implicit `linux/amd64`. +- **Setup wizard targets:** every wizard target refers to a service and environment variable that exist in the compose (service/environment existence should be checked after merging). - **One next-version rule** shared by the SDK (APM) and tropibot (DApp Explorer +1 patch). ## Consumer contradictions found @@ -108,9 +131,8 @@ These came up while defining the rules and should be fixed regardless of the rol - A custom `--compose_file_name` yields a release DAPPMANAGER cannot install (SPEC-C01). - A YAML manifest is released as `dappnode_package.yml`, which `sdk-publish` and the ISO installer cannot read (SPEC-M01). - The SDK reads notifications with any extension, DAPPMANAGER only `.yaml` (SPEC-F01). -- `globalEnvs: { all: true }` is documented and implemented in DAPPMANAGER but rejected by the schema; the docs should drop it. -- The manifest schema's `version` pattern has unescaped dots and `name` has no pattern; the SDK checks both itself (`validatePackageVersion`, SPEC-M02). -- The variants directory default is `variants` in the bump workflow and in tropibot's package creation guide, but `package_variants` everywhere else (SPEC-T02). +- The published global-env schema rejects the runtime’s legacy `{ all: true }` form and nests `services` under the `envs` schema instead of beside it. The shared-schema correction preserves the runtime form and requires both fields for explicit lists. Publish it together with the matching types. +- Published numeric-version patterns contain unescaped dots. The shared-schema correction uses literal separators for package and minimum-runtime versions. Canonical/range validation remains later-phase work; SPEC-M02 continues checking names in the SDK. ## tropibot @@ -129,11 +151,11 @@ tropibot and the SDK share most package handling. Once a rule is an error, tropi Issues in tropibot independent of the spec: -- Pin cleanup on branch close or delete uses only the root manifest `name`, so it errors for the 22 templates whose root has no name and misses variant packages. The SDK now cleans root and variant names (`cleanPinsFromDeletedBranches`); tropibot should do the same or leave cleanup to the SDK. +- Pin cleanup on branch close or delete uses only the root manifest `name`, so it errors for templates whose root has no name and misses variant packages. The SDK now cleans root and variant names (`cleanPinsFromDeletedBranches`); tropibot should do the same or leave cleanup to the SDK. - Pinata `pinList` is not paginated (default page size 10), so cleanup sees only the first page. - tropibot unpins when a PR closes even if its branch still exists; the SDK only cleans deleted branches. - `docs/PACKAGE_CREATION_GUIDE.md` recommends a `variants//` layout with per-variant Dockerfiles, which contradicts SPEC-T02 and every real template. -- Two upstream bumpers exist: tropibot (46 repositories via workflows) and the SDK's `github-action bump-upstream` (5 repositories). One should become canonical. +- Two upstream bumpers exist: tropibot and the SDK's `github-action bump-upstream`. One should become canonical. Contract between the SDK and tropibot that the spec work must keep: diff --git a/src/files/compose/types.ts b/src/files/compose/types.ts index db537c78..11d9cde2 100644 --- a/src/files/compose/types.ts +++ b/src/files/compose/types.ts @@ -1,8 +1,4 @@ -import { - ComposeNetworks, - ComposeService, - ComposeVolumes -} from "@dappnode/types"; +import { Compose, ComposeService } from "@dappnode/types"; export type OptionalComposeServiceProperties = Partial< Pick @@ -10,11 +6,8 @@ export type OptionalComposeServiceProperties = Partial< export type ComposeServiceFlexible = Omit & OptionalComposeServiceProperties; -export interface FlexibleCompose { - version: string; +export interface FlexibleCompose extends Omit { services: { [dnpName: string]: ComposeServiceFlexible; }; - networks?: ComposeNetworks; - volumes?: ComposeVolumes; } diff --git a/src/spec/checkPackageSpec.ts b/src/spec/checkPackageSpec.ts index e4faeb96..65d0fcb9 100644 --- a/src/spec/checkPackageSpec.ts +++ b/src/spec/checkPackageSpec.ts @@ -8,6 +8,10 @@ import { defaultComposeFileName, defaultVariantsDirName } from "../params.js"; import { parseGithubRepoSlug } from "../files/manifest/getRepoSlugFromManifest.js"; import { shell } from "../utils/shell.js"; import { SpecFinding, SpecRuleId } from "./rules.js"; +import { + checkManifestReferences, + manifestFieldIsDefined +} from "./manifestReferences.js"; const manifestFileName = "dappnode_package.json"; const packageNamePattern = @@ -384,27 +388,43 @@ export function checkPackageSpec({ ); continue; } - // "@sha256:" pins the same version by digest. - const matchesVersion = (value: string) => - value === String(upstream.version) || - value.startsWith(`${upstream.version}@sha256:`); - if (upstream.version !== undefined && ![...values].some(matchesVersion)) { - const argDir = composeOrigin(c => - Object.values(c.services || {}).some( - def => - isObject(def?.build) && - hasBuildArg(def.build, upstream.arg as string) - ) + // A digest suffix pins the same version only when its SHA-256 is complete. + const matchesVersion = (value: string) => { + const prefix = `${upstream.version}@sha256:`; + return ( + value === String(upstream.version) || + (value.startsWith(prefix) && + /^[a-f0-9]{64}$/i.test(value.slice(prefix.length))) ); - const variant = where(upstreamDir) || where(argDir); - add( - "SPEC-M05", - `Upstream ${upstream.repo}${variant} has version "${ - upstream.version - }" but build arg ${upstream.arg} is "${[...values].join('", "')}".`, - manifestFileIn(upstreamDir) - ); - } + }; + if (upstream.version !== undefined) + for (const [service, value] of values) { + if (matchesVersion(value)) continue; + const argDir = composeOrigin(c => { + const build = c.services?.[service]?.build; + return isObject(build) && hasBuildArg(build, upstream.arg); + }); + const variant = where(upstreamDir) || where(argDir); + add( + "SPEC-M05", + `Upstream ${upstream.repo}${variant} has version "${upstream.version}" but services.${service}.build.args.${upstream.arg} is "${value}"; every service using this arg must match (optionally with @sha256:<64 hex digits>).`, + composeFileIn(argDir) + ); + } + } + + for (const finding of checkManifestReferences( + sourceManifest as Manifest, + compose + )) { + const dir = manifestOrigin(m => manifestFieldIsDefined(m, finding.field)); + add( + finding.rule, + `${finding.message}${ + pkg.variant ? ` Checked for variant "${pkg.variant}".` : "" + }`, + manifestFileIn(dir) + ); } const services = Object.keys(compose.services || {}); @@ -600,25 +620,29 @@ function mergeByPosition(parts: T[]): T { return parts.reduce((acc, part) => merge(acc, part) as T, {} as T); } -/** Build arg name -> values across all services; list-style "KEY=value" args included. */ -function collectBuildArgs(compose: Compose): Map> { - const args = new Map>(); - const put = (key: string, value: unknown) => { - if (!args.has(key)) args.set(key, new Set()); +/** Build arg name -> service -> value; list-style "KEY=value" args included. */ +function collectBuildArgs(compose: Compose): Map> { + const args = new Map>(); + const put = (service: string, key: string, value: unknown) => { + if (!args.has(key)) args.set(key, new Map()); args .get(key) - ?.add(value === undefined || value === null ? "" : String(value)); + ?.set( + service, + value === undefined || value === null ? "" : String(value) + ); }; - for (const def of Object.values(compose.services || {})) { + for (const [service, def] of Object.entries(compose.services || {})) { if (!def || typeof def.build !== "object" || !def.build.args) continue; const buildArgs: unknown = def.build.args; if (Array.isArray(buildArgs)) for (const item of buildArgs) { const [key, ...rest] = String(item).split("="); - put(key, rest.join("=")); + put(service, key, rest.join("=")); } else if (isObject(buildArgs)) - for (const [key, value] of Object.entries(buildArgs)) put(key, value); + for (const [key, value] of Object.entries(buildArgs)) + put(service, key, value); } return args; } diff --git a/src/spec/manifestReferences.ts b/src/spec/manifestReferences.ts new file mode 100644 index 00000000..eb5cec4b --- /dev/null +++ b/src/spec/manifestReferences.ts @@ -0,0 +1,85 @@ +import { Compose, GLOBAL_ENVS, Manifest } from "@dappnode/types"; + +interface ManifestReferenceFinding { + rule: "SPEC-M10" | "SPEC-M11"; + field: string; + message: string; +} + +const globalEnvPrefix = "_DAPPNODE_GLOBAL_"; +// The installer prefixes manifest envs before looking them up in this catalogue. +// Use values rather than keys: ACTIVE, for example, maps to ENVS_ACTIVE. +const globalEnvNames = new Set( + Object.values(GLOBAL_ENVS).map(name => name.slice(globalEnvPrefix.length)) +); + +/** Checks the resolved manifest against its resolved compose, after variant merging. */ +export function checkManifestReferences( + manifest: Manifest, + compose: Compose +): ManifestReferenceFinding[] { + const findings: ManifestReferenceFinding[] = []; + const services = new Set(Object.keys(compose.services || {})); + const checkService = (field: string, value: unknown) => { + // Schema validation owns malformed values; spec checks must not throw first. + if (typeof value === "string" && !services.has(value)) + findings.push({ + rule: "SPEC-M10", + field, + message: `${field} refers to service "${value}", which is not in the compose.` + }); + }; + + checkService("mainService", manifest.mainService); + if (manifest.chain && typeof manifest.chain === "object") + checkService("chain.serviceName", manifest.chain.serviceName); + if (Array.isArray(manifest.backup)) + manifest.backup.forEach((entry, i) => + checkService(`backup[${i}].service`, entry?.service) + ); + + if (Array.isArray(manifest.globalEnvs)) + manifest.globalEnvs.forEach((entry, i) => { + if (Array.isArray(entry?.services)) + entry.services.forEach((service, j) => + checkService(`globalEnvs[${i}].services[${j}]`, service) + ); + if (Array.isArray(entry?.envs)) + entry.envs.forEach((env, j) => { + if (typeof env !== "string" || globalEnvNames.has(env)) return; + const shortName = env.startsWith(globalEnvPrefix) + ? env.slice(globalEnvPrefix.length) + : undefined; + const suggestion = + shortName && globalEnvNames.has(shortName) + ? ` Use "${shortName}"; DAPPMANAGER adds ${globalEnvPrefix} when installing.` + : " Use a short name from the shared GLOBAL_ENVS catalogue."; + const field = `globalEnvs[${i}].envs[${j}]`; + findings.push({ + rule: "SPEC-M11", + field, + message: `${field} requests unsupported global env "${env}".${suggestion}` + }); + }); + }); + + return findings; +} + +/** Used to attribute an inherited reference to the source fragment that defines it. */ +export function manifestFieldIsDefined( + manifest: Partial, + field: string +): boolean { + let value: unknown = manifest; + for (const key of field.replace(/\[(\d+)\]/g, ".$1").split(".")) { + if ( + value === null || + typeof value !== "object" || + !Object.hasOwn(value, key) + ) + return false; + value = (value as Record)[key]; + } + return value !== undefined; +} diff --git a/src/spec/rules.ts b/src/spec/rules.ts index e4a6660f..80a623d8 100644 --- a/src/spec/rules.ts +++ b/src/spec/rules.ts @@ -25,11 +25,11 @@ export const specRules = { }, "SPEC-M05": { title: "Upstream version equals the build arg value", - why: "Manifest and compose must describe the same upstream version; tools otherwise have to translate between tag formats." + why: "Every service using an upstream arg must match its declared version, optionally pinned by a complete SHA-256 digest." }, "SPEC-M06": { title: "Variant upstream[] matches the root order", - why: "Root and variant manifests are merged by array position, so a reordered or shortened variant upstream[] mixes entries." + why: "Root and variant manifests are merged by array position; reordered entries mix upstreams, while shorter lists inherit the root tail." }, "SPEC-M07": { title: "Only known manifest fields", @@ -43,6 +43,14 @@ export const specRules = { title: "Exposable services exist in compose", why: "An exposable entry for a service that is not in the compose cannot be exposed." }, + "SPEC-M10": { + title: "Manifest service references exist in compose", + why: "mainService, chain, backup and global-env service references must resolve after root and variant files are merged." + }, + "SPEC-M11": { + title: "Global environment names match the runtime catalogue", + why: "DAPPMANAGER prefixes short global-env names at install; unknown or already-prefixed names cannot be injected." + }, // Templates (multi-variant packages) "SPEC-T01": { title: "Template root manifest has no name or version", diff --git a/test/spec/checkPackageSpec.test.ts b/test/spec/checkPackageSpec.test.ts index 02f65953..66c6d0f7 100644 --- a/test/spec/checkPackageSpec.test.ts +++ b/test/spec/checkPackageSpec.test.ts @@ -222,7 +222,7 @@ describe("package spec checks", function () { build: { context: ".", args: { - UPSTREAM_VERSION: "1.0.0@sha256:abc", + UPSTREAM_VERSION: `1.0.0@sha256:${"a".repeat(64)}`, STAKER_SCRIPTS_VERSION: "v0.1.2" } }, @@ -287,6 +287,202 @@ describe("package spec checks", function () { expect(rulesOf(check(dir))).to.deep.equal(["SPEC-M06"]); }); + it("checks every service using an upstream arg, including digest pins", () => { + write("dappnode_package.json", { + name: "demo.dnp.dappnode.eth", + version: "0.1.0", + upstream: [{ repo: "acme/app", version: "1.2.3", arg: "APP_VERSION" }] + }); + for (const [value, valid] of [ + ["1.2.3", true], + [`1.2.3@sha256:${"a".repeat(64)}`, true], + ["1.2.2", false], + ["1.2.3@sha256:abc", false], + [`1.2.3@sha256:${"g".repeat(64)}`, false], + ["", false] + ] as const) { + write( + "docker-compose.yml", + yaml.dump({ + services: { + app: { build: { context: ".", args: { APP_VERSION: "1.2.3" } } }, + worker: { build: { context: ".", args: { APP_VERSION: value } } } + } + }) + ); + const findings = check(dir).filter(f => f.rule === "SPEC-M05"); + expect(findings, value).to.have.length(valid ? 0 : 1); + if (!valid) { + expect(findings[0].message).to.include( + "services.worker.build.args.APP_VERSION" + ); + expect(findings[0].file).to.equal(path.join(dir, "docker-compose.yml")); + } + } + }); + + it("attributes conflicting upstream values to the variant compose", () => { + write("dappnode_package.json", { + upstream: [{ repo: "acme/app", version: "1.2.3", arg: "APP_VERSION" }] + }); + write( + "docker-compose.yml", + yaml.dump({ + services: { + app: { build: { context: ".", args: { APP_VERSION: "1.2.3" } } } + } + }) + ); + write("package_variants/main/dappnode_package.json", { + name: "demo.dnp.dappnode.eth", + version: "0.1.0" + }); + write( + "package_variants/main/docker-compose.yml", + yaml.dump({ + services: { + worker: { build: { context: ".", args: { APP_VERSION: "1.2.2" } } } + } + }) + ); + const findings = check(dir).filter(f => f.rule === "SPEC-M05"); + expect(findings).to.have.length(1); + expect(findings[0].file).to.equal( + path.join(dir, "package_variants/main/docker-compose.yml") + ); + expect(findings[0].message).to.include('variant "main"'); + }); + + it("validates all explicit manifest service references", () => { + write("dappnode_package.json", { + name: "demo.dnp.dappnode.eth", + version: "0.1.0", + mainService: "missing-main", + chain: { driver: "ethereum", serviceName: "missing-chain" }, + backup: [{ name: "data", path: "/data", service: "missing-backup" }], + globalEnvs: [{ envs: ["PUBLIC_IP"], services: ["app", "missing-global"] }] + }); + write( + "docker-compose.yml", + yaml.dump({ services: { app: { image: "alpine:3.21" } } }) + ); + const findings = check(dir).filter(f => f.rule === "SPEC-M10"); + expect(findings).to.have.length(4); + for (const field of [ + "mainService", + "chain.serviceName", + "backup[0].service", + "globalEnvs[0].services[1]" + ]) + expect(findings.some(f => f.message.startsWith(field))).to.equal(true); + expect( + findings.every(f => f.file === path.join(dir, "dappnode_package.json")) + ).to.equal(true); + }); + + it("accepts valid references, implicit service defaults and runtime global-env names", () => { + write("dappnode_package.json", { + name: "demo.dnp.dappnode.eth", + version: "0.1.0", + mainService: "app", + chain: { driver: "ethereum", serviceName: "app" }, + backup: [ + { name: "data", path: "/data" }, + { name: "config", path: "/config", service: "app" } + ], + globalEnvs: [ + { + envs: ["PUBLIC_IP", "ENVS_ACTIVE", "EXECUTION_CLIENT_MAINNET"], + services: ["app"] + } + ] + }); + write( + "docker-compose.yml", + yaml.dump({ services: { app: { image: "alpine:3.21" } } }) + ); + expect(check(dir)).to.deep.equal([]); + write("dappnode_package.json", { + name: "demo.dnp.dappnode.eth", + version: "0.1.0", + chain: "ethereum", + globalEnvs: { all: true } + }); + expect(check(dir)).to.deep.equal([]); + }); + + it("rejects prefixed and unknown global-env names with actionable paths", () => { + write("dappnode_package.json", { + name: "demo.dnp.dappnode.eth", + version: "0.1.0", + globalEnvs: [ + { + envs: ["_DAPPNODE_GLOBAL_PUBLIC_IP", "PUBLIC_IPP", "ACTIVE"], + services: ["app"] + } + ] + }); + write( + "docker-compose.yml", + yaml.dump({ services: { app: { image: "alpine:3.21" } } }) + ); + const findings = check(dir).filter(f => f.rule === "SPEC-M11"); + expect(findings).to.have.length(3); + expect(findings[0].message) + .to.include("globalEnvs[0].envs[0]") + .and.include('Use "PUBLIC_IP"'); + expect(findings[1].message).to.include("PUBLIC_IPP"); + expect(findings[2].message).to.include('"ACTIVE"'); + }); + + it("checks references after merging and attributes individual array overrides", () => { + write("dappnode_package.json", { + mainService: "worker", + globalEnvs: [{ envs: ["PUBLIC_IP", "HOSTNAME"], services: ["app"] }] + }); + write( + "docker-compose.yml", + yaml.dump({ services: { app: { image: "alpine:3.21" } } }) + ); + for (const variant of ["good", "bad"]) { + write(`package_variants/${variant}/dappnode_package.json`, { + name: `demo-${variant}.dnp.dappnode.eth`, + version: "0.1.0", + ...(variant === "bad" + ? { globalEnvs: [{ envs: ["PUBLIC_IPP"], services: ["missing"] }] } + : {}) + }); + write( + `package_variants/${variant}/docker-compose.yml`, + yaml.dump({ + services: + variant === "good" + ? { worker: { image: "alpine:3.21" } } + : { app: {} } + }) + ); + } + const findings = check(dir).filter( + f => f.rule === "SPEC-M10" || f.rule === "SPEC-M11" + ); + expect(findings).to.have.length(3); + expect(findings.every(f => f.message.includes('variant "bad"'))).to.equal( + true + ); + expect( + findings.find(f => f.message.startsWith("mainService"))?.file + ).to.equal(path.join(dir, "dappnode_package.json")); + expect( + findings + .filter(f => f.message.startsWith("globalEnvs")) + .every( + f => + f.file === + path.join(dir, "package_variants/bad/dappnode_package.json") + ) + ).to.equal(true); + }); + it("lets core packages keep container_name, which the installer relies on", () => { write("dappnode_package.json", { name: "bind.dnp.dappnode.eth", @@ -333,6 +529,10 @@ describe("package spec checks", function () { it("collects findings in warn mode and fails validation only in error mode", async () => { await initHandler({ dir, yes: true, force: true }); + const manifestPath = path.join(dir, "dappnode_package.json"); + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); + manifest.mainService = "missing"; + fs.writeFileSync(manifestPath, JSON.stringify(manifest)); const compose = yaml.load( fs.readFileSync(path.join(dir, "docker-compose.yml"), "utf8") ) as { @@ -377,10 +577,11 @@ describe("package spec checks", function () { expect(getSpecMode({})).to.equal("warn"); const warn = await run(undefined); expect(warn.error).to.equal(null); - expect(rulesOf(warn.specFindings)).to.deep.equal(["SPEC-C02"]); + expect(rulesOf(warn.specFindings)).to.deep.equal(["SPEC-C02", "SPEC-M10"]); const strict = await run("error"); expect(strict.error?.message) .to.include("SPEC-C02") + .and.include("SPEC-M10") .and.include("DAPPNODE_SDK_SPEC=error"); const off = await run("off"); expect(off.error).to.equal(null);