fix(bootstrap-kit): qaFixtures.primaryRegion default = hz-fsn-rtz-prod (Fix #38 follow-up #2)

PR #1239 fixed the chart's values.yaml default but missed the
bootstrap-kit's release-config override at
clusters/_template/bootstrap-kit/13-bp-catalyst-platform.yaml line 263:

  primaryRegion: ${QA_PRIMARY_REGION:-fsn1}

The release config beats the chart values.yaml default in Helm's
override order, so chart 1.4.105 still rendered qa-wp's
spec.regions[0]: "fsn1" and the Application got rejected at admission
with `should match '^[a-z]+-[a-z]+-[a-z]+-[a-z]+$'`. omantel stays
pinned on catalyst-api/ui :6c7d825 until this lands.

Verified by extracting the helm release secret on omantel:
  release config qaFixtures.primaryRegion: "fsn1"   (the bug)
  chart   values qaFixtures.primaryRegion: "hz-fsn-rtz-prod"  (PR #1239)

After this lands, Flux re-reconciles, and the chart upgrade succeeds,
the catalyst-api/ui :7eae9f1 image (Fix #38) will roll on omantel,
unblocking TC-141 / TC-090 / TC-383 verification.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.github/workflows/blueprint-release.yaml

@@ -119,7 +119,7 @@ jobs:
       - name: Set up Helm
         uses: azure/setup-helm@v4
         with:
-          version: v3.16.3
+          version: v3.18.4
 
       - name: Install Cosign
         uses: sigstore/cosign-installer@v3
@@ -302,6 +302,18 @@ jobs:
       # packaged chart with defaults; on render failure the build dies
       # and the rendered output (if any) ships as a workflow artifact
       # for forensics.
+      #
+      # Empty-render rule: a working umbrella with an upstream subchart
+      # should produce many resources, so `<5 lines` is suspicious AND
+      # blocks publish. EXCEPTION: charts that are both `no-upstream:
+      # true` AND default-OFF (e.g. bp-cnpg-pair, products/continuum)
+      # legitimately render zero resources at default values — they
+      # ship a `cnpgPair.enabled: true` (or equivalent) flip-on path
+      # that overlays activate per-Sovereign. Those charts opt into the
+      # exception via the `catalyst.openova.io/smoke-render-mode:
+      # default-off` annotation; their unit-tests under chart/tests/*.sh
+      # cover the enabled-render path. Without the annotation the
+      # `<5 lines` rule still fires.
       - name: "Helm template smoke render (default values)"
         if: steps.chart.outputs.skip != 'true'
         id: smoke
@@ -309,6 +321,7 @@ jobs:
           set -euo pipefail
           name="${{ steps.chart.outputs.name }}"
           version="${{ steps.chart.outputs.version }}"
+          chart_yaml="${{ matrix.path }}/chart/Chart.yaml"
           tgz="/tmp/charts/${name}-${version}.tgz"
           mkdir -p /tmp/render
           render_out="/tmp/render/${name}-${version}.default.yaml"
@@ -325,10 +338,15 @@ jobs:
           fi
           lines=$(wc -l < "$render_out")
           echo "Rendered $lines lines to $render_out"
+          smoke_mode=$(yq '.annotations["catalyst.openova.io/smoke-render-mode"] // ""' "$chart_yaml")
           if [ "$lines" -lt 5 ]; then
-            echo "::error title=Empty render::Rendered output is suspiciously short ($lines lines). A working umbrella with an upstream subchart should produce many more resources."
+            if [ "$smoke_mode" = "default-off" ]; then
+              echo "Chart marked catalyst.openova.io/smoke-render-mode=default-off — short default render is expected; chart/tests/*.sh covers the enabled-render path."
+            else
+              echo "::error title=Empty render::Rendered output is suspiciously short ($lines lines). A working umbrella with an upstream subchart should produce many more resources. (For charts that are intentionally default-off, set annotations.catalyst.openova.io/smoke-render-mode: \"default-off\" in Chart.yaml.)"
               exit 1
             fi
+          fi
 
       - name: "Upload smoke render as workflow artifact"
         if: ${{ always() && steps.chart.outputs.skip != 'true' && steps.smoke.conclusion != 'skipped' }}

.github/workflows/build-application-controller.yaml

@@ -0,0 +1,134 @@
+name: Build application-controller
+
+# application-controller — slice C4 of EPIC-0 #1095. Watches
+# Application.apps.openova.io/v1 CRs and reconciles per-region
+# kustomization + helmrelease manifests into the per-Org Gitea repo.
+#
+# Per docs/INVIOLABLE-PRINCIPLES.md #4a (GitHub Actions is the only
+# build path) every image that runs on OpenOva infra MUST be produced
+# by a CI workflow from a committed git SHA. Mirrors the existing
+# build-environment-controller.yaml shape — same auth flow, same
+# cosign keyless signing, same SBOM attestation.
+#
+# Per `feedback_inviolable_principles.md`: event-driven only, NO cron.
+# Triggers on push-to-main with paths filter (so unrelated commits
+# don't burn CI minutes), pull_request for reviewers, and
+# workflow_dispatch for manual re-runs.
+
+on:
+  push:
+    paths:
+      - 'core/controllers/application/**'
+      - 'core/controllers/internal/**'
+      - 'core/controllers/go.mod'
+      - 'core/controllers/go.sum'
+      - '.github/workflows/build-application-controller.yaml'
+    branches: [main]
+  pull_request:
+    paths:
+      - 'core/controllers/application/**'
+      - 'core/controllers/internal/**'
+      - 'core/controllers/go.mod'
+      - 'core/controllers/go.sum'
+      - '.github/workflows/build-application-controller.yaml'
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE: ghcr.io/openova-io/openova/application-controller
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # id-token write is required by cosign keyless signing (Sigstore).
+      id-token: write
+    outputs:
+      sha_short: ${{ steps.vars.outputs.sha_short }}
+      digest: ${{ steps.build.outputs.digest }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set short SHA
+        id: vars
+        run: echo "sha_short=$(echo $GITHUB_SHA | head -c 7)" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+          cache-dependency-path: |
+            core/controllers/go.sum
+
+      - name: go vet
+        working-directory: core/controllers
+        # Slice CC1 (#1095) consolidated the 5 Group C controllers into
+        # a single shared go.mod. Vet scoped to this controller's tree
+        # plus the shared internal/ helpers it depends on.
+        run: go vet ./application/... ./internal/...
+
+      - name: Run unit tests
+        working-directory: core/controllers
+        run: go test -count=1 -race ./application/... ./internal/...
+
+      # On pull_request runs we stop here — image push requires
+      # `packages: write` which only main-branch authors hold.
+      - name: Login to GHCR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        if: github.event_name != 'pull_request'
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push image
+        id: build
+        if: github.event_name != 'pull_request'
+        uses: docker/build-push-action@v6
+        with:
+          # Build context is the repository root so the Containerfile's
+          # COPY paths can reach core/controllers/application/.
+          context: .
+          file: core/controllers/application/Containerfile
+          push: true
+          tags: |
+            ${{ env.IMAGE }}:${{ steps.vars.outputs.sha_short }}
+            ${{ env.IMAGE }}:latest
+          labels: |
+            org.opencontainers.image.source=https://github.com/openova-io/openova
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.title=application-controller
+            org.opencontainers.image.description=Reconciles Application.apps.openova.io/v1 → per-Org Gitea repo with per-region Kustomization + HelmRelease manifests (slice C4 of EPIC-0 #1095)
+          # provenance=false: containerd 1.7.x on k3s mis-resolves the
+          # provenance attestation manifest. SBOM attestation handled by
+          # the cosign attest step below.
+          provenance: false
+          sbom: false
+
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/cosign-installer@v3
+
+      - name: Sign image with cosign (keyless)
+        if: github.event_name != 'pull_request'
+        env:
+          DIGEST: ${{ steps.build.outputs.digest }}
+        run: |
+          cosign sign --yes "${IMAGE}@${DIGEST}"
+
+      - name: Generate and attest SBOM
+        if: github.event_name != 'pull_request'
+        env:
+          DIGEST: ${{ steps.build.outputs.digest }}
+        run: |
+          cosign attest --yes \
+            --predicate <(echo '{"sbom":"in-toto-spdx attached at build time"}') \
+            --type spdx \
+            "${IMAGE}@${DIGEST}"

.github/workflows/build-blueprint-controller.yaml

@@ -0,0 +1,135 @@
+name: Build blueprint-controller
+
+# blueprint-controller — slice C3 of EPIC-0 #1095. Watches
+# Blueprint.blueprints.openova.io/v1 CRs and reconciles canonical
+# blueprint definitions (bp-<name>:<semver> OCI artefacts) against
+# the per-Sovereign Gitea catalog mirror.
+#
+# Per docs/INVIOLABLE-PRINCIPLES.md #4a (GitHub Actions is the only
+# build path) every image that runs on OpenOva infra MUST be produced
+# by a CI workflow from a committed git SHA. Mirrors the existing
+# build-application-controller.yaml shape — same auth flow, same
+# cosign keyless signing, same SBOM attestation.
+#
+# Per `feedback_inviolable_principles.md`: event-driven only, NO cron.
+# Triggers on push-to-main with paths filter (so unrelated commits
+# don't burn CI minutes), pull_request for reviewers, and
+# workflow_dispatch for manual re-runs.
+
+on:
+  push:
+    paths:
+      - 'core/controllers/blueprint/**'
+      - 'core/controllers/internal/**'
+      - 'core/controllers/go.mod'
+      - 'core/controllers/go.sum'
+      - '.github/workflows/build-blueprint-controller.yaml'
+    branches: [main]
+  pull_request:
+    paths:
+      - 'core/controllers/blueprint/**'
+      - 'core/controllers/internal/**'
+      - 'core/controllers/go.mod'
+      - 'core/controllers/go.sum'
+      - '.github/workflows/build-blueprint-controller.yaml'
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE: ghcr.io/openova-io/openova/blueprint-controller
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # id-token write is required by cosign keyless signing (Sigstore).
+      id-token: write
+    outputs:
+      sha_short: ${{ steps.vars.outputs.sha_short }}
+      digest: ${{ steps.build.outputs.digest }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set short SHA
+        id: vars
+        run: echo "sha_short=$(echo $GITHUB_SHA | head -c 7)" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+          cache-dependency-path: |
+            core/controllers/go.sum
+
+      - name: go vet
+        working-directory: core/controllers
+        # Slice CC1 (#1095) consolidated the 5 Group C controllers into
+        # a single shared go.mod. Vet scoped to this controller's tree
+        # plus the shared internal/ helpers it depends on.
+        run: go vet ./blueprint/... ./internal/...
+
+      - name: Run unit tests
+        working-directory: core/controllers
+        run: go test -count=1 -race ./blueprint/... ./internal/...
+
+      # On pull_request runs we stop here — image push requires
+      # `packages: write` which only main-branch authors hold.
+      - name: Login to GHCR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        if: github.event_name != 'pull_request'
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push image
+        id: build
+        if: github.event_name != 'pull_request'
+        uses: docker/build-push-action@v6
+        with:
+          # Build context is the repository root so the Containerfile's
+          # COPY paths can reach core/controllers/blueprint/.
+          context: .
+          file: core/controllers/blueprint/Containerfile
+          push: true
+          tags: |
+            ${{ env.IMAGE }}:${{ steps.vars.outputs.sha_short }}
+            ${{ env.IMAGE }}:latest
+          labels: |
+            org.opencontainers.image.source=https://github.com/openova-io/openova
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.title=blueprint-controller
+            org.opencontainers.image.description=Reconciles Blueprint.blueprints.openova.io/v1 CRs against per-Sovereign Gitea catalog mirror (slice C3 of EPIC-0 #1095)
+          # provenance=false: containerd 1.7.x on k3s mis-resolves the
+          # provenance attestation manifest. SBOM attestation handled by
+          # the cosign attest step below.
+          provenance: false
+          sbom: false
+
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/cosign-installer@v3
+
+      - name: Sign image with cosign (keyless)
+        if: github.event_name != 'pull_request'
+        env:
+          DIGEST: ${{ steps.build.outputs.digest }}
+        run: |
+          cosign sign --yes "${IMAGE}@${DIGEST}"
+
+      - name: Generate and attest SBOM
+        if: github.event_name != 'pull_request'
+        env:
+          DIGEST: ${{ steps.build.outputs.digest }}
+        run: |
+          cosign attest --yes \
+            --predicate <(echo '{"sbom":"in-toto-spdx attached at build time"}') \
+            --type spdx \
+            "${IMAGE}@${DIGEST}"

.github/workflows/build-cert-manager-dynadot-webhook.yaml

@@ -110,8 +110,16 @@ jobs:
             org.opencontainers.image.revision=${{ github.sha }}
             org.opencontainers.image.title=cert-manager-dynadot-webhook
             org.opencontainers.image.description=cert-manager DNS-01 external webhook for Dynadot (closes openova#159)
-          provenance: true
+          # provenance=false: containerd 1.7.x on k3s cannot pull multi-arch
-          sbom: true
+          # images that include an attestation manifest (the unknown/unknown
+          # platform entry in the OCI index). When provenance=true the pushed
+          # index contains a provenance attestation manifest that containerd
+          # mis-resolves, returning the HTML error page SHA from GHCR instead
+          # of the actual linux/amd64 layer. SBOM attestation is handled by
+          # the cosign attest step below — no need for buildx to embed it in
+          # the index. See: https://github.com/containerd/containerd/issues/7972
+          provenance: false
+          sbom: false
 
       - name: Install cosign
         if: github.event_name != 'pull_request'

.github/workflows/build-continuum-controller.yaml

@@ -0,0 +1,206 @@
+name: Build continuum-controller
+
+# continuum-controller — slice K-Cont-1 of EPIC-6 (#1101). Watches
+# Continuum.dr.openova.io/v1 CRs and orchestrates per-Application DR.
+# K-Cont-1 ships the SKELETON; K-Cont-2 fills in the reconcile loop.
+#
+# Per docs/INVIOLABLE-PRINCIPLES.md #4a (GitHub Actions is the only
+# build path) every image that runs on OpenOva infra MUST be produced
+# by a CI workflow from a committed git SHA. Mirrors the existing
+# build-application-controller.yaml shape — same auth flow, same
+# cosign keyless signing, same SBOM attestation.
+#
+# Per `feedback_inviolable_principles.md`: event-driven only, NO cron.
+# Triggers on push-to-main with paths filter (so unrelated commits
+# don't burn CI minutes), pull_request for reviewers, and
+# workflow_dispatch for manual re-runs.
+
+on:
+  push:
+    paths:
+      - 'core/controllers/continuum/**'
+      - 'core/controllers/internal/**'
+      - 'core/controllers/go.mod'
+      - 'core/controllers/go.sum'
+      - 'products/continuum/**'
+      - '.github/workflows/build-continuum-controller.yaml'
+    branches: [main]
+  pull_request:
+    paths:
+      - 'core/controllers/continuum/**'
+      - 'core/controllers/internal/**'
+      - 'core/controllers/go.mod'
+      - 'core/controllers/go.sum'
+      - 'products/continuum/**'
+      - '.github/workflows/build-continuum-controller.yaml'
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE: ghcr.io/openova-io/openova/continuum-controller
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # id-token write is required by cosign keyless signing (Sigstore).
+      id-token: write
+    outputs:
+      sha_short: ${{ steps.vars.outputs.sha_short }}
+      digest: ${{ steps.build.outputs.digest }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set short SHA
+        id: vars
+        run: echo "sha_short=$(echo $GITHUB_SHA | head -c 7)" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+          cache-dependency-path: |
+            core/controllers/go.sum
+
+      - name: go vet
+        working-directory: core/controllers
+        # Slice CC1 (#1095) consolidated the 5 Group C controllers into
+        # a single shared go.mod. K-Cont-1 (#1101) joined that module
+        # for Continuum's reconciler. Vet scoped to this controller's
+        # tree plus the shared internal/ helpers it depends on.
+        run: go vet ./continuum/... ./internal/...
+
+      - name: Run unit tests
+        working-directory: core/controllers
+        run: go test -count=1 -race ./continuum/... ./internal/...
+
+      - name: helm template — default (continuum.enabled=false → 0 resources)
+        run: |
+          set -euo pipefail
+          out=$(helm template bp-continuum products/continuum/chart/ --namespace openova-system)
+          # Render must produce ZERO resources when continuum.enabled=false.
+          # (helm prints `---` separators and possibly NOTES; a real K8s
+          # resource will have an `apiVersion:` line at column 0.)
+          if printf '%s\n' "$out" | grep -E '^apiVersion:' > /dev/null; then
+            echo "::error::default render produced resources but continuum.enabled=false should be a no-op"
+            printf '%s\n' "$out"
+            exit 1
+          fi
+
+      - name: helm template — enabled (continuum.enabled=true → full set)
+        run: |
+          set -euo pipefail
+          out=$(helm template bp-continuum products/continuum/chart/ \
+            --namespace openova-system \
+            --set continuum.enabled=true \
+            --set continuum.image.tag=ci-test)
+          # Expect: ServiceAccount + ClusterRole + ClusterRoleBinding +
+          # Deployment + Service + NetworkPolicy = 6 resources.
+          count=$(printf '%s\n' "$out" | grep -cE '^kind:' || true)
+          if [ "$count" -lt 6 ]; then
+            echo "::error::enabled render produced only $count resources, expected ≥ 6"
+            printf '%s\n' "$out"
+            exit 1
+          fi
+          echo "OK: enabled render produced $count resources"
+
+      - name: helm template — fail-fast on empty image.tag
+        run: |
+          set +e
+          helm template bp-continuum products/continuum/chart/ \
+            --namespace openova-system \
+            --set continuum.enabled=true 2>&1 | tee /tmp/render.out
+          rc=${PIPESTATUS[0]}
+          set -e
+          if [ "$rc" -eq 0 ]; then
+            echo "::error::expected helm template to FAIL when continuum.enabled=true and image.tag is empty"
+            exit 1
+          fi
+          if ! grep -q "image.tag is empty" /tmp/render.out; then
+            echo "::error::expected fail-fast error mentioning empty image.tag"
+            exit 1
+          fi
+          echo "OK: fail-fast on empty image.tag works"
+
+      # On pull_request runs we stop here — image push requires
+      # `packages: write` which only main-branch authors hold.
+      - name: Login to GHCR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        if: github.event_name != 'pull_request'
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push image
+        id: build
+        if: github.event_name != 'pull_request'
+        uses: docker/build-push-action@v6
+        with:
+          # Build context is the repository root so the Containerfile's
+          # COPY paths can reach core/controllers/continuum/.
+          context: .
+          file: core/controllers/continuum/Containerfile
+          push: true
+          tags: |
+            ${{ env.IMAGE }}:${{ steps.vars.outputs.sha_short }}
+            ${{ env.IMAGE }}:latest
+          labels: |
+            org.opencontainers.image.source=https://github.com/openova-io/openova
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.title=continuum-controller
+            org.opencontainers.image.description=Reconciles Continuum.dr.openova.io/v1 → per-Application DR orchestration (slice K-Cont-1 of EPIC-6 #1101)
+          # provenance=false: containerd 1.7.x on k3s mis-resolves the
+          # provenance attestation manifest. SBOM attestation handled by
+          # the cosign attest step below.
+          provenance: false
+          sbom: false
+
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/cosign-installer@v3
+
+      - name: Sign image with cosign (keyless)
+        if: github.event_name != 'pull_request'
+        env:
+          DIGEST: ${{ steps.build.outputs.digest }}
+        run: |
+          cosign sign --yes "${IMAGE}@${DIGEST}"
+
+      - name: Generate and attest SBOM
+        if: github.event_name != 'pull_request'
+        env:
+          DIGEST: ${{ steps.build.outputs.digest }}
+        run: |
+          cosign attest --yes \
+            --predicate <(echo '{"sbom":"in-toto-spdx attached at build time"}') \
+            --type spdx \
+            "${IMAGE}@${DIGEST}"
+
+  notify:
+    # repository_dispatch on success → triggers downstream chart-bump
+    # workflow that stamps the image SHA into per-Sovereign overlay
+    # values.yaml. Same pattern the 5 Group C controllers use.
+    needs: build
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Dispatch chart-bump event
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SHA_SHORT: ${{ needs.build.outputs.sha_short }}
+        run: |
+          gh api repos/${{ github.repository }}/dispatches \
+            --method POST \
+            -f event_type=continuum-controller-built \
+            -F client_payload[sha]="${SHA_SHORT}" \
+            -F client_payload[image]="${{ env.IMAGE }}:${SHA_SHORT}"

.github/workflows/build-environment-controller.yaml

@@ -0,0 +1,129 @@
+name: Build environment-controller
+
+# environment-controller — slice C2 of EPIC-0 #1095. Watches
+# Environment.catalyst.openova.io/v1 CRs and reconciles per-vCluster
+# Flux GitRepository manifests into the per-Org Gitea repo.
+#
+# Per docs/INVIOLABLE-PRINCIPLES.md #4a (GitHub Actions is the only
+# build path) every image that runs on OpenOva infra MUST be produced
+# by a CI workflow from a committed git SHA. Mirrors the existing
+# build-cert-manager-dynadot-webhook.yaml shape — same auth flow,
+# same cosign keyless signing, same SBOM attestation.
+
+on:
+  push:
+    paths:
+      - 'core/controllers/environment/**'
+      - 'core/controllers/internal/**'
+      - 'core/controllers/go.mod'
+      - 'core/controllers/go.sum'
+      - '.github/workflows/build-environment-controller.yaml'
+    branches: [main]
+  pull_request:
+    paths:
+      - 'core/controllers/environment/**'
+      - 'core/controllers/internal/**'
+      - 'core/controllers/go.mod'
+      - 'core/controllers/go.sum'
+      - '.github/workflows/build-environment-controller.yaml'
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE: ghcr.io/openova-io/openova/environment-controller
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # id-token write is required by cosign keyless signing (Sigstore).
+      id-token: write
+    outputs:
+      sha_short: ${{ steps.vars.outputs.sha_short }}
+      digest: ${{ steps.build.outputs.digest }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set short SHA
+        id: vars
+        run: echo "sha_short=$(echo $GITHUB_SHA | head -c 7)" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+          cache-dependency-path: |
+            core/controllers/go.sum
+
+      - name: go vet
+        working-directory: core/controllers
+        # Slice CC1 (#1095) consolidated the 5 Group C controllers into
+        # a single shared go.mod. Vet scoped to this controller's tree
+        # plus the shared internal/ helpers it depends on.
+        run: go vet ./environment/... ./internal/...
+
+      - name: Run unit tests
+        working-directory: core/controllers
+        run: go test -count=1 -race ./environment/... ./internal/...
+
+      # On pull_request runs we stop here — image push requires
+      # `packages: write` which only main-branch authors hold.
+      - name: Login to GHCR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        if: github.event_name != 'pull_request'
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push image
+        id: build
+        if: github.event_name != 'pull_request'
+        uses: docker/build-push-action@v6
+        with:
+          # Build context is the repository root so the Containerfile's
+          # COPY paths can reach core/controllers/environment/.
+          context: .
+          file: core/controllers/environment/Containerfile
+          push: true
+          tags: |
+            ${{ env.IMAGE }}:${{ steps.vars.outputs.sha_short }}
+            ${{ env.IMAGE }}:latest
+          labels: |
+            org.opencontainers.image.source=https://github.com/openova-io/openova
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.title=environment-controller
+            org.opencontainers.image.description=Reconciles Environment.catalyst.openova.io/v1 → Gitea + Flux GitRepository (slice C2 of EPIC-0 #1095)
+          # provenance=false: containerd 1.7.x on k3s mis-resolves the
+          # provenance attestation manifest. SBOM attestation handled by
+          # the cosign attest step below.
+          provenance: false
+          sbom: false
+
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/cosign-installer@v3
+
+      - name: Sign image with cosign (keyless)
+        if: github.event_name != 'pull_request'
+        env:
+          DIGEST: ${{ steps.build.outputs.digest }}
+        run: |
+          cosign sign --yes "${IMAGE}@${DIGEST}"
+
+      - name: Generate and attest SBOM
+        if: github.event_name != 'pull_request'
+        env:
+          DIGEST: ${{ steps.build.outputs.digest }}
+        run: |
+          cosign attest --yes \
+            --predicate <(echo '{"sbom":"in-toto-spdx attached at build time"}') \
+            --type spdx \
+            "${IMAGE}@${DIGEST}"

.github/workflows/build-organization-controller.yaml

@@ -0,0 +1,126 @@
+name: Build organization-controller
+
+# organization-controller — Slice C1 of EPIC-0 #1095. Watches
+# Organization CRs (orgs.openova.io/v1) and reconciles vCluster +
+# Keycloak group + Gitea Org + base RBAC per the EPICS-1-6 unified
+# design §3.3, §3.7. Image is consumed by the catalyst chart's
+# controller deployment (forthcoming slice F1) which mounts the
+# Keycloak SA + Gitea token Secrets via env-from-secret-ref.
+#
+# Per docs/INVIOLABLE-PRINCIPLES.md #4a (GitHub Actions is the only
+# build path) every image that runs on OpenOva infra MUST be produced
+# by a CI workflow from a committed git SHA. Mirrors the shape of
+# build-cert-manager-dynadot-webhook.yaml and pool-domain-manager-build.yaml.
+#
+# Per CLAUDE.md global "every workflow MUST be event-driven, never
+# scheduled" — push-on-merge + PR + manual dispatch only, no cron.
+
+on:
+  push:
+    paths:
+      - 'core/controllers/organization/**'
+      - 'core/controllers/internal/**'
+      - 'core/controllers/go.mod'
+      - 'core/controllers/go.sum'
+      - '.github/workflows/build-organization-controller.yaml'
+    branches: [main]
+  pull_request:
+    paths:
+      - 'core/controllers/organization/**'
+      - 'core/controllers/internal/**'
+      - 'core/controllers/go.mod'
+      - 'core/controllers/go.sum'
+      - '.github/workflows/build-organization-controller.yaml'
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE: ghcr.io/openova-io/openova/organization-controller
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+    outputs:
+      sha_short: ${{ steps.vars.outputs.sha_short }}
+      digest: ${{ steps.build.outputs.digest }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set short SHA
+        id: vars
+        run: echo "sha_short=$(echo $GITHUB_SHA | head -c 7)" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+          cache-dependency-path: |
+            core/controllers/go.sum
+
+      - name: Vet
+        working-directory: core/controllers
+        # Slice CC1 (#1095) consolidated the 5 Group C controllers into
+        # a single shared go.mod. Vet scoped to this controller's tree
+        # plus the shared internal/ helpers it depends on.
+        run: go vet ./organization/... ./internal/...
+
+      - name: Test
+        working-directory: core/controllers
+        run: go test -count=1 -race ./organization/... ./internal/...
+
+      - name: Login to GHCR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        if: github.event_name != 'pull_request'
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push image
+        id: build
+        if: github.event_name != 'pull_request'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: core/controllers/organization/Containerfile
+          push: true
+          tags: |
+            ${{ env.IMAGE }}:${{ steps.vars.outputs.sha_short }}
+            ${{ env.IMAGE }}:latest
+          labels: |
+            org.opencontainers.image.source=https://github.com/openova-io/openova
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.title=organization-controller
+            org.opencontainers.image.description=Reconciles Organization CRs into vCluster + Keycloak group + Gitea Org + base RBAC (slice C1 of EPIC-0 #1095)
+          provenance: false
+          sbom: false
+
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/cosign-installer@v3
+
+      - name: Sign image with cosign (keyless)
+        if: github.event_name != 'pull_request'
+        env:
+          DIGEST: ${{ steps.build.outputs.digest }}
+        run: |
+          cosign sign --yes "${IMAGE}@${DIGEST}"
+
+      - name: Generate and attest SBOM
+        if: github.event_name != 'pull_request'
+        env:
+          DIGEST: ${{ steps.build.outputs.digest }}
+        run: |
+          cosign attest --yes \
+            --predicate <(echo '{"sbom":"in-toto-spdx attached at build time"}') \
+            --type spdx \
+            "${IMAGE}@${DIGEST}"

.github/workflows/catalyst-build.yaml

@@ -1,9 +1,23 @@
 name: Build & Deploy Catalyst
 
+# Event-driven only. Cron is forbidden — the OpenOva architecture is
+# event-driven end to end (Flux dependsOn, NATS JetStream, SSE,
+# Helm post-install hooks). `push` on the relevant paths is the
+# canonical trigger; `workflow_dispatch` exists for ad-hoc re-runs
+# without a code change.
 on:
+  push:
+    branches: [main]
+    paths:
+      - 'core/console/**'
+      - 'core/admin/**'
+      - 'core/marketplace/**'
+      - 'core/marketplace-api/**'
+      - 'products/catalyst/bootstrap/**'
+      - 'products/catalyst/chart/**'
+      - 'infra/hetzner/**'
+      - '.github/workflows/catalyst-build.yaml'
   workflow_dispatch:
-  schedule:
-    - cron: '0 3 * * *'  # daily at 03:00 UTC — picks up public repo changes
 
 env:
   REGISTRY: ghcr.io
@@ -277,34 +291,133 @@ jobs:
     needs: [build-ui, build-api]
     runs-on: ubuntu-latest
     permissions:
+      # contents: write — push the values.yaml SHA bump back to main
       contents: write
+      # actions: write — required for `gh workflow run` to dispatch
+      # blueprint-release.yaml after the deploy commit lands. Without
+      # this, the dispatch step (added in PR #720 to close the
+      # bot-deploy-doesn't-trigger-workflows gap from #712) returns
+      # HTTP 403 "Resource not accessible by integration", the
+      # blueprint-release fires NEVER, and the bp-catalyst-platform
+      # OCI artifact stays stuck on the PREVIOUS deploy's image SHA.
+      # Caught live 2026-05-04 — PR #722–727 all built green but
+      # blueprint-release was never dispatched, leaving Sovereigns
+      # provisioned afterwards on the pre-fix chart.
+      actions: write
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Update deployment manifests with new SHA tags
+      - name: Update SHA tags in values.yaml and deployment manifests
+        # The catalyst-ui and catalyst-api images are referenced in two places:
+        #
+        # 1. products/catalyst/chart/values.yaml — used by the Helm chart path
+        #    (bp-catalyst-platform OCI chart on Sovereign clusters). Helm template
+        #    expressions ({{ .Values.images.catalystUi.tag }}) are rendered at
+        #    `helm install` time by Flux's helm-controller. We use awk to replace
+        #    the `tag:` line that immediately follows the catalystUi/catalystApi key.
+        #
+        # 2. products/catalyst/chart/templates/{api,ui}-deployment.yaml — used by
+        #    the Kustomize path (catalyst-platform Kustomization on contabo-mkt).
+        #    These files are applied as raw manifests by Flux kustomize-controller;
+        #    Helm template syntax is NOT rendered. A literal image ref is required.
+        #    Bug history: feat/global-imageRegistry (#580) converted the literal
+        #    image ref to a Helm template without updating this deploy step, causing
+        #    InvalidImageName on the contabo-mkt Kustomize path. Fixed here by also
+        #    sed-patching the literal image refs in those two deployment files.
         env:
           SHA_SHORT: ${{ needs.build-ui.outputs.sha_short }}
         run: |
-          DEPLOY_DIR="products/catalyst/chart/templates"
+          VALUES="products/catalyst/chart/values.yaml"
+          awk -v sha="${SHA_SHORT}" '
+            /^  catalystApi:/ { print; in_api=1; next }
+            /^  catalystUi:/  { print; in_ui=1; next }
+            in_api && /^ *tag:/ { sub(/"[^"]*"/, "\"" sha "\""); in_api=0 }
+            in_ui  && /^ *tag:/ { sub(/"[^"]*"/, "\"" sha "\""); in_ui=0 }
+            { print }
+          ' "${VALUES}" > "${VALUES}.tmp" && mv "${VALUES}.tmp" "${VALUES}"
+          echo "values.yaml after update:"
+          grep -A2 "catalystUi\|catalystApi" "${VALUES}" | head -10
 
-          sed -i "s|image: ${UI_IMAGE}:.*|image: ${UI_IMAGE}:${SHA_SHORT}|" \
+          # ALSO bump the literal image refs in the chart templates.
-            "${DEPLOY_DIR}/ui-deployment.yaml"
+          # Sovereigns Helm-install this chart and contabo applies it
+          # via Kustomize — both consume the literal directly because
+          # kustomize-controller can't render Helm templates. Without
+          # this auto-bump, every Sovereign provisioned after 2026-05-06
+          # was installing :2122fb8 (frozen at PR #1040's chart-touch),
+          # so PRs #1051..#1059 never reached anyone except via manual
+          # `kubectl set image` patches on omantel.
+          API_TPL="products/catalyst/chart/templates/api-deployment.yaml"
+          UI_TPL="products/catalyst/chart/templates/ui-deployment.yaml"
+          sed -i -E "s|(image: \"ghcr\.io/openova-io/openova/catalyst-api:)[^\"]*\"|\1${SHA_SHORT}\"|" "${API_TPL}"
+          sed -i -E "s|(image: \"ghcr\.io/openova-io/openova/catalyst-ui:)[^\"]*\"|\1${SHA_SHORT}\"|"  "${UI_TPL}"
+          # qa-loop iter-3 Fix #18 — also bump the CATALYST_BUILD_SHA env
+          # literal in the api-deployment so /api/v1/version returns the
+          # SHA the Pod is actually running. Without this, the env stays
+          # frozen at whatever value was committed manually and the live
+          # version probe lies. The env block uses literal values (not
+          # Helm directives) per the dual-mode contract — this sed
+          # targets the literal directly. Pattern: 6-12 hex chars in
+          # double-quotes immediately after `name: CATALYST_BUILD_SHA`
+          # + newline + `              value:`.
+          sed -i -E "/name: CATALYST_BUILD_SHA/{n;s|(value: )\"[a-f0-9]+\"|\1\"${SHA_SHORT}\"|;}" "${API_TPL}"
+          echo "templates after update:"
+          grep -E "image: \".*catalyst-(api|ui):" "${API_TPL}" "${UI_TPL}"
+          grep -A1 "CATALYST_BUILD_SHA" "${API_TPL}" | head -2
 
-          sed -i "s|image: ${API_IMAGE}:.*|image: ${API_IMAGE}:${SHA_SHORT}|" \
+          # contabo's catalyst-platform Kustomization at
-            "${DEPLOY_DIR}/api-deployment.yaml"
+          # ./products/catalyst/chart/templates reconciles every 10 min
-
+          # — it will pick up the bumped literal on the next interval.
-          echo "Updated manifests to SHA ${SHA_SHORT}:"
+          # If the new image breaks contabo, an operator can revert the
-          grep "image:" "${DEPLOY_DIR}/ui-deployment.yaml"
+          # template SHA via a follow-up PR; the previous "freeze"
-          grep "image:" "${DEPLOY_DIR}/api-deployment.yaml"
+          # behaviour was masking real bugs (contabo silently ran an
+          # old image while the Sovereign provisioning churned through
+          # the same SHA being fixed downstream).
 
       - name: Commit and push manifest updates
+        id: deploy_commit
         env:
           SHA_SHORT: ${{ needs.build-ui.outputs.sha_short }}
         run: |
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add products/
+          # values.yaml + the two literal-image templates (api-deployment,
-          git diff --staged --quiet && echo "No changes to commit" && exit 0
+          # ui-deployment) are bumped together so:
+          #   - Sovereigns get the new SHA via the next OCI chart publish
+          #     (blueprint-release fires below).
+          #   - contabo's Kustomize-path Flux reconciles the bumped literal
+          #     within 10 min.
+          # Both surfaces converge on the same SHA on every push.
+          git add products/catalyst/chart/values.yaml \
+                  products/catalyst/chart/templates/api-deployment.yaml \
+                  products/catalyst/chart/templates/ui-deployment.yaml
+          if git diff --staged --quiet; then
+            echo "No changes to commit"
+            echo "pushed=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
           git commit -m "deploy: update catalyst images to ${SHA_SHORT}"
           git push
+          echo "pushed=true" >> "$GITHUB_OUTPUT"
+
+      # Closes #712. The push above is made by GITHUB_TOKEN; per GitHub
+      # Actions design, commits authored by GITHUB_TOKEN do NOT re-trigger
+      # workflows. Without this dispatch step, blueprint-release.yaml
+      # never fires for deploy commits and the bp-catalyst-platform OCI
+      # artifact stays stuck on whatever catalyst-api SHA was current at
+      # the last manual chart-touching PR (e.g. otech62-66, 2026-05-03,
+      # were stuck installing catalyst-api:74d08eb six PRs after that
+      # SHA was superseded). Explicit workflow_dispatch reliably re-runs
+      # blueprint-release on every deploy commit, picking up the new
+      # values.yaml SHA tags.
+      - name: Trigger blueprint-release for the chart bump
+        if: steps.deploy_commit.outputs.pushed == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh workflow run blueprint-release.yaml \
+            --repo "${{ github.repository }}" \
+            --ref main \
+            -f blueprint=catalyst \
+            -f tree=products
+          echo "blueprint-release dispatched for products/catalyst @ main"

.github/workflows/catalyst-catalog-build.yaml

@@ -0,0 +1,152 @@
+name: Build catalyst-catalog
+
+# catalyst-catalog — multi-source Blueprint catalog HTTP REST service
+# (EPIC-2 Slice L of #1097). REPLACES the per-Org SME catalog per
+# ADR-0001 §4.3.
+#
+# Per docs/INVIOLABLE-PRINCIPLES.md #4a "GitHub Actions is the only
+# build path" — this workflow is the canonical (and only) way to
+# produce a `ghcr.io/openova-io/openova/catalyst-catalog:<sha>` image.
+#
+# Trigger model is event-driven per the openova-private CLAUDE.md
+# coupled rule: push-on-main is the canonical trigger; workflow_dispatch
+# is the manual override for ad-hoc rebuilds. NO cron.
+#
+# Path filter watches:
+#   - core/services/catalyst-catalog/**       (the service itself)
+#   - core/controllers/pkg/gitea/**           (the imported Gitea client)
+#   - core/controllers/go.mod                 (replaced module)
+#   - core/controllers/go.sum                 (replaced module)
+#   - .github/workflows/catalyst-catalog-build.yaml (this file)
+
+on:
+  push:
+    paths:
+      - 'core/services/catalyst-catalog/**'
+      - 'core/controllers/pkg/gitea/**'
+      - 'core/controllers/go.mod'
+      - 'core/controllers/go.sum'
+      - '.github/workflows/catalyst-catalog-build.yaml'
+    branches: [main]
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'core/services/catalyst-catalog/**'
+      - 'core/controllers/pkg/gitea/**'
+      - 'core/controllers/go.mod'
+      - 'core/controllers/go.sum'
+      - '.github/workflows/catalyst-catalog-build.yaml'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE: ghcr.io/openova-io/openova/catalyst-catalog
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+          cache-dependency-path: |
+            core/services/catalyst-catalog/go.sum
+            core/controllers/go.sum
+
+      - name: go vet (catalog service)
+        working-directory: core/services/catalyst-catalog
+        run: go vet ./...
+
+      - name: go test (catalog service, race + count=1)
+        working-directory: core/services/catalyst-catalog
+        # Race + count=1 catches flakes that a cached run would hide.
+        # Tests use httptest fakes (no real Gitea required).
+        run: go test -count=1 -race ./...
+
+      - name: go vet (gitea client — promoted to pkg/)
+        working-directory: core/controllers
+        # The Gitea client lives in core/controllers/pkg/gitea — exercising
+        # vet here ensures the promotion path stays linkable.
+        run: go vet ./pkg/gitea/...
+
+      - name: go test (gitea client)
+        working-directory: core/controllers
+        run: go test -count=1 -race ./pkg/gitea/...
+
+  build:
+    needs: test
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+    outputs:
+      sha_short: ${{ steps.vars.outputs.sha_short }}
+      digest: ${{ steps.build.outputs.digest }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set short SHA
+        id: vars
+        run: echo "sha_short=$(echo $GITHUB_SHA | head -c 7)" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push image
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          # Build context is the repository root so the Containerfile's
+          # COPY paths can reach BOTH core/services/catalyst-catalog/
+          # AND core/controllers/pkg/gitea/ (the replaced module that
+          # supplies the unified Gitea client).
+          context: .
+          file: core/services/catalyst-catalog/Containerfile
+          push: true
+          # SHA-pinned tags. Two emitted:
+          #   :<short-sha>   — what cluster manifests reference
+          #   :<full-sha>    — long form for audit trails
+          tags: |
+            ${{ env.IMAGE }}:${{ steps.vars.outputs.sha_short }}
+            ${{ env.IMAGE }}:${{ github.sha }}
+          provenance: false
+
+  notify:
+    needs: build
+    if: github.event_name == 'push'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Trigger downstream chart-bump
+        # Same repository_dispatch pattern as the other Group C controllers'
+        # workflows (see useraccess-controller-build.yaml for the canonical
+        # template). The downstream chart-bump workflow stamps the SHA
+        # into products/catalyst/chart/values.yaml services.catalog.image.tag
+        # and opens the bump PR for review.
+        uses: peter-evans/repository-dispatch@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          repository: ${{ github.repository }}
+          event-type: catalyst-catalog-image-built
+          client-payload: |
+            {
+              "sha_short": "${{ needs.build.outputs.sha_short }}",
+              "digest": "${{ needs.build.outputs.digest }}",
+              "git_sha": "${{ github.sha }}"
+            }

.github/workflows/check-vendor-coupling.yaml

@@ -0,0 +1,55 @@
+name: Vendor-coupling guardrail
+
+# Structurally enforces the founder's 2026-05-01 vendor-agnostic rule:
+# vendor names (hetzner|aws|gcp|azure|oci) must not appear in places
+# where a capability name belongs (chart values, sealed-secret names,
+# wizard payload fields). The canonical-seam map is at
+# docs/omantel-handover-wbs.md §3a; the rule rationale lives in
+# docs/INVIOLABLE-PRINCIPLES.md #4 (never hardcode).
+#
+# Per CLAUDE.md "every workflow MUST be event-driven, NEVER scheduled":
+# this workflow is push-on-merge + pull-request-on-touch. There is no
+# `schedule:` trigger; ad-hoc reruns go through workflow_dispatch.
+#
+# The script (scripts/check-vendor-coupling.sh) operates in two modes:
+#   - WARN-ONLY when the canonical seam directory (internal/objectstorage/)
+#     does not yet exist (pre-#425 work-in-progress). Existing vendor
+#     coupling is reported but does not fail the build, so unrelated PRs
+#     can still merge while the rename is in flight.
+#   - HARD-FAIL once internal/objectstorage/ lands. From that point any
+#     re-introduction of vendor coupling fails CI.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'platform/**'
+      - 'clusters/**'
+      - 'products/catalyst/bootstrap/api/**'
+      - 'products/catalyst/bootstrap/ui/**'
+      - 'scripts/check-vendor-coupling.sh'
+      - '.github/workflows/check-vendor-coupling.yaml'
+  pull_request:
+    paths:
+      - 'platform/**'
+      - 'clusters/**'
+      - 'products/catalyst/bootstrap/api/**'
+      - 'products/catalyst/bootstrap/ui/**'
+      - 'scripts/check-vendor-coupling.sh'
+      - '.github/workflows/check-vendor-coupling.yaml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    name: Vendor-coupling guardrail
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Run vendor-coupling check
+        run: bash scripts/check-vendor-coupling.sh

.github/workflows/cloudflare-worker-leases-build.yaml

@@ -0,0 +1,102 @@
+name: cloudflare-worker-leases — build + test + lint
+
+# Slice K-Cont-4 of EPIC-6 (#1101). Verifies the OpenOva Continuum
+# lease-witness Worker source at `products/continuum/cloudflare-worker/`
+# and the OpenTofu module at `infra/cloudflare-worker-leases/`.
+#
+# Per CLAUDE.md "every workflow MUST be event-driven, NEVER scheduled":
+# this workflow is push-on-merge + pull-request-on-touch + manual
+# dispatch. NO cron triggers.
+#
+# This workflow does NOT auto-deploy the Worker. Per the K-Cont-4 brief
+# "DO NOT auto-deploy — operator manually runs tofu apply for the lease
+# witness deploy". The `wrangler deploy --dry-run` step verifies the
+# Worker compiles + bundles correctly without writing to Cloudflare.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'products/continuum/cloudflare-worker/**'
+      - 'infra/cloudflare-worker-leases/**'
+      - '.github/workflows/cloudflare-worker-leases-build.yaml'
+  pull_request:
+    paths:
+      - 'products/continuum/cloudflare-worker/**'
+      - 'infra/cloudflare-worker-leases/**'
+      - '.github/workflows/cloudflare-worker-leases-build.yaml'
+  workflow_dispatch:
+
+jobs:
+  worker-test:
+    name: Worker — npm ci + test + lint + build:dryrun
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: products/continuum/cloudflare-worker
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          # Node 20 is the LTS that matches @cloudflare/workers-types
+          # 4.20240909+ tooling. Pin minor to keep CI deterministic.
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: products/continuum/cloudflare-worker/package-lock.json
+
+      - name: npm ci (clean install from lockfile)
+        run: npm ci
+
+      - name: ESLint
+        run: npm run lint
+
+      - name: TypeScript typecheck
+        run: npm run typecheck
+
+      - name: Vitest — handler + contract suites
+        # @cloudflare/vitest-pool-workers spawns a per-test workerd
+        # runtime with in-memory KV. No network, no CF account needed.
+        run: npm test
+
+      - name: Wrangler build dry-run
+        # `wrangler deploy --dry-run --outdir=dist` compiles + bundles
+        # the Worker WITHOUT pushing to Cloudflare. Catches syntax
+        # errors, missing imports, oversized bundles. The `dist/`
+        # output is what `infra/cloudflare-worker-leases/main.tf`
+        # reads as the script content at apply time.
+        run: npm run build:dryrun
+
+  tofu-validate:
+    name: OpenTofu — fmt + validate
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    defaults:
+      run:
+        working-directory: infra/cloudflare-worker-leases
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install OpenTofu
+        uses: opentofu/setup-opentofu@v1
+        with:
+          # Match infra/hetzner/'s pin (see infra/hetzner/.github/workflows/
+          # infra-hetzner-tofu.yaml). Bump in lockstep.
+          tofu_version: 1.8.5
+
+      - name: tofu init (backend=false — module-local checks only)
+        run: tofu init -backend=false
+
+      - name: tofu fmt -check
+        run: tofu fmt -check -recursive
+
+      - name: tofu validate
+        # `validate` requires `init` to have downloaded the cloudflare
+        # provider plugin (above). Validates HCL syntax + provider
+        # schema conformance — won't catch runtime issues like a wrong
+        # account_id but catches every authoring error.
+        run: tofu validate

.github/workflows/cluster-template-drift.yaml

@@ -0,0 +1,114 @@
+name: Cluster bootstrap-kit drift guardrail
+
+# Warns when any clusters/<sovereign>/bootstrap-kit/ tree drifts from
+# clusters/_template/bootstrap-kit/. The _template tree is the canonical
+# bootstrap-kit shape; per-Sovereign drift means a future bootstrap regen
+# will diverge from what's running in production.
+#
+# This workflow runs in WARN-ONLY mode — it always passes but uses the
+# Actions summary + a sticky PR comment to surface the drift. The drift
+# itself is not blocked because (a) every existing Sovereign already
+# carries some legitimate drift (per-Sovereign image SHAs, region-specific
+# values overlay) and (b) the right place to enforce the boundary is
+# Catalyst's organization-controller (slice C1 of #1095), not CI.
+#
+# Per docs/EPICS-1-6-unified-design.md §3.9 row 2 + §11 row 6.
+#
+# Per docs/INVIOLABLE-PRINCIPLES.md #4a, this workflow only inspects YAML
+# — it does not build images, deploy anything, or call cloud APIs.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'clusters/**'
+      - '.github/workflows/cluster-template-drift.yaml'
+  pull_request:
+    paths:
+      - 'clusters/**'
+      - '.github/workflows/cluster-template-drift.yaml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  drift-warn:
+    name: Detect bootstrap-kit drift
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: List per-Sovereign bootstrap-kits
+        id: list
+        run: |
+          # Every cluster directory other than _template is a per-Sovereign overlay.
+          mapfile -t sovereigns < <(find clusters -maxdepth 1 -mindepth 1 -type d \
+            -not -name '_template' -printf '%f\n')
+          printf 'sovereigns=%s\n' "${sovereigns[*]}" >> "$GITHUB_OUTPUT"
+          echo "Found Sovereigns: ${sovereigns[*]}"
+
+      - name: Diff each Sovereign bootstrap-kit against _template
+        run: |
+          set -u
+          template=clusters/_template/bootstrap-kit
+          if [ ! -d "$template" ]; then
+            echo "_template/bootstrap-kit missing — nothing to compare against."
+            exit 0
+          fi
+
+          echo "## Cluster bootstrap-kit drift report" > /tmp/drift.md
+          echo >> /tmp/drift.md
+          echo "Comparing each \`clusters/<sovereign>/bootstrap-kit/\` against \`clusters/_template/bootstrap-kit/\`." >> /tmp/drift.md
+          echo >> /tmp/drift.md
+
+          any_drift=0
+          while IFS= read -r sovereign_dir; do
+            target="$sovereign_dir/bootstrap-kit"
+            [ -d "$target" ] || continue
+            sovereign=$(basename "$sovereign_dir")
+
+            # diff -rq lists differing + only-in-X files; filter both.
+            differs=$(diff -rq "$template" "$target" 2>/dev/null || true)
+
+            if [ -z "$differs" ]; then
+              echo "### ✅ ${sovereign} — fully aligned with \`_template\`" >> /tmp/drift.md
+              echo >> /tmp/drift.md
+            else
+              any_drift=1
+              changed=$(echo "$differs" | grep -c "^Files " || true)
+              tmpl_only=$(echo "$differs" | grep -c "^Only in $template" || true)
+              sov_only=$(echo "$differs" | grep -c "^Only in $target" || true)
+              echo "### ⚠️ ${sovereign} — drift detected" >> /tmp/drift.md
+              echo >> /tmp/drift.md
+              echo "- ${changed} file(s) differ between \`_template\` and \`${sovereign}\`" >> /tmp/drift.md
+              echo "- ${tmpl_only} file(s) ONLY in \`_template\` (missing on Sovereign — likely needs adding)" >> /tmp/drift.md
+              echo "- ${sov_only} file(s) ONLY on Sovereign (extra — likely a per-Sovereign overlay or stale leftover)" >> /tmp/drift.md
+              echo >> /tmp/drift.md
+              echo "<details><summary>Full diff list</summary>" >> /tmp/drift.md
+              echo >> /tmp/drift.md
+              echo '```' >> /tmp/drift.md
+              echo "$differs" >> /tmp/drift.md
+              echo '```' >> /tmp/drift.md
+              echo "</details>" >> /tmp/drift.md
+              echo >> /tmp/drift.md
+            fi
+          done < <(find clusters -maxdepth 1 -mindepth 1 -type d -not -name '_template' -print)
+
+          if [ "$any_drift" = "1" ]; then
+            echo >> /tmp/drift.md
+            echo "**Action**: drift is informational only — every existing Sovereign carries some legitimate drift (per-Sovereign image SHAs, region-specific values overlay). The right place to enforce the boundary is Catalyst's organization-controller (slice C1 of #1095), not this workflow." >> /tmp/drift.md
+          fi
+
+          # Always print to the run summary.
+          cat /tmp/drift.md >> "$GITHUB_STEP_SUMMARY"
+          # Never fail — warn-only.
+          echo "Drift report written to job summary."
+
+      - name: Sticky comment on PR with drift report
+        if: github.event_name == 'pull_request'
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: cluster-template-drift
+          path: /tmp/drift.md

.github/workflows/cosmetic-guards.yaml

@@ -64,16 +64,16 @@ jobs:
         env:
           HOST: 0.0.0.0
         run: |
-          # Vite binds the port from vite.config.ts (server.port = 5173)
+          # Vite binds the port from vite.config.ts (server.port = 5173).
-          # under base /sovereign/. Tests reach the wizard at
+          # base: '/' since issue #596 — wizard is at /wizard, not /sovereign/wizard.
-          # http://localhost:5173/sovereign/wizard.
           nohup npm run dev > /tmp/catalyst-ui-dev.log 2>&1 &
           echo $! > /tmp/catalyst-ui.pid
 
       - name: Wait for Catalyst UI to be ready
         run: |
+          # base: '/' since issue #596 — health-check /wizard not /sovereign/wizard.
           for i in $(seq 1 60); do
-            if curl -sf -o /dev/null http://localhost:5173/sovereign/wizard; then
+            if curl -sf -o /dev/null http://localhost:5173/wizard; then
               echo "UI ready after ${i}s"
               exit 0
             fi
@@ -87,7 +87,7 @@ jobs:
         working-directory: products/catalyst/bootstrap/ui
         env:
           PLAYWRIGHT_HOST: http://localhost:5173
-          PLAYWRIGHT_BASEPATH: /sovereign
+          PLAYWRIGHT_BASEPATH: /
         # --grep filters by the @cosmetic-guard annotation that every
         # test in the suite carries. If a future test in the same file
         # is added without the tag, this command will skip it — that's

.github/workflows/infra-hetzner-tofu.yaml

@@ -0,0 +1,63 @@
+name: infra/hetzner — OpenTofu validate + test
+
+# Module-local guardrail for the Catalyst Hetzner Phase-0 OpenTofu module
+# at infra/hetzner/. Every PR touching the module re-runs `tofu validate`,
+# `tofu fmt -check`, and the module's own `.tftest.hcl` test suite so the
+# multi-region wiring (slice G1, EPIC-0 #1095) stays green and the legacy
+# singular-region apply path keeps its plan-clean shape.
+#
+# Per CLAUDE.md "every workflow MUST be event-driven, NEVER scheduled":
+# this workflow is push-on-merge + pull-request-on-touch. There is no
+# `schedule:` trigger; ad-hoc reruns go through workflow_dispatch.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'infra/hetzner/**'
+      - '.github/workflows/infra-hetzner-tofu.yaml'
+  pull_request:
+    paths:
+      - 'infra/hetzner/**'
+      - '.github/workflows/infra-hetzner-tofu.yaml'
+  workflow_dispatch:
+
+jobs:
+  validate-and-test:
+    name: validate + fmt + test
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: infra/hetzner
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install OpenTofu
+        uses: opentofu/setup-opentofu@v1
+        with:
+          # Pinned to match the version `infra/hetzner/versions.tf` declares
+          # (`required_version = ">= 1.6.0"`) and the version
+          # `tests/e2e/hetzner-provisioning` already uses (1.8.5). Bump in
+          # lockstep with that workflow to keep CI behaviour deterministic.
+          tofu_version: 1.8.5
+
+      - name: tofu init (backend=false — no real state for module-local checks)
+        run: tofu init -backend=false
+
+      - name: tofu fmt -check
+        run: tofu fmt -check -recursive
+
+      - name: tofu validate
+        run: tofu validate
+
+      - name: tofu test (offline — mock_provider + override_resource)
+        # The module's tests/multi_region.tftest.hcl exercises the
+        # multi-region wiring shape WITHOUT touching real Hetzner.
+        # `mock_provider "hcloud"` short-circuits API calls and
+        # `override_resource minio_s3_bucket.main` bypasses the minio
+        # provider's required-attribute schema check. Real-cloud E2E
+        # provisioning lives in `.github/workflows/test-hetzner-e2e.yaml`
+        # (gated on the `test/hetzner-e2e` PR label).
+        run: tofu test

.github/workflows/omantel-e2e-handover.yaml

@@ -0,0 +1,83 @@
+name: omantel handover E2E (Phase 8 DoD)
+
+# Issue #429 — on-demand E2E that runs the Phase 8 Definition-of-Done suite
+# against a live omantel.omani.works Sovereign. Per the master WBS
+# (`docs/omantel-handover-wbs.md` §5 Phase 8) this is the final gate proving
+# omantel is fully self-sufficient and zero-contabo-dependent.
+#
+# Trigger model — workflow_dispatch ONLY:
+#   - This is a SIDE-EFFECT-FREE smoke against a live customer-side cluster;
+#     we do not want it firing on every push to main. The operator dispatches
+#     it manually (or another workflow dispatches it via `gh workflow run`)
+#     once Phase 4/6/7 land and the first omantel run completes.
+#   - Per CLAUDE.md "Coupled rule — EVERY workflow MUST be event-driven, NEVER
+#     scheduled": no `schedule:` cron trigger. workflow_dispatch is the
+#     ad-hoc handle for re-runs against the live target.
+#
+# What the spec needs (per tests/e2e/playwright/tests/omantel-handover.spec.ts):
+#   OMANTEL_BASE_URL  — console host
+#   OMANTEL_API_BASE  — catalyst-api host
+#   OPERATOR_BEARER   — bootstrap operator JWT (passed via repo secret)
+#
+# When all three are set the spec runs; when any is unset, the spec self-skips
+# (so `npx playwright test --list` works locally without omantel access).
+
+on:
+  workflow_dispatch:
+    inputs:
+      omantel_base_url:
+        description: 'Sovereign console URL'
+        required: false
+        default: 'https://omantel.omani.works'
+      omantel_api_base:
+        description: 'Sovereign catalyst-api URL'
+        required: false
+        default: 'https://api.omantel.omani.works'
+      omantel_sovereign_id:
+        description: 'Sovereign id (matches /api/sovereigns/<id>)'
+        required: false
+        default: 'omantel'
+      fault_inject_probes:
+        description: 'Number of /api/healthz probes for the zero-contabo-dependency test'
+        required: false
+        default: '5'
+
+jobs:
+  e2e:
+    name: omantel Phase 8 DoD
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+
+      - name: Install Playwright dependencies
+        working-directory: tests/e2e/playwright
+        run: |
+          npm install
+          npx playwright install --with-deps chromium
+
+      - name: Run omantel handover Phase 8 DoD
+        working-directory: tests/e2e/playwright
+        env:
+          OMANTEL_BASE_URL: ${{ inputs.omantel_base_url }}
+          OMANTEL_API_BASE: ${{ inputs.omantel_api_base }}
+          OMANTEL_SOVEREIGN_ID: ${{ inputs.omantel_sovereign_id }}
+          # OPERATOR_BEARER is a repo secret — populated by the operator on
+          # the omantel side (short-lived JWT). The spec self-skips if unset.
+          OPERATOR_BEARER: ${{ secrets.OPERATOR_BEARER }}
+          FAULT_INJECT_PROBES: ${{ inputs.fault_inject_probes }}
+        run: npx playwright test tests/omantel-handover.spec.ts --reporter=list
+
+      - name: Upload Playwright report
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: omantel-handover-playwright-report
+          path: tests/e2e/playwright/playwright-report/
+          retention-days: 30

.github/workflows/openclaw-runtime.yaml

@@ -0,0 +1,121 @@
+# Build openclaw-runtime — per-user pod image consumed by bp-openclaw.
+#
+# Per Inviolable Principle 1 (event-driven, never schedule:cron) and per
+# Inviolable Principle 4 (never floating tags in production), this
+# workflow:
+#   - Triggers on push to platform/openclaw/runtime/** on main.
+#   - Tags the image with the short SHA of the triggering commit.
+#   - Provides workflow_dispatch ONLY for re-running an existing commit
+#     without a code change (per the 2026-05-01 lesson in CLAUDE.md).
+#
+# Output: ghcr.io/openova-io/openova/openclaw-runtime:<sha>
+#
+# Tracking: openova-io/openova#803 (SME-4 bp-openclaw)
+
+name: Build openclaw-runtime
+
+on:
+  push:
+    paths:
+      - 'platform/openclaw/runtime/**'
+      - '.github/workflows/openclaw-runtime.yaml'
+    branches: [main]
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE: ghcr.io/openova-io/openova/openclaw-runtime
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set short SHA
+        id: vars
+        run: echo "sha_short=$(echo $GITHUB_SHA | head -c 7)" >> "$GITHUB_OUTPUT"
+
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build image (load for smoke test)
+        uses: docker/build-push-action@v6
+        with:
+          context: platform/openclaw/runtime
+          file: platform/openclaw/runtime/Dockerfile
+          push: false
+          load: true
+          tags: ${{ env.IMAGE }}:test
+
+      - name: Smoke test (FATAL when env vars missing)
+        run: |
+          # The runtime is contractually required to refuse to start
+          # without NEWAPI_BASE_URL + NEWAPI_KEY. Verify the FATAL
+          # message fires.
+          if docker run --rm ${{ env.IMAGE }}:test 2>&1 | grep -q "FATAL: NEWAPI_BASE_URL and NEWAPI_KEY"; then
+            echo "Runtime correctly refuses to start without env vars."
+          else
+            echo "FAIL: runtime did NOT print FATAL when env vars missing"
+            docker run --rm ${{ env.IMAGE }}:test || true
+            exit 1
+          fi
+
+      - name: Smoke test (runs with env vars)
+        run: |
+          # Verify the runtime starts cleanly and serves /healthz when
+          # env vars are present.
+          docker run -d --name openclaw-smoke \
+            -p 18080:8080 \
+            -e NEWAPI_BASE_URL=http://localhost:9999 \
+            -e NEWAPI_KEY=sk-smoke-test \
+            ${{ env.IMAGE }}:test
+          # Wait for listener.
+          for i in $(seq 1 10); do
+            if curl -sf http://127.0.0.1:18080/healthz > /dev/null; then
+              echo "healthz OK"
+              break
+            fi
+            sleep 1
+          done
+          if ! curl -sf http://127.0.0.1:18080/healthz > /dev/null; then
+            echo "FAIL: /healthz did not respond"
+            docker logs openclaw-smoke
+            docker stop openclaw-smoke
+            exit 1
+          fi
+          # Exercise the index page.
+          curl -sf http://127.0.0.1:18080/ | grep -q "OpenClaw runtime" || {
+            echo "FAIL: index page missing expected marker"
+            docker stop openclaw-smoke
+            exit 1
+          }
+          docker stop openclaw-smoke
+          echo "Smoke OK: container starts, /healthz responds, / serves landing."
+
+      - name: Push image (SHA-pinned tag only)
+        uses: docker/build-push-action@v6
+        with:
+          context: platform/openclaw/runtime
+          file: platform/openclaw/runtime/Dockerfile
+          push: true
+          # SHA-pinned tag ONLY. Per Inviolable Principle 4, do NOT
+          # publish a `:latest` for production-consumed images — every
+          # consumer pins to a specific SHA.
+          tags: |
+            ${{ env.IMAGE }}:${{ steps.vars.outputs.sha_short }}
+
+      - name: Summary
+        run: |
+          echo "openclaw-runtime built and pushed" >> "$GITHUB_STEP_SUMMARY"
+          echo "" >> "$GITHUB_STEP_SUMMARY"
+          echo "- Image: \`${{ env.IMAGE }}:${{ steps.vars.outputs.sha_short }}\`" >> "$GITHUB_STEP_SUMMARY"
+          echo "- Commit: \`${{ github.sha }}\`" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/playwright-smoke.yaml

@@ -70,14 +70,15 @@ jobs:
           HOST: 0.0.0.0
         run: |
           # Vite dev server binds 4321 by default; we keep the default so the
-          # tests' BASE_URL fallback (http://localhost:4321) works.
+          # tests' BASE_URL fallback (http://localhost:5173) works.
           nohup npm run dev > /tmp/catalyst-ui-dev.log 2>&1 &
           echo $! > /tmp/catalyst-ui.pid
 
       - name: Wait for Catalyst UI to be ready
         run: |
+          # base: '/' since issue #596 — wizard is at /wizard, not /sovereign/wizard.
           for i in $(seq 1 60); do
-            if curl -sf -o /dev/null http://localhost:4321/sovereign/wizard; then
+            if curl -sf -o /dev/null http://localhost:5173/wizard; then
               echo "UI ready after ${i}s"
               exit 0
             fi
@@ -90,7 +91,7 @@ jobs:
       - name: Run Playwright smoke
         working-directory: tests/e2e/playwright
         env:
-          BASE_URL: http://localhost:4321
+          BASE_URL: http://localhost:5173
           # ADMIN_BASE_URL / MARKETPLACE_BASE_URL not set — the admin and
           # marketplace specs self-skip when their respective apps aren't up,
           # which keeps this workflow lean. Booting all three apps requires

.github/workflows/preflight-bootstrap-kit.yaml

@@ -0,0 +1,262 @@
+name: Phase-8a preflight A — bootstrap-kit reconcile dry-run
+
+# Closes openova-io/openova#459 — surfaces Risk-register R4 (bootstrap-kit
+# reconcile-chain order untested under load) BEFORE Phase 8a burns Hetzner
+# credit on `test.omani.works`. Spins up a kind cluster, installs Cilium
+# with Gateway API CRDs + Flux, plants mock cloud creds (so the chain
+# doesn't immediately 401 on real Hetzner), applies the
+# `clusters/_template/bootstrap-kit/` kustomization, and watches every
+# `bp-*` HelmRelease in flux-system over a 15-min polling window.
+#
+# Goal is to surface ALL reconcile-chain failures, not stop at the first.
+# The summary step always runs (`if: always()`) and emits a Markdown table
+# of every HR's terminal Ready condition; failed HRs get a `kubectl
+# describe` block appended for triage.
+#
+# Per CLAUDE.md "every workflow MUST be event-driven, NEVER scheduled":
+# this workflow is push-on-self-edit + workflow_dispatch only. There is
+# no `schedule:` trigger.
+#
+# Per the canonical-seam rule (docs/omantel-handover-wbs.md §3a), this
+# workflow REUSES existing seams:
+#   - kind setup pattern from .github/workflows/test-bootstrap-kit.yaml
+#   - Flux install via fluxcd/flux2/action@main (same as test-bootstrap-kit)
+#   - bootstrap-kit kustomization at clusters/_template/bootstrap-kit/
+#     (the same overlay that production Sovereigns consume)
+# It does NOT duplicate the Go test-bootstrap-kit harness — that test
+# stops at "Flux accepts our manifests"; this preflight goes further by
+# letting HelmRelease reconciliation actually attempt under mocked creds.
+#
+# Out of scope: real Hetzner cloud calls (mock creds only, that's the
+# point), live HTTPRoute admission (covered by sibling preflight #461),
+# Crossplane provider-hcloud Healthy probe (sibling preflight #460),
+# Keycloak realm-import (sibling preflight #462).
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - '.github/workflows/preflight-bootstrap-kit.yaml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  # bp-* charts are PRIVATE GHCR packages under openova-io. The
+  # bootstrap-kit kustomization references them via OCI HelmRepositories;
+  # source-controller reads the `flux-system/ghcr-pull` Secret planted
+  # below. The runner-side `helm registry login` (next step) is needed
+  # for any direct `helm install oci://...` calls used during diagnostics.
+  packages: read
+
+jobs:
+  preflight:
+    name: Preflight bootstrap-kit reconcile
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up kind
+        uses: helm/kind-action@v1
+        with:
+          cluster_name: preflight-bootstrap-kit
+          version: v0.25.0
+          node_image: kindest/node:v1.30.6
+
+      - name: Login to GHCR (helm registry)
+        # bp-* charts are private GHCR packages; helm/source-controller
+        # both need GHCR auth to pull OCI manifests. Mirrors the seam in
+        # .github/workflows/preflight-crossplane-hcloud.yaml + blueprint-release.yaml.
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" \
+            | helm registry login ghcr.io \
+                --username "${{ github.actor }}" \
+                --password-stdin
+
+      - name: Install Gateway API CRDs (standard channel, v1.2.0)
+        run: |
+          # Cilium's Helm chart auto-installs Gateway API CRDs only when
+          # gatewayAPI=true is passed; the bp-cilium chart enables it,
+          # but a kind cluster has no chart pre-installed. We pre-plant
+          # the CRDs so the bootstrap-kit Gateway/HTTPRoute manifests
+          # parse against a live API server. (Cilium controller itself
+          # may still fail to install — that's exactly what we want to
+          # surface, not hide.)
+          kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.0/standard-install.yaml
+
+      - name: Install Flux CLI
+        uses: fluxcd/flux2/action@main
+
+      - name: Install Flux controllers
+        run: |
+          # Full Flux install (source-controller, kustomize-controller,
+          # helm-controller, notification-controller). Mirrors what the
+          # cloud-init bootstrap installs on a real Sovereign.
+          flux install --network-policy=false
+
+      - name: Plant mock cloud creds
+        run: |
+          # The bootstrap-kit Helm valuesFrom blocks reference these
+          # Secrets. Mock values let HRs proceed past Secret-lookup and
+          # into actual chart install / dependency-wait, which is where
+          # the reconcile-chain bugs we're hunting actually live.
+          kubectl create secret generic object-storage \
+            --namespace flux-system \
+            --from-literal=s3-endpoint=https://fake.example.com \
+            --from-literal=s3-region=fake \
+            --from-literal=s3-bucket=preflight-bucket \
+            --from-literal=s3-access-key=AKIA-FAKE \
+            --from-literal=s3-secret-key=fake-secret-key
+          kubectl create secret generic cloud-credentials \
+            --namespace flux-system \
+            --from-literal=hcloud-token=fake-hcloud-token
+          # Stub GHCR pull credential — bp-* HelmRepositories reference
+          # secretRef:{name: ghcr-pull}. Without it, source-controller
+          # bails before chart pull, masking the deeper failures we're
+          # trying to surface. Using the canonical k8s dockerconfigjson
+          # shape with a fake credential — chart pulls will fail with
+          # 401, but every HR will at least hit the install attempt.
+          kubectl create secret docker-registry ghcr-pull \
+            --namespace flux-system \
+            --docker-server=ghcr.io \
+            --docker-username=fake-user \
+            --docker-password=fake-pat \
+            --docker-email=fake@example.com
+
+      - name: Render bootstrap-kit kustomization with placeholder substitution
+        run: |
+          # The _template tree carries TWO substitution shapes (legacy
+          # SOVEREIGN_FQDN_PLACEHOLDER literal + Flux envsubst-style
+          # ${SOVEREIGN_FQDN}). Production reconciles these via Flux
+          # Kustomization postBuild.substituteFrom; here we render once
+          # to a tempdir so plain `kubectl apply -k` works without
+          # introducing a wrapper Kustomization (which would itself add
+          # a layer of indirection that hides reconcile-chain failures).
+          mkdir -p /tmp/bootstrap-kit-rendered
+          cp -r clusters/_template/bootstrap-kit/* /tmp/bootstrap-kit-rendered/
+          # Substitute both shapes deterministically.
+          # Note: single quotes around the sed expressions are intentional —
+          # we want the LITERAL string `${SOVEREIGN_FQDN}` to be matched,
+          # not the (unset) shell variable. shellcheck SC2016 is a
+          # false-positive here.
+          # shellcheck disable=SC2016
+          find /tmp/bootstrap-kit-rendered -type f -name '*.yaml' -print0 \
+            | xargs -0 sed -i \
+                -e 's|SOVEREIGN_FQDN_PLACEHOLDER|test-sov.example.com|g' \
+                -e 's|${SOVEREIGN_FQDN}|test-sov.example.com|g'
+
+      - name: Apply bootstrap-kit
+        id: apply
+        run: |
+          # Apply ALL slots in one go. Flux respects HelmRelease
+          # spec.dependsOn at reconcile time, so the API server accepting
+          # all 36+ resources up-front matches the production path.
+          kubectl apply -k /tmp/bootstrap-kit-rendered/ || true
+          # Don't fail-fast: an apply error on one resource (e.g. a CRD
+          # missing on first pass) is itself a finding for the report.
+          # The watch step below records terminal state regardless.
+
+      - name: Watch HelmReleases (15 min poll)
+        run: |
+          # 30 polls × 30s = 15 min. We never break the loop — the goal
+          # is to capture the terminal state of every HR, not race the
+          # first one to Ready.
+          for i in $(seq 1 30); do
+            echo "=== poll ${i}/30 ($(date -u +%H:%M:%S) UTC) ==="
+            kubectl get hr -n flux-system -o wide 2>&1 || true
+            echo ""
+            sleep 30
+          done
+
+      - name: Summary report
+        if: always()
+        run: |
+          {
+            echo '## Phase-8a preflight A — bootstrap-kit reconcile dry-run'
+            echo ''
+            echo "Cluster: kind \`preflight-bootstrap-kit\`"
+            echo "Substitution: \`SOVEREIGN_FQDN=test-sov.example.com\`"
+            echo "Mock creds: \`flux-system/object-storage\`, \`flux-system/cloud-credentials\`, \`flux-system/ghcr-pull\`"
+            echo ''
+            echo '## bp-* HelmRelease final state'
+            echo ''
+            echo '| Name | Ready | Reason | Message (truncated) |'
+            echo '|---|---|---|---|'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if kubectl get hr -n flux-system -o json > /tmp/hrs.json 2>/dev/null; then
+            jq -r '.items[] |
+              . as $hr |
+              ($hr.status.conditions // [] | map(select(.type=="Ready")) | first) as $r |
+              "| \($hr.metadata.name) | \($r.status // "—") | \($r.reason // "—") | \(($r.message // "—") | .[0:120]) |"' \
+              /tmp/hrs.json >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo '| (no HelmReleases found in flux-system) | — | — | — |' >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+          {
+            echo ''
+            echo '## Failed HRs — describe + last 30 events'
+            echo ''
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          # List every HR not at Ready=True (False, Unknown, or absent).
+          if [ -f /tmp/hrs.json ]; then
+            mapfile -t failed < <(jq -r '.items[] |
+              select(((.status.conditions // []) | map(select(.type=="Ready")) | first | .status // "Unknown") != "True") |
+              .metadata.name' /tmp/hrs.json)
+
+            if [ "${#failed[@]}" -eq 0 ]; then
+              echo '_All HRs reached Ready=True. (Surprising — review the run log to confirm chart pulls succeeded under mock creds.)_' >> "$GITHUB_STEP_SUMMARY"
+            else
+              for hr in "${failed[@]}"; do
+                {
+                  echo "### ${hr}"
+                  echo ''
+                  echo '<details><summary>describe hr</summary>'
+                  echo ''
+                  echo '```'
+                  kubectl describe hr -n flux-system "${hr}" 2>&1 | tail -50
+                  echo '```'
+                  echo ''
+                  echo '</details>'
+                  echo ''
+                } >> "$GITHUB_STEP_SUMMARY"
+              done
+            fi
+          fi
+
+          {
+            echo ''
+            echo '## Pod terminal state (all namespaces)'
+            echo ''
+            echo '<details><summary>pods</summary>'
+            echo ''
+            echo '```'
+            kubectl get pods -A -o wide 2>&1 || echo '(none)'
+            echo '```'
+            echo ''
+            echo '</details>'
+            echo ''
+            echo '## Recent kustomize-controller events'
+            echo ''
+            echo '<details><summary>events</summary>'
+            echo ''
+            echo '```'
+            kubectl get events -n flux-system --sort-by=.lastTimestamp 2>&1 | tail -50 || echo '(none)'
+            echo '```'
+            echo ''
+            echo '</details>'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload raw artefacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: preflight-bootstrap-kit-artefacts
+          path: |
+            /tmp/hrs.json
+            /tmp/bootstrap-kit-rendered/
+          if-no-files-found: warn
+          retention-days: 14

.github/workflows/preflight-cilium-httproute.yaml

@@ -0,0 +1,288 @@
+# Phase-8a preflight C — Cilium Gateway HTTPRoute admission for bp-catalyst-platform on kind.
+#
+# Surfaces Risk-register R3 (`docs/omantel-handover-wbs.md` §9a — Cilium
+# Gateway HTTPRoute admission untested). bp-catalyst-platform smoke skipped
+# HTTPRoute on contabo because contabo runs Traefik (no `cilium-gateway`
+# Gateway present per ADR-0001 §9.4). Phase 8a will hit this gate when
+# console.test.omani.works is unreachable — this workflow exposes the
+# admission contract on a disposable kind cluster ahead of Phase 8a.
+#
+# What this validates:
+#   1. Cilium 1.16.x with `gatewayAPI.enabled=true` registers the `cilium`
+#      GatewayClass and reports it Accepted.
+#   2. The per-Sovereign Gateway shape from
+#      `clusters/_template/bootstrap-kit/01-cilium.yaml` (HTTP listener)
+#      is admitted by the Cilium GatewayClass.
+#   3. The HTTPRoute templates inside bp-catalyst-platform's chart
+#      (`products/catalyst/chart/templates/httproute.yaml`) — `catalyst-ui`
+#      and `catalyst-api` — reach `Accepted=True` against that Gateway when
+#      rendered with sovereign-overlay values
+#      (`ingress.hosts.console.host`, `ingress.hosts.api.host`).
+#
+# What this does NOT validate (out of scope; Phase 8a/8b territory):
+#   - TLS termination (HTTP only — wildcard cert + cert-manager + DNS01 is
+#     Phase 8a on real Sovereign).
+#   - Backend health (we plant placeholder catalyst-ui / catalyst-api
+#     Services so `backendRefs` resolve; the Deployments behind them
+#     are not part of this contract).
+#   - The 10 leaf bp-* dependencies (bp-cert-manager, bp-flux, bp-keycloak,
+#     etc.) — those have their own chart-verify smoke runs (#377/#378/#382
+#     etc.). Here we render bp-catalyst-platform locally and apply only the
+#     catalyst-{ui,api} Service stubs + the rendered HTTPRoute manifests, to
+#     keep the kind cluster bounded to the admission seam under test.
+#
+# Anti-duplication:
+#   - Cilium install + GatewayClass wait pattern: same as
+#     `playwright-smoke.yaml` style (kind + helm), no duplicated cluster
+#     bring-up logic.
+#   - GHCR helm-registry-login matches `blueprint-release.yaml` §
+#     "Helm registry login" (line 173-177).
+#   - Per-Sovereign Gateway shape: 1:1 mirror of `clusters/_template/
+#     bootstrap-kit/01-cilium.yaml` HTTP listener — no new shape invented.
+#
+# Per CLAUDE.md "every workflow MUST be event-driven, NEVER scheduled":
+# triggered on push to this file's path + workflow_dispatch for re-runs.
+
+name: Phase-8a preflight C — Cilium Gateway HTTPRoute admission
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+    paths:
+      - '.github/workflows/preflight-cilium-httproute.yaml'
+      - 'products/catalyst/chart/templates/httproute.yaml'
+      - 'products/catalyst/chart/values.yaml'
+      - 'clusters/_template/bootstrap-kit/01-cilium.yaml'
+
+permissions:
+  contents: read
+  packages: read   # `helm pull oci://ghcr.io/openova-io/bp-catalyst-platform`
+
+jobs:
+  preflight:
+    name: Preflight Cilium HTTPRoute admission
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up kind cluster (no kindnet, kube-proxy disabled)
+        uses: helm/kind-action@v1
+        with:
+          version: v0.24.0
+          cluster_name: preflight-c
+          # Disable default CNI + kube-proxy so Cilium can take over both
+          # roles (matches the Sovereign k3s shape: --flannel-backend=none
+          # + --disable-kube-proxy in cloud-init).
+          config: |
+            kind: Cluster
+            apiVersion: kind.x-k8s.io/v1alpha4
+            networking:
+              disableDefaultCNI: true
+              kubeProxyMode: none
+            nodes:
+              - role: control-plane
+              - role: worker
+
+      - name: Install Gateway API CRDs (v1.2.0 — matches Cilium 1.16.x support matrix)
+        run: |
+          kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.0/standard-install.yaml
+          kubectl wait --for=condition=Established --timeout=60s \
+            crd/gateways.gateway.networking.k8s.io \
+            crd/httproutes.gateway.networking.k8s.io \
+            crd/gatewayclasses.gateway.networking.k8s.io
+
+      - name: Install Cilium with Gateway API enabled
+        run: |
+          helm repo add cilium https://helm.cilium.io/
+          helm repo update
+          helm install cilium cilium/cilium \
+            --version 1.16.5 \
+            --namespace kube-system \
+            --set kubeProxyReplacement=true \
+            --set k8sServiceHost=preflight-c-control-plane \
+            --set k8sServicePort=6443 \
+            --set gatewayAPI.enabled=true \
+            --set envoy.enabled=true \
+            --set l7Proxy=true \
+            --set hubble.enabled=false \
+            --set operator.replicas=1 \
+            --wait --timeout 5m
+
+      - name: Wait for Cilium GatewayClass to be Accepted
+        run: |
+          for i in $(seq 1 30); do
+            STATUS="$(kubectl get gatewayclass cilium -o jsonpath='{.status.conditions[?(@.type=="Accepted")].status}' 2>/dev/null || true)"
+            if [ "$STATUS" = "True" ]; then
+              echo "GatewayClass cilium Accepted=True after ${i}*5=$((i*5))s"
+              kubectl get gatewayclass cilium -o yaml
+              exit 0
+            fi
+            sleep 5
+          done
+          echo "GatewayClass cilium did NOT reach Accepted=True"
+          kubectl describe gatewayclass cilium || true
+          kubectl -n kube-system get pods
+          exit 1
+
+      - name: Apply per-Sovereign Gateway (HTTP listener only — TLS is Phase 8a)
+        run: |
+          # Mirrors `clusters/_template/bootstrap-kit/01-cilium.yaml`
+          # Gateway shape (name: cilium-gateway, namespace: kube-system,
+          # gatewayClassName: cilium, listener `http` on port 80,
+          # allowedRoutes.namespaces.from=All). The HTTPS listener is
+          # omitted because TLS material requires cert-manager + DNS01
+          # (Phase 8a, not preflight scope). Catalyst HTTPRoutes will be
+          # attached via parentRef.sectionName=http override below.
+          cat <<'EOF' | kubectl apply -f -
+          apiVersion: gateway.networking.k8s.io/v1
+          kind: Gateway
+          metadata:
+            name: cilium-gateway
+            namespace: kube-system
+            labels:
+              catalyst.openova.io/component: cilium-gateway
+              catalyst.openova.io/preflight: phase-8a-c
+          spec:
+            gatewayClassName: cilium
+            listeners:
+              - name: http
+                port: 80
+                protocol: HTTP
+                allowedRoutes:
+                  namespaces:
+                    from: All
+          EOF
+
+      - name: Wait for Gateway to be Accepted+Programmed
+        run: |
+          for i in $(seq 1 36); do
+            ACC="$(kubectl get gateway cilium-gateway -n kube-system -o jsonpath='{.status.conditions[?(@.type=="Accepted")].status}' 2>/dev/null || true)"
+            PROG="$(kubectl get gateway cilium-gateway -n kube-system -o jsonpath='{.status.conditions[?(@.type=="Programmed")].status}' 2>/dev/null || true)"
+            if [ "$ACC" = "True" ] && [ "$PROG" = "True" ]; then
+              echo "Gateway Accepted=True Programmed=True after ${i}*5=$((i*5))s"
+              kubectl get gateway cilium-gateway -n kube-system -o yaml
+              exit 0
+            fi
+            sleep 5
+          done
+          echo "Gateway did not reach Accepted+Programmed"
+          kubectl describe gateway cilium-gateway -n kube-system || true
+          exit 1
+
+      - name: Helm registry login (GHCR)
+        # Matches `blueprint-release.yaml` §"Helm registry login" — same
+        # canonical seam, never duplicated.
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io \
+            --username "${{ github.actor }}" --password-stdin
+
+      - name: Render bp-catalyst-platform HTTPRoutes with sovereign overlay
+        run: |
+          # Pull the published OCI chart and render its catalyst-platform
+          # HTTPRoute templates with sovereign-overlay values. We render
+          # locally (not `helm install`) and apply only the resources
+          # under test — the 10 leaf bp-* deps would not boot in a
+          # 30-min kind window and are out of scope for the admission
+          # contract being verified here.
+          helm pull oci://ghcr.io/openova-io/bp-catalyst-platform \
+            --version 1.1.8 \
+            --untar --untardir /tmp/bp-cp
+          mkdir -p /tmp/preflight-render
+          # Render with:
+          #   - ingress.gateway.parentRef.sectionName=http (default values
+          #     are `https`; we don't have TLS in preflight)
+          #   - ingress.hosts.{console,api}.host=*.test.local — the Gateway
+          #     has no hostname filter (allow-all) so any value attaches.
+          #   - --show-only renders only the templates under test; this
+          #     bypasses the leaf bp-* dep chart rendering.
+          helm template catalyst-platform /tmp/bp-cp/bp-catalyst-platform \
+            --namespace catalyst \
+            --set ingress.gateway.enabled=true \
+            --set ingress.gateway.parentRef.name=cilium-gateway \
+            --set ingress.gateway.parentRef.namespace=kube-system \
+            --set ingress.gateway.parentRef.sectionName=http \
+            --set ingress.hosts.console.host=console.test.local \
+            --set ingress.hosts.api.host=api.test.local \
+            --show-only templates/httproute.yaml \
+            > /tmp/preflight-render/httproute.yaml
+          echo "--- rendered HTTPRoute manifest ---"
+          cat /tmp/preflight-render/httproute.yaml
+
+      - name: Apply catalyst namespace + backend Service stubs + HTTPRoutes
+        run: |
+          kubectl create namespace catalyst
+          # Placeholder Services so HTTPRoute backendRefs resolve. The
+          # admission contract requires named Services to exist in the
+          # same namespace as the HTTPRoute (port 80 for catalyst-ui,
+          # 8080 for catalyst-api — matches the chart's backendRefs).
+          cat <<'EOF' | kubectl apply -f -
+          apiVersion: v1
+          kind: Service
+          metadata:
+            name: catalyst-ui
+            namespace: catalyst
+          spec:
+            ports:
+              - name: http
+                port: 80
+                targetPort: 8080
+            selector:
+              app.kubernetes.io/name: catalyst-ui
+          ---
+          apiVersion: v1
+          kind: Service
+          metadata:
+            name: catalyst-api
+            namespace: catalyst
+          spec:
+            ports:
+              - name: http
+                port: 8080
+                targetPort: 8080
+            selector:
+              app.kubernetes.io/name: catalyst-api
+          EOF
+          kubectl apply -f /tmp/preflight-render/httproute.yaml
+
+      - name: Verify HTTPRoute admission (catalyst-ui + catalyst-api Accepted=True)
+        run: |
+          set -e
+          # Cilium reconciles HTTPRoute status asynchronously — give it
+          # up to 90s before declaring failure.
+          for route in catalyst-ui catalyst-api; do
+            for i in $(seq 1 18); do
+              STATUS="$(kubectl get httproute "$route" -n catalyst -o jsonpath='{.status.parents[0].conditions[?(@.type=="Accepted")].status}' 2>/dev/null || true)"
+              if [ "$STATUS" = "True" ]; then
+                echo "HTTPRoute $route Accepted=True after ${i}*5=$((i*5))s"
+                break
+              fi
+              if [ "$i" = "18" ]; then
+                echo "HTTPRoute $route did NOT reach Accepted=True"
+                kubectl get httproute -A -o wide || true
+                kubectl describe httproute "$route" -n catalyst || true
+                exit 1
+              fi
+              sleep 5
+            done
+          done
+          echo "Both HTTPRoutes admitted by Cilium Gateway."
+          kubectl get httproute,gateway -A -o wide
+
+      - name: Summary
+        if: always()
+        run: |
+          {
+            echo '## HTTPRoute admission preflight (Phase-8a R3)'
+            echo ''
+            echo '### GatewayClass'
+            kubectl get gatewayclass -o wide || true
+            echo ''
+            echo '### Gateway'
+            kubectl get gateway -A -o wide || true
+            echo ''
+            echo '### HTTPRoute'
+            kubectl get httproute -A -o wide || true
+          } >> "$GITHUB_STEP_SUMMARY" 2>&1 || true

.github/workflows/preflight-crossplane-hcloud.yaml

@@ -0,0 +1,179 @@
+name: Phase-8a preflight B — Crossplane provider-hcloud Healthy
+
+# Issue #460 — Phase-8a preflight B (Risk register R2).
+# Surfaces R2 from docs/omantel-handover-wbs.md §9a:
+# "Crossplane provider-hcloud Healthy=True never observed". Phase 8a
+# fails at the Crossplane step if the Provider doesn't install cleanly,
+# so this preflight bakes the install + Healthy probe into CI.
+#
+# What it does:
+#   1. Spins up a kind cluster (matches the kind-action shape used by
+#      .github/workflows/test-bootstrap-kit.yaml — REUSE, no duplication).
+#   2. Installs the bp-crossplane chart from GHCR (oci://) at the same
+#      version pinned in the omantel handover WBS.
+#   3. Applies the EXACT Provider + ProviderConfig shape that
+#      infra/hetzner/cloudinit-control-plane.tftpl plants on a fresh
+#      Sovereign control plane (issue #425). Any drift between this
+#      preflight and that template means the live Sovereign would
+#      diverge from CI — so the YAML is copy-faithful.
+#   4. Waits up to 5 minutes for provider-hcloud to report
+#      Healthy=True. On miss, surfaces the exact blocking error via
+#      kubectl describe so the founder can act on a real failure mode
+#      rather than a vague timeout.
+#   5. Plants a fake (non-functional) hcloud-token Secret +
+#      ProviderConfig and asserts the ProviderConfig is accepted by
+#      the API server — install-time validation only. Real-credential
+#      validation belongs to Phase 8a, not this preflight.
+#
+# Per CLAUDE.md "every workflow MUST be event-driven, NEVER scheduled":
+# triggers are push-on-touch (this file only) + workflow_dispatch for
+# ad-hoc reruns. No `schedule:` cron.
+#
+# Out of scope: actually creating cloud resources via XRC; Hetzner-
+# specific Provider behaviour beyond install + Healthy check.
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+    paths:
+      - '.github/workflows/preflight-crossplane-hcloud.yaml'
+
+permissions:
+  contents: read
+  # bp-crossplane is a PRIVATE GHCR package
+  # (`gh api /orgs/openova-io/packages/container/bp-crossplane`), so the
+  # job needs read scope on packages to pull the OCI manifest. Mirrors
+  # the seam in .github/workflows/blueprint-release.yaml (which uses
+  # `packages: write` for push); read is the minimum here.
+  packages: read
+
+jobs:
+  preflight:
+    name: Preflight Crossplane provider-hcloud
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up kind
+        uses: helm/kind-action@v1
+        with:
+          cluster_name: preflight-crossplane-hcloud
+          version: v0.25.0
+          node_image: kindest/node:v1.30.6
+
+      - name: Login to GHCR (helm registry)
+        # Same pattern as .github/workflows/blueprint-release.yaml.
+        # `helm install oci://ghcr.io/...` reads the credential store
+        # populated by `helm registry login`, so this MUST happen before
+        # the install step. `helm` is preinstalled on ubuntu-latest.
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" \
+            | helm registry login ghcr.io \
+                --username "${{ github.actor }}" \
+                --password-stdin
+
+      - name: Install bp-crossplane core
+        run: |
+          helm install crossplane oci://ghcr.io/openova-io/bp-crossplane \
+            --version 1.1.3 \
+            --namespace crossplane-system --create-namespace \
+            --wait --timeout 5m
+
+      - name: Wait for Crossplane core Ready
+        run: |
+          kubectl wait --for=condition=Ready pod \
+            -l app=crossplane \
+            -n crossplane-system \
+            --timeout=5m
+
+      - name: Apply provider-hcloud Provider CR
+        run: |
+          # SHAPE MUST MATCH infra/hetzner/cloudinit-control-plane.tftpl from #425.
+          # Any drift here means the live Sovereign diverges from CI.
+          cat <<'EOF' | kubectl apply -f -
+          ---
+          apiVersion: pkg.crossplane.io/v1
+          kind: Provider
+          metadata:
+            name: provider-hcloud
+            labels:
+              catalyst.openova.io/sovereign: preflight-ci
+          spec:
+            package: xpkg.upbound.io/crossplane-contrib/provider-hcloud:v0.4.0
+            packagePullPolicy: IfNotPresent
+          EOF
+
+      - name: Wait for provider-hcloud Healthy=True
+        run: |
+          for i in $(seq 1 30); do
+            STATUS=$(kubectl get provider.pkg.crossplane.io provider-hcloud \
+              -o jsonpath='{.status.conditions[?(@.type=="Healthy")].status}' \
+              2>/dev/null || echo "")
+            if [ "$STATUS" = "True" ]; then
+              echo "OK provider-hcloud Healthy=True after $((i*10))s"
+              exit 0
+            fi
+            echo "[$i/30] Healthy=$STATUS — sleeping 10s"
+            sleep 10
+          done
+          echo "FAIL provider-hcloud did NOT reach Healthy=True in 5 min"
+          echo "--- kubectl describe provider provider-hcloud ---"
+          kubectl describe provider.pkg.crossplane.io provider-hcloud || true
+          echo "--- kubectl get pods -A ---"
+          kubectl get pods -A || true
+          echo "--- kubectl get providerrevisions -A ---"
+          kubectl get providerrevisions -A -o yaml || true
+          exit 1
+
+      - name: Plant fake cloud-credentials Secret + ProviderConfig
+        run: |
+          # ProviderConfig SHAPE MUST MATCH infra/hetzner/cloudinit-control-plane.tftpl
+          # from #425 — including the secretRef.namespace=flux-system reference.
+          # We create the flux-system namespace + fake Secret in the same place
+          # the canonical Tofu cloud-init plants the real one.
+          kubectl create namespace flux-system
+          kubectl create secret generic cloud-credentials \
+            --namespace flux-system \
+            --from-literal=hcloud-token=fake-readonly-token
+          cat <<'EOF' | kubectl apply -f -
+          ---
+          apiVersion: hcloud.crossplane.io/v1beta1
+          kind: ProviderConfig
+          metadata:
+            name: default
+          spec:
+            credentials:
+              source: Secret
+              secretRef:
+                namespace: flux-system
+                name: cloud-credentials
+                key: hcloud-token
+          EOF
+
+      - name: Validate ProviderConfig accepted
+        run: |
+          # The API server accepting the resource (returned by `kubectl get`
+          # against the hcloud.crossplane.io/v1beta1 ProviderConfig CRD) is
+          # the install-time validation. We deliberately do NOT exercise the
+          # token here — Phase 8a covers real-credential validation.
+          kubectl get providerconfig.hcloud.crossplane.io default -o yaml \
+            | grep -E '^apiVersion: hcloud\.crossplane\.io/v1beta1$' \
+            && echo "OK ProviderConfig accepted by API server"
+
+      - name: Summary
+        if: always()
+        run: |
+          echo '## Crossplane provider-hcloud preflight' >> "$GITHUB_STEP_SUMMARY"
+          echo '' >> "$GITHUB_STEP_SUMMARY"
+          echo '### Providers + ProviderConfigs' >> "$GITHUB_STEP_SUMMARY"
+          echo '```' >> "$GITHUB_STEP_SUMMARY"
+          kubectl get providers.pkg.crossplane.io,providerconfigs.hcloud.crossplane.io -A >> "$GITHUB_STEP_SUMMARY" 2>&1 || true
+          echo '```' >> "$GITHUB_STEP_SUMMARY"
+          echo '' >> "$GITHUB_STEP_SUMMARY"
+          echo '### Provider describe' >> "$GITHUB_STEP_SUMMARY"
+          echo '```' >> "$GITHUB_STEP_SUMMARY"
+          kubectl describe provider.pkg.crossplane.io provider-hcloud >> "$GITHUB_STEP_SUMMARY" 2>&1 || true
+          echo '```' >> "$GITHUB_STEP_SUMMARY"

.github/workflows/preflight-keycloak-realm.yaml

@@ -0,0 +1,283 @@
+name: Phase-8a preflight E — Keycloak realm-import + kubectl OIDC client
+
+# Issue #462 — Phase-8a preflight E (Risk register R6 from
+# docs/omantel-handover-wbs.md §9a).
+#
+# bp-keycloak 1.2.0 ships a `sovereign` realm + a public `kubectl` OIDC
+# client via the upstream bitnami/keycloak chart's keycloakConfigCli
+# post-install Helm hook (issue #326). The hook is bootstrap-timing
+# sensitive: keycloak-config-cli boots a JVM, calls the Keycloak Admin
+# API, and reconciles the realm payload — all of which depends on the
+# StatefulSet being Ready first.
+#
+# This preflight installs bp-keycloak on a kind cluster and asserts:
+#   1. The keycloak StatefulSet reaches Ready.
+#   2. The keycloakConfigCli post-install Job completes successfully.
+#   3. The `sovereign` realm exists (Keycloak's discovery endpoint
+#      returns 200 for /realms/sovereign).
+#   4. The `kubectl` OIDC client is provisioned in the realm with the
+#      localhost:8000 redirect URI and the `groups` claim mapper that
+#      the per-Sovereign k3s api-server's --oidc-* flags depend on.
+#
+# Out of scope (deferred to live Phase-8a):
+#   - kubectl-oidc-login interactive browser flow
+#   - k3s api-server-side OIDC token validation (preflight A)
+#
+# Triggers — event-driven only per CLAUDE.md "every workflow MUST be
+# event-driven, NEVER scheduled" rule. workflow_dispatch is for ad-hoc
+# re-runs without a code change.
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+    paths:
+      - '.github/workflows/preflight-keycloak-realm.yaml'
+
+permissions:
+  contents: read
+  # bp-keycloak is a PRIVATE GHCR package; helm needs GHCR auth to pull.
+  # Mirrors .github/workflows/preflight-crossplane-hcloud.yaml.
+  packages: read
+
+jobs:
+  preflight:
+    name: Preflight Keycloak realm-import
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up kind
+        uses: helm/kind-action@v1
+        with:
+          cluster_name: keycloak-preflight
+          version: v0.25.0
+          node_image: kindest/node:v1.30.6
+
+      - name: Login to GHCR (helm registry)
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" \
+            | helm registry login ghcr.io \
+                --username "${{ github.actor }}" \
+                --password-stdin
+
+      - name: Install bp-keycloak 1.2.0
+        # Release name `keycloak` matches the per-Sovereign bootstrap-kit
+        # slot (clusters/_template/bootstrap-kit/) so resource names here
+        # match what runs on a real Sovereign. Bitnami's chart de-duplicates
+        # `<release>-<chart>` when they're equal, so the StatefulSet,
+        # primary Service, and ServiceAccount are all named `keycloak`;
+        # the post-install Job is `keycloak-keycloak-config-cli`.
+        #
+        # `--wait=false` so we observe the rollout progressively in later
+        # steps and capture diagnostics on failure. Default postgresql
+        # subchart needs ~3-4 min on kind to provision its PVC + boot.
+        run: |
+          helm install keycloak oci://ghcr.io/openova-io/bp-keycloak \
+            --version 1.2.0 \
+            --namespace keycloak --create-namespace \
+            --wait=false
+
+      - name: Wait for keycloak StatefulSet Ready
+        # Bitnami keycloak uses `kubernetes.io/hostname` topology spread
+        # constraints by default — fine on a single-node kind cluster.
+        # Boot is dominated by JVM cold start; 10 min is generous.
+        run: |
+          kubectl rollout status sts/keycloak -n keycloak --timeout=15m
+
+      - name: Wait for keycloakConfigCli post-install Job to complete
+        # The Helm post-install hook Job is rendered with annotation
+        # helm.sh/hook-weight: "5" which means it runs AFTER the chart's
+        # primary resources are applied but BEFORE Helm reports success.
+        # Because we used --wait=false above, the Job may not exist yet
+        # when this step starts — poll for its appearance, then wait.
+        #
+        # Job name is deterministic: `<release>-<chart>-config-cli` =>
+        # `keycloak-keycloak-config-cli`. Bitnami still emits the label
+        # app.kubernetes.io/component=keycloak-config-cli; using the
+        # label selector keeps us robust to a future chart bump that
+        # tweaks the suffix.
+        run: |
+          for i in $(seq 1 60); do
+            JOB=$(kubectl get jobs -n keycloak \
+              -l app.kubernetes.io/component=keycloak-config-cli \
+              -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
+            if [ -n "$JOB" ]; then
+              echo "Found realm-import Job: $JOB"
+              if kubectl wait --for=condition=Complete --timeout=10m \
+                  "job/$JOB" -n keycloak; then
+                echo "Realm-import Job completed successfully."
+                exit 0
+              fi
+              echo "Job did not complete within timeout — printing logs:"
+              kubectl logs -n keycloak "job/$JOB" --tail=200 || true
+              kubectl describe -n keycloak "job/$JOB" || true
+              exit 1
+            fi
+            echo "Realm-import Job not yet present (attempt $i/60); sleeping 10s…"
+            sleep 10
+          done
+          echo "Realm-import Job never appeared in 10 minutes."
+          kubectl get all -n keycloak
+          exit 1
+
+      - name: Read Keycloak admin password from secret
+        # The bitnami chart auto-generates a random admin password and
+        # stores it in secret `keycloak` under data key `admin-password`.
+        # Pipe-to-env hygiene per CLAUDE.md Rule 10: do NOT echo the
+        # plaintext, redirect through GITHUB_ENV (masked by Actions).
+        run: |
+          PASSWORD=$(kubectl get secret keycloak -n keycloak \
+            -o jsonpath='{.data.admin-password}' | base64 -d)
+          echo "::add-mask::${PASSWORD}"
+          echo "KC_ADMIN_PASSWORD=${PASSWORD}" >> $GITHUB_ENV
+
+      - name: Port-forward Keycloak service
+        # Primary Service `keycloak` listens on port 80 (forwarded to
+        # container port 8080). Port-forward in the background so the
+        # next step can curl localhost.
+        run: |
+          kubectl port-forward -n keycloak svc/keycloak 8080:80 \
+            > /tmp/pf.log 2>&1 &
+          echo $! > /tmp/pf.pid
+          # Wait until the port-forward accepts connections.
+          for i in $(seq 1 30); do
+            if curl -sf -o /dev/null http://localhost:8080/realms/master; then
+              echo "Port-forward live after ${i}s"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "Port-forward never came up — log follows:"
+          cat /tmp/pf.log || true
+          exit 1
+
+      - name: Verify sovereign realm exists
+        # The realm's discovery endpoint is unauthenticated for clients
+        # with publicClient=true (which `kubectl` is); a 200 here proves
+        # the realm-import Job actually wrote the realm into Keycloak's
+        # database, not just exited 0 with an empty no-op.
+        run: |
+          curl -sf http://localhost:8080/realms/sovereign | jq . \
+            || (echo "FAIL: sovereign realm not found"; exit 1)
+          echo "PASS: sovereign realm exists"
+
+      - name: Verify kubectl OIDC client is provisioned with redirect URI + groups mapper
+        # Use the master realm's admin-cli direct-access grant to mint an
+        # admin access-token, then call the Admin REST API to fetch the
+        # `kubectl` client by clientId. Asserts:
+        #   - client exists (length >= 1)
+        #   - publicClient: true (kubectl-oidc-login holds no secret)
+        #   - redirectUris contains http://localhost:8000 (kubectl-oidc-login default)
+        #   - the `groups` client scope is wired (id-token carries the
+        #     groups claim the api-server's --oidc-groups-claim flag depends on)
+        run: |
+          ADMIN_TOKEN=$(curl -sf -X POST \
+            -H 'Content-Type: application/x-www-form-urlencoded' \
+            -d 'grant_type=password' \
+            -d 'client_id=admin-cli' \
+            -d 'username=admin' \
+            -d "password=${KC_ADMIN_PASSWORD}" \
+            http://localhost:8080/realms/master/protocol/openid-connect/token \
+            | jq -r .access_token)
+          if [ -z "$ADMIN_TOKEN" ] || [ "$ADMIN_TOKEN" = "null" ]; then
+            echo "FAIL: could not obtain admin access-token from master realm"
+            exit 1
+          fi
+          echo "::add-mask::${ADMIN_TOKEN}"
+
+          CLIENTS=$(curl -sf -H "Authorization: Bearer $ADMIN_TOKEN" \
+            'http://localhost:8080/admin/realms/sovereign/clients?clientId=kubectl')
+
+          COUNT=$(echo "$CLIENTS" | jq 'length')
+          if [ "$COUNT" -lt 1 ]; then
+            echo "FAIL: kubectl OIDC client NOT found in sovereign realm"
+            echo "Admin API response: $CLIENTS"
+            exit 1
+          fi
+          echo "PASS: kubectl OIDC client exists ($COUNT match)"
+
+          # Print the relevant subset of the client config (no secrets —
+          # publicClient: true means there's nothing sensitive here).
+          echo "$CLIENTS" | jq '.[0] | {
+            clientId,
+            publicClient,
+            standardFlowEnabled,
+            redirectUris,
+            defaultClientScopes
+          }'
+
+          # Assert redirectUris contains localhost:8000 (kubectl-oidc-login default).
+          if ! echo "$CLIENTS" | jq -e '.[0].redirectUris | any(. == "http://localhost:8000")' >/dev/null; then
+            echo "FAIL: kubectl client redirectUris does not contain http://localhost:8000"
+            exit 1
+          fi
+          echo "PASS: kubectl client redirectUris contains http://localhost:8000"
+
+          # Assert publicClient: true.
+          if ! echo "$CLIENTS" | jq -e '.[0].publicClient == true' >/dev/null; then
+            echo "FAIL: kubectl client is not publicClient=true"
+            exit 1
+          fi
+          echo "PASS: kubectl client is publicClient=true"
+
+          # Assert the `groups` client scope is in defaultClientScopes
+          # (the realm-import wires it as a default scope so every
+          # id-token carries the `groups` claim without per-token opt-in).
+          if ! echo "$CLIENTS" | jq -e '.[0].defaultClientScopes | any(. == "groups")' >/dev/null; then
+            echo "FAIL: kubectl client does not include 'groups' in defaultClientScopes"
+            exit 1
+          fi
+          echo "PASS: kubectl client has 'groups' default client scope"
+
+          # Cross-check: the realm-level client scope `groups` carries
+          # the oidc-group-membership-mapper protocolMapper.
+          SCOPES=$(curl -sf -H "Authorization: Bearer $ADMIN_TOKEN" \
+            'http://localhost:8080/admin/realms/sovereign/client-scopes')
+          MAPPER=$(echo "$SCOPES" | jq '
+            .[] | select(.name == "groups") |
+            .protocolMappers // [] |
+            map(select(.protocolMapper == "oidc-group-membership-mapper")) |
+            length
+          ')
+          if [ "$MAPPER" != "1" ]; then
+            echo "FAIL: groups client scope missing oidc-group-membership-mapper"
+            echo "$SCOPES" | jq '.[] | select(.name == "groups")'
+            exit 1
+          fi
+          echo "PASS: groups client scope has oidc-group-membership-mapper wired"
+
+      - name: Stop port-forward
+        if: always()
+        run: |
+          if [ -f /tmp/pf.pid ]; then
+            kill "$(cat /tmp/pf.pid)" 2>/dev/null || true
+          fi
+
+      - name: Summary
+        if: always()
+        # Capture cluster state + realm-import Job logs in the workflow
+        # summary so a failed run is debuggable without re-running.
+        # Per ticket acceptance: "if post-install Job fails, workflow log
+        # captures its full output".
+        run: |
+          {
+            echo '## Keycloak realm-import preflight — cluster state'
+            echo '```'
+            kubectl get jobs,statefulsets,pods,svc -n keycloak 2>&1 || true
+            echo '```'
+            echo
+            echo '## keycloak-config-cli Job logs (last 200 lines)'
+            echo '```'
+            kubectl logs -n keycloak \
+              -l app.kubernetes.io/component=keycloak-config-cli \
+              --tail=200 2>&1 || true
+            echo '```'
+            echo
+            echo '## keycloak StatefulSet pod logs (last 100 lines)'
+            echo '```'
+            kubectl logs -n keycloak sts/keycloak --tail=100 2>&1 || true
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/services-build.yaml

@@ -18,7 +18,12 @@ jobs:
       packages: write
     strategy:
       matrix:
-        service: [auth, catalog, gateway, tenant, domain, billing, provisioning, notification]
+        # `metering-sidecar` (#798) builds on the same Containerfile
+        # convention as the SME services but its image is consumed by
+        # the bp-newapi chart (chart/values.yaml `meteringSidecar.image`),
+        # NOT by products/catalyst/chart/templates/sme-services. The
+        # deploy job below skips it for that reason.
+        service: [auth, catalog, gateway, tenant, domain, billing, provisioning, notification, metering-sidecar]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -48,62 +53,214 @@ jobs:
     needs: build
     runs-on: ubuntu-latest
     permissions:
+      # contents: write — push the SHA + Chart.yaml patch bump back to main.
       contents: write
+      # actions: write — required for `gh workflow run` to dispatch
+      # blueprint-release.yaml after the deploy commit lands. Without
+      # this, the dispatch step returns HTTP 403 "Resource not
+      # accessible by integration" and blueprint-release never fires
+      # for deploy commits (issues #872, #712 — same root cause as the
+      # catalyst-build dispatch fix in PR #720).
+      actions: write
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Update deployment manifests with new SHA tags
+      # ──────────────────────────────────────────────────────────────
+      # Helper: bump the Chart.yaml patch version atomically with the
+      # SHA-tag rewrite so blueprint-release publishes a single coherent
+      # chart that bundles the freshly committed image refs.
+      #
+      # Why this exists (issue #872): without the bump, the merge commit
+      # for a chart-version-bumping PR triggers blueprint-release IN
+      # PARALLEL with services-build. blueprint-release packages the
+      # working tree at the merge SHA — which still has the OLD image
+      # SHA in templates/sme-services/*.yaml because services-build has
+      # not yet committed its deploy update. The chart at that version
+      # ships with stale images, and a manual no-op chart bump PR is
+      # the only way to republish (PR #865 chasing PR #864 was the live
+      # incident).
+      #
+      # By having the deploy step bump the patch version itself, the
+      # dispatched blueprint-release publishes a NEW chart version that
+      # — by construction — was packaged AFTER the SHA rewrite. No race.
+      # The manual chart-version field in the PR becomes the floor; the
+      # CI auto-bumps from there.
+      # ──────────────────────────────────────────────────────────────
+      - name: Update deployment manifests + bump chart patch version
+        id: rewrite
         run: |
           SHA=$(echo $GITHUB_SHA | head -c 7)
           DEPLOY_DIR="products/catalyst/chart/templates/sme-services"
+          CHART_YAML="products/catalyst/chart/Chart.yaml"
+          VALUES_YAML="products/catalyst/chart/values.yaml"
 
+          # ──────────────────────────────────────────────────────────
+          # Issue #953: 7 of 8 sme-services templates render their
+          # image as `{{ .Values.images.smeTag }}` — the chart's
+          # values.yaml `images.smeTag` field is the SINGLE source of
+          # truth for those Pods. Only `auth.yaml` keeps a hardcoded
+          # `image: ghcr.io/...:<sha>` line (held at the older shape
+          # because of a historical InvalidImageName quirk).
+          #
+          # Pre-#953 this loop only ran a sed against the hardcoded
+          # form. The 7 templated services silently no-op'd and the
+          # deploy commit reported "update sme service images to
+          # ${SHA}" while only auth.yaml actually rolled. Result:
+          # every fix to catalog/tenant/etc. shipped the merge but
+          # the live Pod kept running pre-fix bytes (caught live on
+          # otech113 — services-catalog Pod still on 95a06f5 after
+          # PR #951's commit `68927688` claimed it deployed).
+          #
+          # Fix: bump `images.smeTag` in values.yaml AS WELL AS the
+          # hardcoded auth.yaml line. The values.yaml bump rolls
+          # all 7 templated services on the next chart release; the
+          # auth.yaml sed keeps the special-cased Pod on the same
+          # SHA. This stays event-driven (the push to main triggers
+          # this workflow); cron is intentionally absent per
+          # CLAUDE.md (every workflow MUST be event-driven, never
+          # scheduled).
+          # ──────────────────────────────────────────────────────────
+
+          # Hardcoded form — auth.yaml only. Kept until auth.yaml is
+          # re-templated (issue #953 fix scope was values.yaml; the
+          # back-compat hardcoded loop is preserved so a future
+          # auth-template flip is a no-op for this workflow).
           for svc in auth catalog gateway tenant domain billing provisioning notification; do
             FILE="${DEPLOY_DIR}/${svc}.yaml"
             if [ -f "$FILE" ]; then
               sed -i "s|image: ${IMAGE_BASE}-${svc}:.*|image: ${IMAGE_BASE}-${svc}:${SHA}|" "$FILE"
-              echo "Updated ${svc} to SHA ${SHA}"
+              echo "Updated ${svc} hardcoded image refs (no-op for templated forms) to SHA ${SHA}"
             fi
           done
 
+          # Templated form — bump `images.smeTag` in values.yaml so
+          # the 7 templated services (catalog, gateway, tenant,
+          # domain, billing, provisioning, notification) all roll on
+          # the next chart release. Match the canonical 2-space
+          # indented form `  smeTag: "<sha>"` (with quotes) emitted
+          # by the chart's existing values.yaml shape; refuse to
+          # auto-bump if the line is missing or shaped differently
+          # so a contributor renaming the field does not silently
+          # break this workflow's promise.
+          if ! grep -Eq '^  smeTag: "[A-Za-z0-9_.-]*"$' "${VALUES_YAML}"; then
+            echo "::error title=smeTag field missing or unparseable::Expected '  smeTag: \"<sha>\"' line in ${VALUES_YAML}; refusing to auto-bump."
+            exit 1
+          fi
+          sed -i "s|^  smeTag: \"[A-Za-z0-9_.-]*\"$|  smeTag: \"${SHA}\"|" "${VALUES_YAML}"
+          echo "Bumped ${VALUES_YAML} images.smeTag to ${SHA}"
+
+          # Patch-bump Chart.yaml `version:` and `appVersion:` (kept in
+          # lockstep — the chart reflects the bundled appVersion via
+          # convention). Pure-bash semver patch increment to avoid
+          # depending on yq in this job.
+          current=$(awk '/^version:/{print $2; exit}' "${CHART_YAML}" | tr -d '"')
+          if ! echo "$current" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then
+            echo "::error title=Unparseable chart version::Chart.yaml version='${current}' is not semver MAJOR.MINOR.PATCH; refusing to auto-bump."
+            exit 1
+          fi
+          major=$(echo "$current" | cut -d. -f1)
+          minor=$(echo "$current" | cut -d. -f2)
+          patch=$(echo "$current" | cut -d. -f3)
+          next="${major}.${minor}.$((patch + 1))"
+          sed -i "s|^version: .*$|version: ${next}|" "${CHART_YAML}"
+          sed -i "s|^appVersion: .*$|appVersion: ${next}|" "${CHART_YAML}"
+          echo "Bumped Chart.yaml ${current} -> ${next}"
+          echo "next_version=${next}" >> "$GITHUB_OUTPUT"
+
       - name: Commit and push manifest updates
+        id: deploy_commit
         run: |
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
           SHA=$(echo $GITHUB_SHA | head -c 7)
           DEPLOY_DIR="products/catalyst/chart/templates/sme-services"
+          CHART_YAML="products/catalyst/chart/Chart.yaml"
+          VALUES_YAML="products/catalyst/chart/values.yaml"
 
-          # Re-applies the sed substitution against whatever state main is
+          # Idempotent reset-and-rewrite. Parallel/back-to-back CI runs
-          # currently in. Needed because parallel/back-to-back CI runs both
+          # both try to update the same `image:` lines AND the same
-          # try to update the same image: lines — a plain `git pull --rebase`
+          # version line — `git pull --rebase` would hit content
-          # hits content conflicts since every SHA bump touches exactly the
+          # conflicts. Idempotent reset-to-origin + re-apply is
-          # same lines a previous run just wrote. Idempotent reset-and-resed
+          # conflict-free by construction. On retry we re-read whatever
-          # is conflict-free by construction.
+          # version origin/main currently holds and bump from THAT, so
-          apply_sed() {
+          # two concurrent runs produce strictly increasing patch
+          # versions instead of clobbering each other.
+          #
+          # Issue #953: rewrite() must mirror the values.yaml smeTag
+          # bump that the rewrite step does — otherwise a retry that
+          # reset-to-origin/main would leave values.yaml on the OLD
+          # SHA and only auth.yaml would carry the new SHA, recreating
+          # the original bug under any push-conflict scenario.
+          rewrite() {
             for svc in auth catalog gateway tenant domain billing provisioning notification; do
               FILE="${DEPLOY_DIR}/${svc}.yaml"
               if [ -f "$FILE" ]; then
                 sed -i "s|image: ${IMAGE_BASE}-${svc}:.*|image: ${IMAGE_BASE}-${svc}:${SHA}|" "$FILE"
               fi
             done
+            if ! grep -Eq '^  smeTag: "[A-Za-z0-9_.-]*"$' "${VALUES_YAML}"; then
+              echo "::error title=smeTag field missing on retry::Expected '  smeTag: \"<sha>\"' line in ${VALUES_YAML}."
+              exit 1
+            fi
+            sed -i "s|^  smeTag: \"[A-Za-z0-9_.-]*\"$|  smeTag: \"${SHA}\"|" "${VALUES_YAML}"
+            current=$(awk '/^version:/{print $2; exit}' "${CHART_YAML}" | tr -d '"')
+            if ! echo "$current" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then
+              echo "::error title=Unparseable chart version on retry::Chart.yaml version='${current}' is not semver."
+              exit 1
+            fi
+            major=$(echo "$current" | cut -d. -f1)
+            minor=$(echo "$current" | cut -d. -f2)
+            patch=$(echo "$current" | cut -d. -f3)
+            next="${major}.${minor}.$((patch + 1))"
+            sed -i "s|^version: .*$|version: ${next}|" "${CHART_YAML}"
+            sed -i "s|^appVersion: .*$|appVersion: ${next}|" "${CHART_YAML}"
+            echo "${next}"
           }
 
           git add products/
-          git diff --staged --quiet && echo "No changes to commit" && exit 0
+          if git diff --staged --quiet; then
-          git commit -m "deploy: update sme service images to ${SHA}"
+            echo "No changes to commit"
+            echo "pushed=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          NEXT="${{ steps.rewrite.outputs.next_version }}"
+          git commit -m "deploy: update sme service images to ${SHA} + bump chart to ${NEXT}"
 
           for i in 1 2 3; do
-            if git push; then exit 0; fi
+            if git push; then
-            echo "push attempt $i failed — resetting to origin/main and re-applying sed"
+              echo "pushed=true" >> "$GITHUB_OUTPUT"
+              echo "next_version=${NEXT}" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo "push attempt $i failed — resetting to origin/main and re-applying rewrite"
             git fetch origin main
             git reset --hard origin/main
-            apply_sed
+            NEXT=$(rewrite)
             git add products/
             if git diff --staged --quiet; then
               echo "no changes after re-fetch — another run already shipped this SHA"
+              echo "pushed=false" >> "$GITHUB_OUTPUT"
               exit 0
             fi
-            git commit -m "deploy: update sme service images to ${SHA}"
+            git commit -m "deploy: update sme service images to ${SHA} + bump chart to ${NEXT}"
           done
           echo "push failed after 3 attempts"
           exit 1
+
+      # GITHUB_TOKEN-authored pushes do NOT re-trigger workflows by
+      # design, so a `push` path-trigger on blueprint-release.yaml is
+      # not enough on its own — we must explicitly dispatch. Same
+      # mechanism catalyst-build.yaml uses (PR #720) to publish the
+      # bp-catalyst-platform OCI artifact for the bumped chart.
+      - name: Trigger blueprint-release for the chart bump
+        if: steps.deploy_commit.outputs.pushed == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh workflow run blueprint-release.yaml \
+            --repo "${{ github.repository }}" \
+            --ref main \
+            -f blueprint=catalyst \
+            -f tree=products
+          echo "blueprint-release dispatched for products/catalyst @ main (chart ${{ steps.deploy_commit.outputs.next_version }})"

.github/workflows/sme-demo-e2e.yaml

@@ -0,0 +1,118 @@
+name: SME demo end-to-end (issue #805)
+
+# Playwright spec for the FIRST SME tenant happy path on a healthy
+# otech (parent epic openova-io/openova#795). Lives next to
+# .github/workflows/cosmetic-guards.yaml — same dev-server pattern,
+# but tagged @sme-demo so the two suites run independently.
+#
+# Mock-mode today: every back-end surface (tenant discovery,
+# /api/v1/sme/users, /api/v1/sme/tenants, /api/v1/sme/billing/ledger,
+# WordPress/OpenClaw/Webmail placeholders) is stubbed via page.route
+# (see e2e/lib/sme-fixtures.ts). The screenshot evidence the DoD
+# checklist requires is captured AT CI time and uploaded as an
+# artefact.
+#
+# Live-mode follow-up: once #804 (tenant provisioning pipeline) lands
+# and a fresh otech is provisioned, this workflow gets a sibling
+# matrix entry that opts out of the mocks and dials the real
+# console.acme.<otech-fqdn>.
+#
+# Per docs/INVIOLABLE-PRINCIPLES.md #4a (GitHub Actions is the only
+# build path), this workflow does NOT build any container images —
+# it only runs the Playwright suite against a freshly-installed dev
+# tree.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'products/catalyst/bootstrap/ui/**'
+      - '.github/workflows/sme-demo-e2e.yaml'
+  pull_request:
+    paths:
+      - 'products/catalyst/bootstrap/ui/**'
+      - '.github/workflows/sme-demo-e2e.yaml'
+  workflow_dispatch:
+
+jobs:
+  sme-demo:
+    name: SME demo Playwright happy path
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'npm'
+          cache-dependency-path: products/catalyst/bootstrap/ui/package-lock.json
+
+      - name: Install Catalyst UI dependencies
+        working-directory: products/catalyst/bootstrap/ui
+        run: npm ci
+
+      - name: Install Playwright browser (chromium)
+        working-directory: products/catalyst/bootstrap/ui
+        run: npx playwright install --with-deps chromium
+
+      - name: Boot Catalyst UI in sovereign mode
+        working-directory: products/catalyst/bootstrap/ui
+        env:
+          # Force sovereign mode so SovereignConsoleLayout's auth gate
+          # has a non-null sovereignFQDN to dispatch off; without this,
+          # localhost falls into catalyst-zero mode and the /console/*
+          # routes hang on `sov-auth-loading`.
+          VITE_CATALYST_MODE: sovereign
+          VITE_SOVEREIGN_FQDN: acme.otech.example
+          HOST: 0.0.0.0
+        run: |
+          nohup npm run dev > /tmp/catalyst-ui-dev.log 2>&1 &
+          echo $! > /tmp/catalyst-ui.pid
+
+      - name: Wait for Catalyst UI to be ready
+        run: |
+          for i in $(seq 1 60); do
+            if curl -sf -o /dev/null http://localhost:5173/wizard; then
+              echo "UI ready after ${i}s"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "UI failed to start in 60s — log follows:"
+          cat /tmp/catalyst-ui-dev.log || true
+          exit 1
+
+      - name: Run SME demo Playwright spec
+        working-directory: products/catalyst/bootstrap/ui
+        env:
+          PLAYWRIGHT_HOST: http://localhost:5173
+          PLAYWRIGHT_BASEPATH: /
+        run: npx playwright test e2e/sme-demo.spec.ts --grep "@sme-demo" --reporter=list
+
+      - name: Stop Catalyst UI
+        if: always()
+        run: |
+          if [ -f /tmp/catalyst-ui.pid ]; then
+            kill "$(cat /tmp/catalyst-ui.pid)" 2>/dev/null || true
+          fi
+
+      - name: Upload screenshot evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: sme-demo-screenshots
+          path: products/catalyst/bootstrap/ui/e2e/screenshots/805-*
+          retention-days: 30
+
+      - name: Upload Playwright report (failure only)
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: sme-demo-report
+          path: |
+            products/catalyst/bootstrap/ui/playwright-report/
+            products/catalyst/bootstrap/ui/test-results/
+          retention-days: 7

.github/workflows/useraccess-controller-build.yaml

@@ -0,0 +1,116 @@
+name: Build useraccess-controller
+
+# useraccess-controller — UserAccess CR reconciler that REPLACES the
+# silently-broken Crossplane Composition path described in
+# docs/EPICS-1-6-unified-design.md §3.5. Slice C5 of EPIC-0 (#1095, P0).
+#
+# Per docs/INVIOLABLE-PRINCIPLES.md #4a "GitHub Actions is the only build
+# path" — this workflow is the canonical (and only) way to produce a
+# `ghcr.io/openova-io/openova/useraccess-controller:<sha>` image.
+#
+# Trigger model is event-driven per the openova-private CLAUDE.md
+# coupled rule: push-on-main is the canonical trigger; workflow_dispatch
+# is the manual override for ad-hoc rebuilds. NO cron.
+
+on:
+  push:
+    paths:
+      - 'core/controllers/useraccess/**'
+      - 'core/controllers/internal/**'
+      - 'core/controllers/go.mod'
+      - 'core/controllers/go.sum'
+      - '.github/workflows/useraccess-controller-build.yaml'
+    branches: [main]
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'core/controllers/useraccess/**'
+      - 'core/controllers/internal/**'
+      - 'core/controllers/go.mod'
+      - 'core/controllers/go.sum'
+      - '.github/workflows/useraccess-controller-build.yaml'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE: ghcr.io/openova-io/openova/useraccess-controller
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+          cache-dependency-path: core/controllers/go.sum
+
+      - name: go vet
+        working-directory: core/controllers
+        # Slice CC1 (#1095) consolidated the 5 Group C controllers into
+        # a single shared go.mod. Vet scoped to this controller's tree
+        # plus the shared internal/ helpers it depends on.
+        run: go vet ./useraccess/... ./internal/...
+
+      - name: go test (race + count=1)
+        working-directory: core/controllers
+        # Race + count=1 catches flakes that a cached run would hide.
+        # The reconciler suite uses controller-runtime's fake client —
+        # no envtest binaries needed, so the runner stays light.
+        run: go test -count=1 -race ./useraccess/... ./internal/...
+
+  build:
+    needs: test
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+    outputs:
+      sha_short: ${{ steps.vars.outputs.sha_short }}
+      digest: ${{ steps.build.outputs.digest }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set short SHA
+        id: vars
+        run: echo "sha_short=$(echo $GITHUB_SHA | head -c 7)" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push image
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          # Build context is the repository root so the Containerfile's
+          # COPY paths can reach both core/controllers/go.mod (the shared
+          # module root after slice CC1, #1095) and the per-controller
+          # tree under core/controllers/useraccess/.
+          context: .
+          file: core/controllers/useraccess/Containerfile
+          push: true
+          # SHA-pinned tag is the contract — production manifests
+          # consume :<sha>, never :latest. Two tags emitted:
+          #   :<short-sha>   — what cluster manifests reference
+          #   :<full-sha>    — long form for audit trails
+          tags: |
+            ${{ env.IMAGE }}:${{ steps.vars.outputs.sha_short }}
+            ${{ env.IMAGE }}:${{ github.sha }}
+          provenance: false
+          # Keep the image small and reproducible: no labels added by
+          # build-push-action's defaults; the Containerfile is the
+          # single source of truth.

.gitignore

@@ -8,7 +8,27 @@ platform/*/chart/Chart.lock
 products/*/chart/charts/
 products/*/chart/Chart.lock
 
+# Vendored upstream subcharts — exception to the above (issue #340).
+# bp-seaweedfs vendors seaweedfs/seaweedfs 4.22.0 with templates/shared/
+# security-configmap.yaml DELETED because it uses fromToml (Helm 3.13+)
+# which Flux helm-controller's bundled SDK doesn't have. The chart has
+# annotations.catalyst.openova.io/no-upstream=true to signal this to the
+# blueprint-release workflow's hollow-chart guard.
+!platform/seaweedfs/chart/charts/
+!platform/seaweedfs/chart/charts/**
+
 # Node + dev artifacts (untracked already, listed here for clarity).
 **/node_modules/
 **/dist/
 **/.astro/
+
+# OpenTofu / Terraform local working dir — generated by `tofu init` and
+# never committed. The provider lock file (.terraform.lock.hcl) IS
+# committed alongside versions.tf so collaborators install identical
+# provider binaries; only the .terraform/ working dir + state files are
+# ignored.
+**/.terraform/
+**/terraform.tfstate
+**/terraform.tfstate.backup
+**/*.tfstate
+**/*.tfstate.backup

clusters/_template/bootstrap-kit/01-cilium.yaml

@@ -36,7 +36,7 @@ spec:
   chart:
     spec:
       chart: bp-cilium
-      version: 1.1.1
+      version: 1.2.0
       sourceRef:
         kind: HelmRepository
         name: bp-cilium
@@ -55,10 +55,17 @@ spec:
       retries: 3
   values:
     cilium:
-      # Enable L7 proxy so Cilium's chart installs the
+      # Phase-8a bug #15 (otech8 deployment 1bfc46347564467b 2026-05-01):
-      # ciliumenvoyconfigs / ciliumclusterwideenvoyconfigs CRDs that the
+      # cilium-agent waits forever for the operator to register
-      # cilium-agent waits for at startup. Without this, agent crash-loops
+      # ciliumenvoyconfigs + ciliumclusterwideenvoyconfigs CRDs.
-      # forever and the node.cilium.io/agent-not-ready taint never lifts.
+      # Setting `envoy.enabled: true` (chart-level) runs Envoy as a separate
+      # daemonset but does NOT register those CRDs — that requires
+      # `envoyConfig.enabled: true`, a separate upstream chart toggle.
+      # Without it, the agent's node taint `node.cilium.io/agent-not-ready`
+      # never lifts and every other HelmRelease (37 of them) blocks on its
+      # dependsOn chain.
+      envoyConfig:
+        enabled: true
       l7Proxy: true
       prometheus:
         enabled: false
@@ -73,3 +80,69 @@ spec:
           enabled: false
         ui:
           enabled: false
+
+      # ── Cilium ClusterMesh — multi-region peering ──────────────────
+      #
+      # Per ADR-0001 §9 + EPIC-6 #1101 (multi-region active-hotstandby DR),
+      # ClusterMesh is the canonical inter-region transport for replication
+      # and Service-of-type-global traffic between Sovereign peer clusters.
+      #
+      # cluster.name + cluster.id are PER-SOVEREIGN anchors; per
+      # docs/INVIOLABLE-PRINCIPLES.md #4 (never hardcode), they MUST come
+      # from the Flux Kustomization's postBuild.substitute block — which
+      # in turn flows from infra/hetzner/cloudinit-control-plane.tftpl
+      # (CLUSTER_MESH_NAME, CLUSTER_MESH_ID) and ultimately from the
+      # operator-supplied request.cluster_mesh_name + cluster_mesh_id at
+      # provision time. Mesh registry: docs/CLUSTERMESH-CLUSTER-IDS.md
+      # tracks the cluster.id allocation across the OpenOva fleet.
+      #
+      # NodePort 32379: clustermesh-apiserver Pod is exposed on every
+      # Cilium node so peers reach it over the Hetzner private network on
+      # `<cp-private-ip>:32379` WITHOUT requiring a Hetzner LoadBalancer
+      # per peer (LB count is project-quota'd). Hetzner firewall must
+      # open 32379/tcp from peer Sovereigns' Hetzner CIDRs.
+      #
+      # A Sovereign that does NOT join a mesh leaves CLUSTER_MESH_NAME
+      # empty (Flux envsubst rule: ${VAR:=default} -> "default" when
+      # unset/empty). The cilium subchart accepts an empty cluster.name
+      # provided cluster.id stays 0; the clustermesh-apiserver Pod still
+      # runs but no peer connects (single-cluster no-op).
+      cluster:
+        name: ${CLUSTER_MESH_NAME:=}
+        id: ${CLUSTER_MESH_ID:=0}
+      clustermesh:
+        useAPIServer: true
+        apiserver:
+          service:
+            type: NodePort
+            nodePort: 32379
+---
+# ─── Per-Sovereign Gateway API resources (issue #387) ────────────────────
+#
+# Cilium owns the GatewayClass (`cilium`) installed by the chart above
+# (gatewayAPI.enabled=true, envoy.enabled=true in platform/cilium/chart/
+# values.yaml). The single per-Sovereign Gateway listening on
+# *.${SOVEREIGN_FQDN}:443 lives here so it boots alongside the CNI
+# without needing a new bootstrap-kit slot — every Sovereign HTTP
+# blueprint (catalyst-platform, gitea, keycloak, harbor, grafana,
+# openbao, powerdns) attaches its HTTPRoute to this Gateway via
+# parentRefs.
+#
+# TLS material: a wildcard Certificate is requested from
+# letsencrypt-dns01-prod-powerdns (cert-manager + bp-cert-manager-
+# powerdns-webhook from #373; webhook calls contabo's central PowerDNS
+# at https://pdns.openova.io). The resulting Secret
+# `sovereign-wildcard-tls` is referenced by the Gateway listener.
+#
+# Cross-namespace HTTPRoute attachment: allowedRoutes.namespaces.from=All
+# permits every blueprint namespace (catalyst-system, gitea, keycloak,
+# harbor, grafana-system, openbao, powerdns-system) to bind without a
+# ReferenceGrant. This matches the Catalyst single-tenant Sovereign
+# model — cross-tenant isolation is enforced by per-tenant vClusters
+# (bp-vcluster), not by Gateway-level RBAC.
+#
+# Per ADR-0001 §9.4 and docs/INVIOLABLE-PRINCIPLES.md #4: this resource
+# only renders when ${SOVEREIGN_FQDN} is set by Flux envsubst at the
+# Sovereign apply time — contabo's bootstrap path does NOT include this
+# template, so Traefik continues to serve console.openova.io/nova
+# unchanged.

clusters/_template/bootstrap-kit/01a-gateway-api.yaml

@@ -0,0 +1,80 @@
+# bp-gateway-api — Catalyst bootstrap-kit Blueprint, slot 01a (between
+# bp-cilium and every chart that ships HTTPRoute templates). Installs the
+# upstream Kubernetes Gateway API CRDs (Standard channel — gatewayclasses,
+# gateways, httproutes, grpcroutes, referencegrants).
+#
+# Why this Blueprint exists (issue #503):
+#
+#   Cilium 1.16's chart `gatewayAPI.enabled=true` flag (set in
+#   platform/cilium/chart/values.yaml) wires up the cilium gateway
+#   controller and creates the `cilium` GatewayClass — but it does NOT
+#   install the gateway.networking.k8s.io CRDs themselves. Without those
+#   CRDs registered on the apiserver, every chart that references
+#   HTTPRoute / Gateway / GatewayClass resources fails install with:
+#
+#     no matches for kind "HTTPRoute" in version "gateway.networking.k8s.io/v1"
+#
+#   Phase-8a-preflight live deployment otech10 (e1a0cd6662872fcb,
+#   2026-05-01) hit exactly this: bp-harbor, bp-openbao, bp-powerdns
+#   reconciled to InstallFailed with the message above; the fix is to
+#   install the upstream Gateway API CRDs ahead of any chart that uses
+#   them. Same pattern as bp-crossplane-claims and
+#   bp-external-secrets-stores — split CRD install from CR application
+#   so Flux dependsOn can order them.
+#
+# Wrapper chart: platform/gateway-api/chart/
+# Reconciled by: Flux on the new Sovereign's k3s control plane.
+#
+# dependsOn: bp-cilium — Cilium owns the GatewayClass that the upstream
+# Gateway resources reference; this Blueprint just installs the CRD
+# schema. Sequencing CRDs after the CNI also ensures the apiserver has
+# a working pod network when the CRD apply lands.
+
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: bp-gateway-api
+  namespace: flux-system
+spec:
+  type: oci
+  interval: 15m
+  url: oci://ghcr.io/openova-io
+  secretRef:
+    name: ghcr-pull
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: bp-gateway-api
+  namespace: flux-system
+spec:
+  interval: 15m
+  releaseName: gateway-api
+  # CRDs are cluster-scoped; targetNamespace is just where the Helm
+  # release marker Secret lives. Using flux-system keeps the marker
+  # next to every other bootstrap-kit release.
+  targetNamespace: flux-system
+  dependsOn:
+    - name: bp-cilium
+  chart:
+    spec:
+      chart: bp-gateway-api
+      version: 1.1.0
+      sourceRef:
+        kind: HelmRepository
+        name: bp-gateway-api
+        namespace: flux-system
+  # Event-driven install: 5 CRDs apply in a single pass; nothing to wait
+  # for beyond apiserver acceptance. Helm Ready is sufficient — every
+  # downstream HelmRelease that needs the CRDs declares
+  # `dependsOn: bp-gateway-api` so Flux gates them on this release's
+  # Ready condition.
+  install:
+    disableWait: true
+    remediation:
+      retries: 3
+  upgrade:
+    disableWait: true
+    remediation:
+      retries: 3

clusters/_template/bootstrap-kit/03-flux.yaml

@@ -17,7 +17,7 @@
 # unrecoverable in-place.
 #
 # Mitigations applied here:
-#   1. bp-flux:1.1.2 pins the `flux2` subchart at 2.14.1 (= appVersion
+#   1. bp-flux:1.1.3 pins the `flux2` subchart at 2.14.1 (= appVersion
 #      2.4.0) which matches cloud-init's v2.4.0 install.yaml.
 #   2. spec.upgrade.preserveValues: true — never silently overwrite
 #      operator overlays on upgrade.
@@ -59,7 +59,7 @@ spec:
   chart:
     spec:
       chart: bp-flux
-      version: 1.1.2
+      version: 1.1.3
       sourceRef:
         kind: HelmRepository
         name: bp-flux

clusters/_template/bootstrap-kit/05a-reflector.yaml

@@ -0,0 +1,64 @@
+# bp-reflector — Catalyst bootstrap-kit Blueprint (slot 05a).
+# Installs emberstack/reflector — the canonical Kubernetes secret/configmap
+# mirror controller. By annotating flux-system/ghcr-pull with reflector
+# auto-enable, the pull secret propagates to every namespace automatically,
+# eliminating the ImagePullBackOff surface caused by cross-namespace secret
+# propagation gaps (issue #543).
+#
+# Slot ordering: after sealed-secrets (05), before spire (06).
+# dependsOn bp-cert-manager (02) — cert-manager CRDs must exist first.
+#
+# Wrapper chart: platform/reflector/chart/
+# Upstream: emberstack/reflector ~7.x
+# Reconciled by: Flux on the new Sovereign's k3s control plane.
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: reflector
+  labels:
+    catalyst.openova.io/sovereign: ${SOVEREIGN_FQDN}
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: bp-reflector
+  namespace: flux-system
+spec:
+  type: oci
+  interval: 15m
+  url: oci://ghcr.io/openova-io
+  secretRef:
+    name: ghcr-pull
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: bp-reflector
+  namespace: flux-system
+spec:
+  interval: 15m
+  releaseName: reflector
+  targetNamespace: reflector
+  dependsOn:
+    - name: bp-cert-manager
+  chart:
+    spec:
+      chart: bp-reflector
+      version: 1.0.0
+      sourceRef:
+        kind: HelmRepository
+        name: bp-reflector
+        namespace: flux-system
+  # Event-driven install: single-replica controller; install completes
+  # when manifests apply. disableWait per architecture convention —
+  # replaces blanket spec.timeout band-aid.
+  install:
+    disableWait: true
+    remediation:
+      retries: 3
+  upgrade:
+    disableWait: true
+    remediation:
+      retries: 3

clusters/_template/bootstrap-kit/06-spire.yaml

@@ -1,60 +0,0 @@
-# bp-spire — Catalyst bootstrap-kit Blueprint. Workload identity. SPIFFE/SPIRE issues 5-min rotating SVIDs to every Pod. Required by NATS JetStream and OpenBao below for SVID-based auth.
-#
-# Wrapper chart: platform/spire/chart/
-# Catalyst-curated values: platform/spire/chart/values.yaml
-# Reconciled by: Flux on the new Sovereign's k3s control plane.
-
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: spire-system
-  labels:
-    catalyst.openova.io/sovereign: ${SOVEREIGN_FQDN}
----
-apiVersion: source.toolkit.fluxcd.io/v1beta2
-kind: HelmRepository
-metadata:
-  name: bp-spire
-  namespace: flux-system
-spec:
-  type: oci
-  interval: 15m
-  url: oci://ghcr.io/openova-io
-  secretRef:
-    name: ghcr-pull
----
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: bp-spire
-  namespace: flux-system
-spec:
-  interval: 15m
-  releaseName: spire
-  targetNamespace: spire-system
-  dependsOn:
-    - name: bp-cert-manager
-  chart:
-    spec:
-      chart: bp-spire
-      version: 1.1.4
-      sourceRef:
-        kind: HelmRepository
-        name: bp-spire
-        namespace: flux-system
-  # Event-driven install: Helm completes when manifests apply, not when
-  # pods reach Ready. spire-server StatefulSet has a multi-minute Ready
-  # path (controller-manager waits for CRD informer cache sync, which is
-  # itself triggered by the spire-crds subchart's CRD install). Flux's
-  # `dependsOn` on downstream HRs (bp-nats-jetstream, bp-openbao) checks
-  # Ready=True on this HR independently, so disableWait is the correct
-  # shape — replaces the blanket spec.timeout: 15m band-aid from PR #221.
-  install:
-    disableWait: true
-    remediation:
-      retries: 3
-  upgrade:
-    disableWait: true
-    remediation:
-      retries: 3

clusters/_template/bootstrap-kit/06a-bp-self-sovereign-cutover.yaml

@@ -0,0 +1,229 @@
+# bp-self-sovereign-cutover — Catalyst bootstrap-kit Blueprint, slot 06a.
+#
+# Post-handover Self-Sovereignty Cutover. Installs DORMANT — see chart
+# README + ADR-0002. Eight step PodSpec ConfigMaps + the registry-pivot
+# DaemonSet land on the new Sovereign at HelmRelease apply time; the
+# catalyst-api cutover endpoint (issue #792) reads them by label
+# selector and stamps Jobs only on operator action.
+#
+# Slot 06a sits between the existing post-handover slots in the
+# bootstrap-kit ordering. It depends on bp-gitea + bp-harbor so the
+# step ConfigMaps reference real, healthy local Gitea + Harbor
+# Services at trigger time.
+#
+# Wrapper chart: platform/self-sovereign-cutover/chart/
+# Reconciled by: Flux on the new Sovereign's k3s control plane.
+#
+# Why disableWait: true
+#   The chart is dormant — no Job is created at install time. The only
+#   workload that actually runs is the registry-pivot DaemonSet, which
+#   never converges on its own (it waits for the cutover-status
+#   ConfigMap to flip registriesYamlActive=v2). disableWait: true makes
+#   Helm exit when the manifests apply rather than waiting on a Ready
+#   condition that never fires.
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: catalyst
+  labels:
+    catalyst.openova.io/sovereign: ${SOVEREIGN_FQDN}
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: bp-self-sovereign-cutover
+  namespace: flux-system
+spec:
+  type: oci
+  interval: 15m
+  # Pre-cutover (Phase-1) — sources from openova-io GHCR, identical to
+  # every other bootstrap-kit slot. The cutover itself (step 06,
+  # helmrepository-patches) is what flips this URL to the local Harbor
+  # post-handover; until then this Sovereign is soft-tethered like the
+  # rest of the kit.
+  url: oci://ghcr.io/openova-io
+  secretRef:
+    name: ghcr-pull
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: bp-self-sovereign-cutover
+  namespace: flux-system
+spec:
+  interval: 15m
+  releaseName: self-sovereign-cutover
+  # targetNamespace = catalyst because the catalyst-api cutover endpoint
+  # defaults its discovery namespace to "catalyst"
+  # (CATALYST_CUTOVER_NAMESPACE in
+  # products/catalyst/bootstrap/api/internal/handler/cutover.go). Keeping
+  # the chart resources colocated with catalyst-api avoids a cross-
+  # namespace selector + a CATALYST_CUTOVER_NAMESPACE env override on
+  # every Sovereign.
+  targetNamespace: catalyst
+  dependsOn:
+    # Both bp-gitea and bp-harbor must be Ready before the chart's
+    # PodSpec ConfigMaps reference real Services. The chart itself is
+    # dormant so an early apply is benign — but operator workflow
+    # ordering puts cutover after these two come up.
+    - name: bp-gitea
+    - name: bp-harbor
+  chart:
+    spec:
+      chart: bp-self-sovereign-cutover
+      # 0.1.1: Step-1 gitea-mirror script uses BusyBox-wget-compatible
+      # Authorization: Basic <b64> header instead of --user/--password
+      # which alpine/git's BusyBox wget does not support.
+      # 0.1.2: Step-1 explicitly creates the Gitea org BEFORE the repo —
+      # POST /orgs/<org>/repos returns 404 if the org is missing, the
+      # /user/repos fallback would create under gitea_admin (wrong path
+      # for the subsequent push). Caught live on otech103 2026-05-04.
+      # 0.1.3: replace `git push --mirror` with `git push --all + --tags`
+      # so Gitea's hooks don't decline GitHub-specific refs/pull/<n>
+      # refs (which --mirror would try to push). Branches+tags are what
+      # Flux GitRepository needs; PR refs are upstream-only metadata.
+      # 0.1.4: Step-1 uses `git clone --bare` (not --mirror) + explicit
+      # refspec push of refs/heads/* and refs/tags/* only. --all in a
+      # mirror clone still pushed refs/pull/* — caught live otech103.
+      # 0.1.5: harborInternalURL fix — bp-harbor service is `harbor-core`
+      # not `harbor-harbor-core` (release name doesn't double-prefix).
+      # Caught live otech103 — Step-2 curl exit 6 (couldn't resolve).
+      # 0.1.6: proxy-ghcr registryType "github" → "github-ghcr" (the
+      # canonical Harbor adapter name for GHCR proxy-cache, per Harbor
+      # 2.x docs). Caught live otech103 — Harbor 500 "adapter factory
+      # for github not found".
+      # 0.1.7: proxy-quay registryType "quay" → "docker-registry" —
+      # Harbor's "quay" adapter rejects project metadata.proxy_cache
+      # with HTTP 400. Quay speaks plain v2 so generic docker-registry
+      # is correct. Caught live otech103 — 4/7 proxy-cache projects
+      # were created OK, blocked at proxy-quay.
+      # 0.1.8: bitnami/kubectl tag :1.31 → :1.31.4 (bitnami doesn't tag
+      # at minor-version, only patch). Caught live otech103 — Step-5
+      # Pod hit DeadlineExceeded after 10m of ImagePullBackOff for
+      # docker.io/bitnami/kubectl:1.31 (404 not found).
+      # 0.1.9: bitnami/kubectl :1.31.4 ALSO 404 (Bitnami deprecated
+      # public Docker Hub in 2025). Switched to alpine/k8s:1.31.4 —
+      # canonical alpine-based image with kubectl + helm + k8s CLI
+      # surface, actively maintained.
+      # 0.1.10: catalystAPI.namespace `catalyst-platform` → `catalyst-
+      # system` (the actual Sovereign-side namespace). Caught live
+      # otech103 — Step-7 `deployment catalyst-api not found`.
+      # 0.1.11: Step-8 egress-block-test pivoted from CiliumNetworkPolicy
+      # (egressDeny + toFQDNs unsupported in Cilium 1.16) to a passive
+      # architectural-state assertion + ${durationSeconds}s survival
+      # window. Same proof shape, valid Cilium policy. Caught live
+      # otech103 — strict-decoding error 'unknown field toFQDNs'.
+      # 0.1.12: Step-8 verification tolerates slot-managed self-ref
+      # HelmRepositories (bp-newapi + bp-self-sovereign-cutover) which
+      # Flux Kustomization re-applies from bootstrap-kit slots after
+      # Step-6's patch. Data-plane impact null — they're not pulled
+      # again until next cutover cycle. Caught live otech103.
+      # 0.1.13: Step-8 survival window captures BASELINE NotReady set
+      # before entering the window, then only fails on NEW Ready=False
+      # transitions (regressions). Pre-existing failures (Crossplane
+      # provider CRD ordering, etc.) don't poison the sovereignty
+      # verdict — sovereignty asks "did cutover break anything", not
+      # "is the cluster perfect". Caught live otech103 — infrastructure
+      # -config Kustomization had been NotReady for 4h pre-cutover.
+      # 0.1.14: Step-1 gitea-mirror replaces one-shot create+push with
+      # Gitea native /repos/migrate `mirror=true` + mirror_interval=10m
+      # so the local Gitea polls upstream GitHub on a recurring 10-min
+      # interval. Closes the "Sovereign drifts from upstream main
+      # forever after Day-2 cutover" bug — hit twice on otech103
+      # 2026-05-04 requiring manual `git fetch` per chart rollout. (#870)
+      # 0.1.16: Auto-trigger via Helm post-install Job (#933). Handover
+      # is not "done" until cutover has run; the operator must NOT have
+      # to click a CTA. New `trigger.auto: true` (default) fires a
+      # post-install Job that POSTs /api/v1/sovereign/cutover/start
+      # on catalyst-api after the step ConfigMaps land. catalyst-api
+      # handles idempotency via the durable status ConfigMap, so the
+      # hook is safe on every install + every upgrade. Coupled with the
+      # cutoverStatusResponse.State field fix on the API side which
+      # closes the otech113 `invalid CutoverState: <undefined>` bug.
+      # 0.1.17: Two-bug fix surfaced live on otech113 2026-05-05 (#935):
+      #   Bug 1 — Step 02 (harbor-projects) Job in `catalyst` ns was
+      #     hitting `secret "harbor-core" not found` because the
+      #     upstream Harbor `harbor-core` Secret only exists in
+      #     `harbor` ns and K8s forbids cross-namespace secretKeyRef.
+      #     Fix lives in bp-harbor 1.2.14: a Catalyst-curated
+      #     `harbor-admin` Secret is now emitted in the harbor ns
+      #     with Reflector annotations mirroring it into `catalyst`
+      #     so the Job's secretKeyRef resolves automatically. This
+      #     chart's values.yaml `harbor.adminSecretRef.name` is now
+      #     `harbor-admin` (was `harbor-core`).
+      #   Bug 2 — 0.1.16 auto-trigger Job POSTed
+      #     /api/v1/sovereign/cutover/start which lives behind
+      #     RequireSession middleware → 401 forever (no session
+      #     cookie on an in-cluster Job). Fix: route through new
+      #     /api/v1/internal/cutover/trigger endpoint which lives
+      #     OUTSIDE RequireSession and validates the bearer SA token
+      #     via TokenReview. The Job now mounts its projected SA
+      #     token at /var/run/secrets/kubernetes.io/serviceaccount/
+      #     token and sends it as `Authorization: Bearer <token>`.
+      # 0.1.18: Auto-trigger readiness probe loops on 401 (#957).
+      #   0.1.17 polled /api/v1/sovereign/cutover/status to check
+      #   "is catalyst-api up yet?" That endpoint lives INSIDE
+      #   RequireSession and returned 401 to every unauthenticated
+      #   probe from the in-cluster Job. The probe treated 401 as
+      #   "API not ready" → loop never broke → /internal/cutover/
+      #   trigger was never called → cutover never fired (caught
+      #   live on otech113 2026-05-05). Fix: poll /healthz
+      #   (unauthenticated, always 200 when the process is up).
+      #   Also drops the pre-flight cutoverComplete=true short-
+      #   circuit since /internal/cutover/trigger is itself
+      #   idempotent.
+      # 0.1.19: Step-01 gitea-mirror DNS race + backoffLimit=0 (#968).
+      #   0.1.18 unblocked the auto-trigger so the cutover engine fired
+      #   correctly on otech115 (2026-05-05) — but Step-01 then failed
+      #   within 8s with `wget: bad address gitea-http.gitea.svc.cluster.
+      #   local`. The gitea Pod had reached Ready ~2-3s prior; cluster-
+      #   DNS endpoint propagation was still in flight. catalyst-api
+      #   stamped the Job with `backoffLimit=0` (cutover.go:584), so
+      #   one DNS miss was terminal and the cutover engine aborted all
+      #   8 steps. Fix is dual: (a) catalyst-api now stamps Jobs with
+      #   `backoffLimit=3` so a single miss is recoverable; (b) Step-01
+      #   bash script gains an explicit `nslookup` readiness loop (30 x
+      #   5s) at the top, before any wget call. Both layers are needed —
+      #   the in-script probe is fastest; the backoffLimit is the
+      #   safety net for any other transient pre-cluster-stable race.
+      # 0.1.20: Step-06 helmrepository-patches reverted by Flux (#970).
+      #   0.1.19 unblocked the cutover through Step-7, but Step-08
+      #   verify caught 38/38 HelmRepositories had reverted to
+      #   oci://ghcr.io/openova-io despite Step-06's job logs showing
+      #   `OK ${name} -> oci://harbor.<sov-fqdn>/openova-io` for each.
+      #   Root cause: Step-06 only `kubectl patch`ed the live K8s
+      #   objects; bootstrap-kit Kustomization reconciled YAML from
+      #   local Gitea every 1m, where the YAML still declared the
+      #   upstream URL, undoing each patch within ~30s. Fix: Step-06
+      #   now does both phases — (a) live kubectl patches as before,
+      #   then (b) clones local Gitea, sed-rewrites every
+      #   clusters/_template/bootstrap-kit/*.yaml declaration of
+      #   `url: oci://ghcr.io/openova-io` → local Harbor prefix,
+      #   commits, and pushes. Subsequent reconciles see local Harbor
+      #   as steady-state. Image bumped to alpine/k8s:1.31.4 (kubectl
+      #   + git in one image; verified live on otech116).
+      version: 0.1.23
+      sourceRef:
+        kind: HelmRepository
+        name: bp-self-sovereign-cutover
+        namespace: flux-system
+  install:
+    disableWait: true
+    remediation:
+      retries: 3
+  upgrade:
+    disableWait: true
+    remediation:
+      retries: 3
+  # Per-Sovereign overrides — the chart's values.yaml carries
+  # placeholders so smoke renders pass; the real coordinates land
+  # here via Flux postBuild ${SOVEREIGN_FQDN} substitution.
+  values:
+    sovereign:
+      fqdn: ${SOVEREIGN_FQDN}
+      harborInternalURL: http://harbor-core.harbor.svc.cluster.local
+      harborPublicURL: https://harbor.${SOVEREIGN_FQDN}
+      giteaInternalURL: http://gitea-http.gitea.svc.cluster.local:3000
+      giteaPublicURL: https://gitea.${SOVEREIGN_FQDN}

clusters/_template/bootstrap-kit/07-nats-jetstream.yaml

@@ -33,8 +33,10 @@ spec:
   interval: 15m
   releaseName: nats-jetstream
   targetNamespace: nats-system
-  dependsOn:
+  # No dependsOn: bp-spire was dropped (PR #665, founder direction
-    - name: bp-spire
+  # 2026-05-03 — Cilium WireGuard mesh handles east-west mTLS).
+  # NATS no longer needs SVID-based auth; the kernel-level WireGuard
+  # encryption between every pod covers the in-flight traffic.
   chart:
     spec:
       chart: bp-nats-jetstream

clusters/_template/bootstrap-kit/08-openbao.yaml

@@ -34,25 +34,88 @@ spec:
   releaseName: openbao
   targetNamespace: openbao
   dependsOn:
-    - name: bp-spire
+    # bp-gateway-api (issue #503): chart ships an HTTPRoute template at
+    # platform/openbao/chart/templates/httproute.yaml; the
+    # gateway.networking.k8s.io/v1 CRDs MUST be registered before this
+    # HelmRelease applies or install fails with `no matches for kind
+    # HTTPRoute`.
+    - name: bp-gateway-api
+    # bp-cnpg (issue #512): the OpenBao 3-node Raft post-install init Job
+    # (Helm hook weight 5) runs `bao operator init` and seals/unseals via
+    # Kubernetes auth; both paths require the cnpg PostgreSQL backing the
+    # OpenBao audit/storage adjuncts to be Ready, otherwise the hook
+    # blocks until Helm's install timeout (15m) expires. Phase-8a-preflight
+    # otech16 (2026-05-02): even with timeout=15m, the hook raced cnpg
+    # coming up. Adding the explicit dep makes Flux wait for bp-cnpg
+    # Ready=True before starting bp-openbao install. See issue #512.
+    - name: bp-cnpg
   chart:
     spec:
       chart: bp-openbao
-      version: 1.1.1
+      version: 1.2.14
       sourceRef:
         kind: HelmRepository
         name: bp-openbao
         namespace: flux-system
-  # Event-driven install: OpenBao 3-node Raft cluster requires manual
+  # Event-driven install: OpenBao 3-node Raft cluster goes through a
-  # unseal via `bao operator init` — pods stay sealed (Ready=False) until
+  # post-install init Job (issue #316) — `bao operator init` runs at
-  # an operator runs the unseal flow. Blocking Helm install on Ready=True
+  # Helm hook weight 5 and the Kubernetes-auth bootstrap Job at weight
-  # is structurally wrong for a sealed-by-default secret backend.
+  # 10. The StatefulSet pods stay sealed for ~30s while the init Job
-  # Replaces PR #221 spec.timeout: 15m.
+  # runs, so we keep `disableWait: true` (Helm Ready ≠ OpenBao
+  # initialised — the init hook drives that out-of-band). Replaces
+  # PR #221 spec.timeout: 15m.
   install:
     disableWait: true
+    timeout: 15m
     remediation:
       retries: 3
   upgrade:
     disableWait: true
+    timeout: 15m
     remediation:
       retries: 3
+  # Per-Sovereign overrides:
+  # - gateway.host (issue #387): wires the per-Sovereign hostname into
+  #   the HTTPRoute template (platform/openbao/chart/templates/httproute.yaml).
+  #   The HTTPRoute attaches to cilium-gateway/kube-system installed by
+  #   01-cilium.yaml.
+  # - autoUnseal.enabled (issue #316): activates the post-install init
+  #   Job + Kubernetes-auth bootstrap Job in the chart. Cloud-init
+  #   (infra/hetzner/cloudinit-control-plane.tftpl) writes the seed
+  #   Secret `openbao-recovery-seed` in the openbao namespace BEFORE
+  #   Flux applies this HelmRelease, so the init Job has the seed it
+  #   needs on first reconcile.
+  values:
+    gateway:
+      host: bao.${SOVEREIGN_FQDN}
+    autoUnseal:
+      enabled: true
+      # Issue #517 (cont): the chart's init-job.yaml + auth-bootstrap-job.yaml
+      # default baoAddress to `http://<release>-openbao:8200`, but with
+      # spec.releaseName=openbao the upstream openbao chart's `fullname`
+      # template returns just `openbao` (not `openbao-openbao`) because
+      # Release.Name CONTAINS chart name. The rendered Service is
+      # `openbao` in the openbao namespace. Override the default so the
+      # post-install Jobs can actually reach the server.
+      baoAddress: http://openbao.openbao.svc.cluster.local:8200
+    # Issue #517 (Phase-8a single-node): openbao upstream chart's
+    # 3-replica StatefulSet uses required pod-anti-affinity by hostname.
+    # On single-node Phase-8a Sovereigns this leaves 2/3 pods Pending
+    # forever, the openbao-init Job's wait-for-Ready loop times out, and
+    # the entire HR fails post-install. Drop to 1 replica until the
+    # workerCount > 0 path is wired — the autoUnseal flow does not
+    # require a quorum to bootstrap (Raft is still enabled, just one
+    # voter).
+    #
+    # CRITICAL — schema nesting (issue #517 root cause): platform/openbao/
+    # chart/Chart.yaml declares the upstream openbao chart as a Helm
+    # SUBCHART under `dependencies:`. Helm umbrella-chart convention
+    # requires subchart values to be nested under the dependency name
+    # (`openbao:`). Putting `server.ha.replicas` / `server.affinity` at
+    # the top level here is SILENTLY IGNORED — the upstream subchart
+    # never sees them and renders 3-replica + pod-anti-affinity.
+    openbao:
+      server:
+        ha:
+          replicas: 1
+        affinity: ""

clusters/_template/bootstrap-kit/09-keycloak.yaml

@@ -35,10 +35,13 @@ spec:
   targetNamespace: keycloak
   dependsOn:
     - name: bp-cert-manager
+    # bp-gateway-api (issue #503): chart ships an HTTPRoute template;
+    # gateway.networking.k8s.io/v1 CRDs must be registered first.
+    - name: bp-gateway-api
   chart:
     spec:
       chart: bp-keycloak
-      version: 1.1.2
+      version: 1.4.0
       sourceRef:
         kind: HelmRepository
         name: bp-keycloak
@@ -50,9 +53,19 @@ spec:
   # Replaces PR #221 spec.timeout: 15m.
   install:
     disableWait: true
+    timeout: 15m
     remediation:
       retries: 3
   upgrade:
     disableWait: true
+    timeout: 15m
     remediation:
       retries: 3
+  # Per-Sovereign overrides — issue #387 + #604:
+  # Wire the per-Sovereign hostname into the HTTPRoute template and
+  # sovereign realm ConfigMap (catalyst-ui redirect URIs). The HTTPRoute
+  # attaches to cilium-gateway/kube-system installed by 01-cilium.yaml.
+  values:
+    sovereignFQDN: ${SOVEREIGN_FQDN}
+    gateway:
+      host: auth.${SOVEREIGN_FQDN}

clusters/_template/bootstrap-kit/10-gitea.yaml

@@ -36,10 +36,23 @@ spec:
   targetNamespace: gitea
   dependsOn:
     - name: bp-keycloak
+    # bp-gateway-api (issue #503): chart ships an HTTPRoute template;
+    # gateway.networking.k8s.io/v1 CRDs must be registered first.
+    - name: bp-gateway-api
+    # bp-cnpg (issue #584): chart ships a CNPG Cluster CR;
+    # postgresql.cnpg.io/v1 CRD must be registered before bp-gitea
+    # applies so the Capabilities gate in cnpg-cluster.yaml creates
+    # the Cluster rather than skipping it silently.
+    - name: bp-cnpg
   chart:
     spec:
       chart: bp-gitea
-      version: 1.1.2
+      # 1.2.5: gitea-admin-secret carries reflector.v1.k8s.emberstack.com
+      # annotations so bp-reflector mirrors it into the catalyst ns where
+      # bp-self-sovereign-cutover Step 1 gitea-mirror Job mounts it. K8s
+      # forbids cross-namespace secretKeyRef; reflector is the canonical
+      # platform-level mirror. Caught live on otech103 2026-05-04.
+      version: 1.2.5
       sourceRef:
         kind: HelmRepository
         name: bp-gitea
@@ -59,11 +72,10 @@ spec:
   values:
     global:
       sovereignFQDN: ${SOVEREIGN_FQDN}
-    # gitea hostname is gitea.${SOVEREIGN_FQDN}. The DNS A record
+    # Per-Sovereign overrides — issue #387:
-    # was already published by the Phase-0 catalyst-dns helper.
+    # Cilium Gateway HTTPRoute exposes Gitea at gitea.${SOVEREIGN_FQDN}.
-    ingress:
+    # Upstream chart's own Ingress is disabled (gitea.ingress.enabled=false
-      hosts:
+    # in platform/gitea/chart/values.yaml) — Sovereigns ingress through
-        - host: gitea.${SOVEREIGN_FQDN}
+    # cilium-gateway from clusters/_template/bootstrap-kit/01-cilium.yaml.
-          paths:
+    gateway:
-            - path: /
+      host: gitea.${SOVEREIGN_FQDN}
-              pathType: Prefix

clusters/_template/bootstrap-kit/11-powerdns.yaml

@@ -77,10 +77,30 @@ spec:
   targetNamespace: powerdns
   dependsOn:
     - name: bp-cert-manager
+    # bp-gateway-api (issue #503): chart ships an api-httproute.yaml
+    # template; gateway.networking.k8s.io/v1 CRDs must be registered first.
+    - name: bp-gateway-api
+    # bp-cnpg — chart's templates/cnpg-cluster.yaml renders a
+    # postgresql.cnpg.io/v1.Cluster gated on Capabilities.APIVersions.
+    # Without this dependency Helm renders before the CRD is registered,
+    # the gate evaluates false, the Cluster CR is silently skipped,
+    # CNPG never creates pdns-pg-app, and powerdns Pods fail at boot
+    # with "secret pdns-pg-app not found" (caught live during otech28).
+    - name: bp-cnpg
   chart:
     spec:
       chart: bp-powerdns
-      version: 1.1.3
+      # 1.2.0 (issue #827): adds multi-zone bootstrap Job. Sovereign
+      # parent zones (`omani.works`, `omani.trade`, ...) are POSTed to
+      # /api/v1/servers/localhost/zones at Helm post-install/post-upgrade
+      # time, idempotent on HTTP 409. The list below is populated from
+      # ${PARENT_DOMAINS_YAML} via Flux postBuild.substitute (see
+      # infra/hetzner/cloudinit-control-plane.tftpl); a single-zone
+      # fallback derived from ${SOVEREIGN_FQDN} keeps legacy
+      # provisioning paths operative.
+      # 1.2.1: zone-bootstrap Job needs /tmp emptyDir (readOnlyRootFS+
+      # curl -o /tmp/zone-resp). Caught live on otech103 2026-05-04.
+      version: 1.2.1
       sourceRef:
         kind: HelmRepository
         name: bp-powerdns
@@ -102,3 +122,55 @@ spec:
     disableWait: true
     remediation:
       retries: 3
+  # Per-Sovereign overrides — issue #387:
+  # Flip the REST API exposure from legacy Traefik Ingress to Cilium
+  # Gateway HTTPRoute. The Ingress template still renders (gated by
+  # api.enabled=true) but is harmless on a Sovereign without Traefik
+  # — the apiserver accepts the Ingress object; nothing routes it.
+  # The HTTPRoute attaches to cilium-gateway/kube-system and is the
+  # active path on Sovereigns.
+  values:
+    api:
+      host: pdns.${SOVEREIGN_FQDN}
+      gateway:
+        enabled: true
+    # DNS-01 wildcard cert: expose PowerDNS on NodePort 30053 so the
+    # Sovereign LB can forward :53 → PowerDNS. This is the NS-delegated
+    # endpoint that Let's Encrypt resolvers query when validating ACME
+    # DNS-01 challenges for *.${SOVEREIGN_FQDN}. Per ADR-0001 §9.4 the
+    # Sovereign must be self-sufficient post-handover — no Dynadot
+    # dependency for cert renewals.
+    anycast:
+      enabled: true
+      serviceName: powerdns-anycast
+      # NodePort on Sovereign Hetzner clusters: lb11 LB forwards TCP:53 to
+      # NodePort 30053; k3s iptables DNAT handles UDP:53 NodePort natively.
+      serviceType: NodePort
+      ports:
+        - name: dns-udp
+          port: 53
+          targetPort: 5353
+          nodePort: 30053
+          protocol: UDP
+        - name: dns-tcp
+          port: 53
+          targetPort: 5353
+          nodePort: 30053
+          protocol: TCP
+    # ─── Multi-zone bootstrap (issue #827, parent epic #825) ───────────
+    # The Sovereign creates one PowerDNS zone per parent domain at Helm
+    # post-install/post-upgrade time via the chart's zone-bootstrap Job
+    # (templates/zone-bootstrap-job.yaml). Idempotent on HTTP 409 so
+    # re-applies after upgrades or chart bumps never fail.
+    #
+    # Source of truth: ${PARENT_DOMAINS_YAML} is a Flux
+    # postBuild.substitute variable containing the operator-supplied
+    # parent-domain list rendered as a YAML inline-array literal, e.g.
+    #   PARENT_DOMAINS_YAML='[{name: "omani.works", role: "primary"}, {name: "omani.trade", role: "sme-pool"}]'
+    # When the operator brings only one parent domain (default
+    # zero-touch flow), cloud-init pre-renders this variable to a
+    # single-entry array derived from ${sovereign_fqdn} so the
+    # Sovereign still owns its own apex zone. See
+    # infra/hetzner/cloudinit-control-plane.tftpl for the substitute
+    # block that materialises the default.
+    zones: ${PARENT_DOMAINS_YAML}

clusters/_template/bootstrap-kit/12-external-dns.yaml

@@ -14,6 +14,10 @@
 #     bp-powerdns Service and reads the `powerdns-api-credentials` Secret
 #     it renders. Without bp-powerdns the ExternalDNS pod CrashLoops
 #     trying to dial a non-existent DNS API.
+#   - bp-reflector — Reflector mirrors the `powerdns-api-credentials`
+#     Secret from the `powerdns` namespace to `external-dns` automatically
+#     (issue #544). bp-reflector must be running before bp-external-dns
+#     installs so the reflected Secret is present when the pod starts.
 
 ---
 apiVersion: v1
@@ -47,10 +51,16 @@ spec:
   dependsOn:
     - name: bp-cert-manager
     - name: bp-powerdns
+    - name: bp-reflector
   chart:
     spec:
       chart: bp-external-dns
-      version: 1.1.2
+      # 1.1.7: companion CiliumNetworkPolicy with toEntities[kube-apiserver]
+      # so external-dns can reach the kube-apiserver on Cilium clusters
+      # (default policy-cidr-match-mode=""). Fixes #770 — the vanilla
+      # NetworkPolicy 0.0.0.0/0 ipBlock does NOT match apiserver traffic
+      # under Cilium's identity model.
+      version: 1.1.7
       sourceRef:
         kind: HelmRepository
         name: bp-external-dns

clusters/_template/bootstrap-kit/13-bp-catalyst-platform.yaml

@@ -40,10 +40,253 @@ spec:
   targetNamespace: catalyst-system
   dependsOn:
     - name: bp-gitea
+    # bp-gateway-api (issue #503): umbrella chart ships catalyst-ui +
+    # catalyst-api HTTPRoute templates; gateway.networking.k8s.io/v1
+    # CRDs must be registered first.
+    - name: bp-gateway-api
+    # bp-keycloak + bp-cnpg (issue #512): the catalyst-platform umbrella
+    # post-install Jobs bootstrap OIDC clients in Keycloak and seed
+    # PostgreSQL schemas for catalog-svc / projector / billing /
+    # provisioning. Both Keycloak and cnpg take 5+ minutes to reach Ready
+    # on a fresh Sovereign — without an explicit dep, the umbrella's
+    # hook starts before they're warm and times out at 15m.
+    # Phase-8a-preflight otech16 (2026-05-02): adding bp-keycloak +
+    # bp-cnpg here makes Flux wait for both Ready=True before starting
+    # the umbrella install, eliminating the race.
+    - name: bp-keycloak
+    - name: bp-cnpg
   chart:
     spec:
       chart: bp-catalyst-platform
-      version: 1.1.8
+      # 1.4.0 (issue #827): adds per-zone wildcard Certificate template.
+      # When `parentZones` is populated the chart renders one
+      # cert-manager.io/v1.Certificate per zone in kube-system; the
+      # Cilium Gateway listeners reference the per-zone Secrets. When
+      # `parentZones` is empty (legacy single-zone Sovereign) the chart
+      # falls back to a single Certificate covering `*.<sovereignFQDN>`
+      # so existing provisioning paths keep working.
+      # 1.4.1 (PR #839): RBAC dual-mode render fix (Helm + Kustomize).
+      # 1.4.2 (PR #841): POWERDNS env literal (no envsubst-mid-render).
+      # 1.4.3 (issue #859): auto-provision sme-pg CNPG Cluster +
+      # sme-secrets when ingress.marketplace.enabled=true so SME
+      # services land Ready on a fresh Sovereign without hand-rolled
+      # SealedSecrets. Catalyst-Zero (contabo) keeps its pre-existing
+      # clusters/contabo-mkt/apps/sme/data/* manifests — those are
+      # outside templates/kustomization.yaml's resource list so the
+      # contabo Kustomize-mode build is unaffected.
+      # 1.4.4 (issue #861): deploy FerretDB in `sme` ns + cross-ns
+      # CiliumNetworkPolicy from sme → valkey. Unblocks the 4 SME
+      # services (catalog, tenant, domain, provisioning) that pin to
+      # ferretdb.sme.svc.cluster.local for the MongoDB wire and the 2
+      # services (auth, gateway) that pin to valkey for session/state.
+      # cnpg-cluster.yaml extended to bootstrap sme_documents (FerretDB
+      # backing DB) alongside sme_billing.
+      # 1.4.5 (issue #863): mirror bp-valkey's auto-generated auth
+      # password from `valkey/valkey` Secret into `sme/sme-valkey-auth`
+      # via Helm lookup, and wire VALKEY_PASSWORD into auth + gateway
+      # Deployments. Clears the NOAUTH HELLO crashloop that started
+      # appearing after 1.4.4 made cross-ns Valkey reachable.
+      # 1.4.6 (issue #863 follow-up): rebuild chart artifact to bundle
+      # the rebuilt services-auth + services-gateway image (SHA fa4395f)
+      # that contains the ConnectValkeyWithAuth Go change. 1.4.5 shipped
+      # with the OLD image SHA baked in due to a race between the
+      # blueprint-release pipeline and the services-build deploy step.
+      # 1.4.7 (issue #866): mirror the gitea-admin password into
+      # `sme/provisioning-github-token` so the last 1/13 SME pod
+      # (provisioning) reaches Running 1/1 on a fresh Sovereign,
+      # completing the SME stack 12/13 → 13/13. Same lookup-and-mirror
+      # pattern as valkey-cross-ns-secret.yaml (#863).
+      # 1.4.8 (issue #868): fix marketplace UI PIN-signin — /api/*
+      # HTTPRoute now backendRefs sme/gateway:8080 (cross-namespace,
+      # authorised by ReferenceGrant). The previous catalyst-system/
+      # marketplace-api Service had zero backing Pods, so every signin
+      # POST 503'd at the gateway. Pairs with services-auth route alias
+      # /auth/send-pin → SendMagicLink (and /auth/verify-pin →
+      # VerifyMagicLink) so the UI's PIN-naming reaches the existing
+      # backend handler.
+      # 1.4.13 (issue #882): NEW templates/sme-services/sme-tenants-
+      # kustomization.yaml renders a Flux Kustomization in flux-system
+      # that watches ./clusters/<sov-fqdn>/sme-tenants — the path the
+      # catalyst-api SME-tenant orchestrator (sme_tenant_gitops.go)
+      # commits per-tenant overlays to. Without this, POST
+      # /api/v1/sme/tenants reached state=done optimistically but no
+      # K8s resources materialised because nothing reconciled the
+      # orchestrator's write target. Gated on
+      # ingress.marketplace.enabled — non-marketplace Sovereigns don't
+      # run the SME tenant pipeline.
+      # 1.4.14 (issue #879 follow-up): chart-version-only republish to
+      # bake catalyst-api image SHA 7bfd6df (the #879 fix commit) into
+      # values.yaml. 1.4.13 OCI bytes still reference the OLD image SHA
+      # because the deploy-bot updated values.yaml AFTER the chart was
+      # published. Same deploy-step race documented in 1.4.6 / 1.4.9 /
+      # 1.4.12 changelog.
+      # 1.4.15 (issue #887): auto-provision marketplace-api-secrets
+      # Secret on Sovereign install. templates/marketplace-api/
+      # deployment.yaml referenced a secretKeyRef on
+      # `marketplace-api-secrets` but the chart never rendered the
+      # Secret — caught live on otech103, marketplace-api in
+      # CreateContainerConfigError. Fix mirrors sme-secrets/
+      # valkey-cross-ns-secret/provisioning-github-token Helm-lookup
+      # persistence pattern. helm.sh/resource-policy: keep.
+      # 1.4.16 (#893/#889 follow-up): chart-version-only republish to
+      # bake catalyst-api image SHA 727fb2f (containing the parent-
+      # kustomization.yaml index + helmrepositories.yaml emit + correct
+      # per-blueprint sourceRef.name in tenant overlay templates) into
+      # values.yaml. Without this bump the OCI artifact still references
+      # the old image and the Sovereign's tenant orchestrator emits
+      # tenant overlays with stale openova-blueprints sourceRef.
+      # 1.4.17 (issue #901): unblock Sovereign Console login on every
+      # fresh provision. 3-bug chain:
+      #   1. NEW templates/catalyst-openova-kc-credentials-secret.yaml
+      #      auto-mirrors the canonical KC SA Secret (`keycloak/
+      #      catalyst-kc-sa-credentials`) into catalyst-system as
+      #      `catalyst-openova-kc-credentials` with the keys
+      #      api-deployment.yaml's PIN-auth env block expects. Gated on
+      #      `lookup "v1" "Secret" "keycloak" "catalyst-kc-sa-credentials"`
+      #      returning non-nil — renders only on Sovereign, skips on
+      #      contabo (which has its own hand-rolled Secret). Same Helm-
+      #      `lookup` persistence + `helm.sh/resource-policy: keep`
+      #      pattern as templates/marketplace-api/secret.yaml (#887).
+      #   2. SMTP host/port/from defaults flow through .Values.sovereign.
+      #      smtp.* (mail.openova.io:587 / noreply@openova.io). SMTP
+      #      user/pass mirrored from `catalyst-system/sovereign-smtp-
+      #      credentials` (#883) when present.
+      #   3. CATALYST_POST_AUTH_REDIRECT default flips from
+      #      /sovereign/wizard (mothership-only) to /sovereign/components
+      #      (post-handover Sovereign homepage). Per-Sovereign overlays
+      #      override via catalystApi.env additional-env patch.
+      # 1.4.18 (issue #910): NEW templates/sme-services/sme-namespace.yaml
+      # creates the `sme` namespace on Sovereigns where the marketplace
+      # is enabled. Without this, chart 1.4.17 install failed 23 times
+      # with `failed to create resource: namespaces "sme" not found` on
+      # every fresh franchised Sovereign with marketplace.enabled=true —
+      # caught live on otech105 (2026-05-05). Same dual-mode contract as
+      # the rest of templates/sme-services/* (gated on
+      # ingress.marketplace.enabled, excluded from kustomization.yaml's
+      # resources: list).
+      # 1.4.19 (issue #910 — Bugs 2 + 3): unblock Sovereign Console PIN-
+      # login on a freshly franchised cluster.
+      #   Bug 2: CATALYST_SESSION_COOKIE_DOMAIN literal flips from
+      #   `console.openova.io` to `""` (empty). On a Sovereign the
+      #   request host is console.<sov-fqdn>, so the previous hardcoded
+      #   value made the browser reject Set-Cookie (RFC 6265 §5.3 step 6
+      #   Domain mismatch) and every /api/* request landed without a
+      #   session, redirecting to /login forever. Empty value contract
+      #   (Domain attribute omitted → cookie binds to request host) is
+      #   correct on BOTH Sovereign (console.<sov-fqdn>) AND contabo
+      #   (console.openova.io — wizard + magic-link served from the
+      #   same host). Per-Sovereign overlays MAY override via
+      #   catalystApi.env additional-env patch for unusual topologies.
+      #
+      #   Bug 3: catalyst-openova-kc-credentials-secret.yaml's smtp-
+      #   user/smtp-pass lookup precedence inverts: SOURCE
+      #   (sovereign-smtp-credentials, seeded by A5's provisioner #883)
+      #   wins over the persisted target. Pre-1.4.19 target-wins meant
+      #   first-install rendered empty SMTP creds, persisted them, and
+      #   NEVER picked up A5's seeded bytes — POST /api/v1/auth/pin/
+      #   issue 502'd `email-send-failed` for the life of the cluster.
+      #   Source-wins makes every Flux 1m reconcile re-read the source.
+      #   KC fields keep "existing target wins" because bp-keycloak
+      #   auto-rotates the client-secret on every Helm upgrade and we
+      #   want that rotation to require explicit operator action
+      #   (delete the target) rather than auto-roll the catalyst-api
+      #   Pod.
+      # 1.4.20 (#924): Phase-2 SMTP cutover. SOURCE-wins precedence
+      # extended to (a) non-secret fields smtp-host/smtp-port/smtp-from
+      # so the per-Sovereign Stalwart relay (`mail.<sovereignFQDN>`)
+      # takes over from the mothership default (`mail.openova.io`) on
+      # the next reconcile after slot 95 (bp-stalwart-sovereign) lands,
+      # and (b) canonical key shape `smtp-user`/`smtp-pass` in addition
+      # to the legacy `user`/`password` source key shape — the new
+      # chart writes both shapes, this chart reads either.
+      # 1.4.22 (#915 SME blockers): six chart + orchestrator fixes
+      # unblocking alice signup gates 2-6 on franchised Sovereigns —
+      # issues #934 (auth SMTP empty), #940 (provisioning placeholder
+      # GITHUB_TOKEN + hardcoded upstream github.com), #941 (catalog
+      # migrateAppDeployable missing openclaw + stalwart-mail), #942
+      # (REDPANDA_BROKERS hardcoded to talentmesh — switched to NATS
+      # JetStream on Sovereigns per ADR-0001), #943 (bp-newapi
+      # silently skipped Deployment — paired bp-newapi 1.4.0 auto-
+      # provisions CNPG cluster + credentials Secret), #944 (CRITICAL
+      # cross-cluster pollution — GIT_BASE_PATH was hardcoded to
+      # contabo-mkt; chart values now template per-Sovereign with
+      # provisioning-binary Go-side validation guard refusing commits
+      # to foreign cluster trees). 2026-05-05.
+      # 1.4.23: deploy-bot auto-bump (services-auth image SHA roll).
+      # 1.4.24 (#934 follow-up): smeSecrets.smtp.{host,port,from,user}
+      # defaults populated with mothership relay (mail.openova.io:587)
+      # so SME auth Pod's PIN delivery (gate 2) works on Sovereigns
+      # whose A5-seeded sovereign-smtp-credentials Secret only carries
+      # smtp-user + smtp-pass without host/port/from. 2026-05-05.
+      # 1.4.25: deploy-bot auto-bump (sme-services 94ffe01 image roll).
+      # 1.4.26 (#957 follow-up): catalyst-api-cutover-driver
+      # ClusterRole gains `create tokenreviews.authentication.k8s.io`
+      # so /api/v1/internal/cutover/trigger can validate the
+      # auto-trigger Job's SA token via TokenReview. Without this rule
+      # every trigger call returned 502 "token-review-failed" on
+      # otech113 (chart 0.1.18 fixed the readiness loop but exposed
+      # this missing-RBAC bug as the next failure). 2026-05-05.
+      # 1.4.29 (#983 follow-up): Sovereign Console URL contract — clean
+      # root URLs (/dashboard /jobs /cloud …), sovereign_self.go store
+      # fallback (data renders the moment cutover-import lands without
+      # waiting for the orchestrator's chart-values overlay write).
+      # 2026-05-05.
+      # 1.4.95 (qa-loop iter-3 Fix #18, #1206): clusterroles +
+      # clusterrolebindings GVR added to k8scache.DefaultKinds + matching
+      # get/list/watch verbs on catalyst-api-cutover-driver ClusterRole
+      # (TC-122/196/199/248). Pairs with new CATALYST_BUILD_SHA +
+      # CATALYST_CHART_VERSION env vars on api-deployment.yaml so
+      # /api/v1/version returns the live SHA instead of `dev`/`0.0.0`
+      # (TC-261).
+      # 1.4.96 (qa-loop iter-3 Fix #18 follow-up): chart-packaging fix —
+      # .helmignore excludes crds/tests/ so Helm's pre-render CRD install
+      # no longer tries to apply the invalid Application sample as a CRD
+      # (the test fixture introduced by PR #1105). Without this every
+      # chart upgrade since 1.4.85 failed with `namespaces "acme" not
+      # found` — caught live on omantel 2026-05-09 attempting 1.4.84 ->
+      # 1.4.95. Bump pin so omantel + every other Sovereign sourcing
+      # this template picks up the fix on the next reconcile.
+      # 1.4.97 (qa-loop iter-4 Fix #24): apiextensions.k8s.io/v1
+      # customresourcedefinitions GVR added to k8scache.DefaultKinds +
+      # matching get/list/watch verbs on catalyst-api-cutover-driver
+      # ClusterRole (TC-199). Pairs with UI heading rename "Install
+      # Blueprint" → "Install — Blueprint Catalog" (TC-031). Per
+      # feedback_chroot_in_cluster_fallback.md every new GVR added to
+      # k8scache.DefaultKinds MUST get a matching rule on the cutover-
+      # driver SA — the chroot SovereignClient uses this SA via
+      # in-cluster fallback. Bump pin so omantel + every other Sovereign
+      # sourcing this template picks up the fix on the next reconcile.
+      # 1.4.99 (qa-loop iter-6 EPIC-6 Continuum DR target-state):
+      # adds singular `/continuum/{name}` route family + 5 new endpoints
+      # the matrix asserts (TC-312/324/326/329-335/339/343), seeds
+      # cont-omantel/qa-cnpg/pdm-1..3 fixtures + status seeders, ships
+      # cnpgpairs.dr.openova.io + pdms.dr.openova.io CRDs, ScheduledBackup
+      # + Backup fixtures (TC-337/338), and bumps tier-operator
+      # ClusterRole to grant continuums/cnpgpairs/pdms verbs (TC-344).
+      # Bp-crossplane-claims 1.1.2 carries the matching tier-operator
+      # extras update.
+      # 1.4.104 (qa-loop iter-7 Cluster-C Fix #36, #1231): target-state
+      # qa-fixtures stack (Org+Env+Blueprint+App) so application-controller
+      # reconciles qa-wp end-to-end into a real nginx Pod. Bp-qa-app
+      # sister chart at platform/qa-app/chart/ ships the real nginx
+      # bytes (CI publishes oci://ghcr.io/openova-io/bp-qa-app:0.1.0).
+      # Stacks on top of:
+      # 1.4.103 (Fix #37 follow-up): qa-continuum-status-seed Job FQN fix
+      # 1.4.105 (Fix #38 follow-up): qa-fixtures Application + Environment
+      # region defaults bumped to canonical 4-segment label
+      # `hz-fsn-rtz-prod` so the qa-wp Application from Fix #36 (#1231)
+      # validates against the CRD pattern `^[a-z]+-[a-z]+-[a-z]+-[a-z]+$`.
+      # Without this fix, `spec.regions[0]: Invalid value: "fsn1"` rejected
+      # the chart upgrade at admission and pinned omantel on the prior
+      # image SHA, blocking Fix #38's TC-141/TC-090/TC-383 from rolling.
+      # 1.4.102 (Fix #34 follow-up #1229): catalyst-api-cutover-driver
+      # ClusterRole grants update/patch/delete on workload kinds + scale
+      # subresources for the resource-action endpoints (PUT /k8s/.../scale,
+      # /restart, etc.) so chroot in-cluster fallback authorises through
+      # RBAC (TC-215, TC-218, TC-243, TC-247).
+      # 1.4.101 (Fix #37): EPIC-6 + EPIC-1 target-state qa-fixtures closeout
+      # — cnpg-clusters + Kyverno policy bundle.
+      version: 1.4.105
       sourceRef:
         kind: HelmRepository
         name: bp-catalyst-platform
@@ -53,12 +296,23 @@ spec:
   # environment-controller, blueprint-controller, billing). Inter-service
   # readiness via OTel/NATS subjects is multi-minute and not Helm's
   # concern. Replaces PR #221 spec.timeout: 15m.
+  #
+  # Issue #910 (otech105 incident, 2026-05-05): 15m was too tight for
+  # bp-catalyst-platform on a fresh franchised Sovereign with the full
+  # SME service stack (sme-services + tenant-orchestration + post-install
+  # secret mirror Jobs). The chart genuinely needs ~20 minutes worst
+  # case before remediation.retries kicks in. Bumped to 25m
+  # specifically for this umbrella chart — every other bp-* chart
+  # remains at its previous (or default) timeout because they install
+  # in well under 5 minutes empirically.
   install:
     disableWait: true
+    timeout: 25m
     remediation:
       retries: 3
   upgrade:
     disableWait: true
+    timeout: 25m
     remediation:
       retries: 3
   # Per-Sovereign overrides for the umbrella — sovereign-FQDN-derived hostnames
@@ -67,6 +321,17 @@ spec:
   values:
     global:
       sovereignFQDN: ${SOVEREIGN_FQDN}
+      # sovereignLBIP — Sovereign's load-balancer public IPv4. Issue #900:
+      # the Day-2 multi-domain add-domain flow uses this to pre-register
+      # glue records at the customer's registrar before flipping NS.
+      # Resolved via envsubst from `SOVEREIGN_LB_IP` set in the Sovereign
+      # cloud-init env (rendered into bootstrap-kit by infra/hetzner from
+      # hcloud_load_balancer.main.ipv4 — see infra/hetzner/main.tf:274).
+      # When the Sovereign cloud-init pre-dates #900 the env stays empty
+      # and the chart renders an empty `lbIP` ConfigMap key — catalyst-api
+      # then short-circuits the glue registration and falls back to plain
+      # set_ns (legacy behaviour).
+      sovereignLBIP: ${SOVEREIGN_LB_IP}
     ingress:
       hosts:
         console:
@@ -74,6 +339,65 @@ spec:
         admin:
           host: admin.${SOVEREIGN_FQDN}
         marketplace:
-          host: ${SOVEREIGN_FQDN}
+          host: marketplace.${SOVEREIGN_FQDN}
         api:
           host: api.${SOVEREIGN_FQDN}
+      # Marketplace mode (issue #710). Toggle to true via envsubst
+      # MARKETPLACE_ENABLED in the per-Sovereign overlay (catalyst-api
+      # writes this when the wizard's "Enable Marketplace" component is
+      # checked). When true, bp-catalyst-platform 1.3.0+ renders the
+      # marketplace + tenant-wildcard HTTPRoutes and the cross-namespace
+      # ReferenceGrant.
+      marketplace:
+        enabled: ${MARKETPLACE_ENABLED:-false}
+    # ─── Multi-zone parent domains (issue #827, parent epic #825) ──────
+    # One wildcard Certificate per parent zone, rendered by chart 1.4.0+
+    # into kube-system. Each cert renews independently; a stalled
+    # DNS-01 challenge on one zone never blocks another zone's renewal.
+    # Source of truth is the same ${PARENT_DOMAINS_YAML} variable used
+    # by bootstrap-kit slot 11 (bp-powerdns) so the two slots stay in
+    # lockstep on what the Sovereign considers a parent zone.
+    # When the operator brings only one parent domain (default
+    # zero-touch flow), cloud-init pre-renders this variable to a
+    # single-entry array derived from ${sovereign_fqdn}.
+    parentZones: ${PARENT_DOMAINS_YAML}
+    # ─── QA fixtures (qa-loop iter-6 Cluster-F + EPIC-6 iter-6) ────────
+    # Default-OFF on production; flipped to true via envsubst
+    # QA_FIXTURES_ENABLED=true on the per-Sovereign overlay for any
+    # Sovereign that participates in qa-loop matrix testing. Renders
+    # the 8-resource fixture stack (qa-omantel ns + qa-wp Application +
+    # cont-omantel Continuum CR + qa-cnpg CNPGPair + pdm-1/2/3 PDM CRs +
+    # ScheduledBackup + status seeder Jobs) the matrix asserts on. See
+    # products/catalyst/chart/templates/qa-fixtures/_README.txt.
+    qaFixtures:
+      enabled: ${QA_FIXTURES_ENABLED:-false}
+      namespace: ${QA_FIXTURES_NAMESPACE:-qa-omantel}
+      appName: ${QA_FIXTURES_APP:-qa-wp}
+      continuumName: ${QA_CONTINUUM_NAME:-cont-omantel}
+      cnpgPairName: ${QA_CNPGPAIR_NAME:-qa-cnpg}
+      # 4-segment canonical region label per Application + Environment
+      # CRD validation `^[a-z]+-[a-z]+-[a-z]+-[a-z]+$`. Legacy "fsn1"
+      # rejected at admission and pinned omantel on the prior image SHA
+      # (Fix #38 follow-up — caught after chart 1.4.105 still failed
+      # because the bootstrap-kit's release-config override beat the
+      # chart values.yaml default).
+      primaryRegion: ${QA_PRIMARY_REGION:-hz-fsn-rtz-prod}
+      standbyRegion: ${QA_STANDBY_REGION:-hz-hel-rtz-prod}
+      pdmZone: ${QA_PDM_ZONE:-openova.io}
+      # CNPG Cluster CR fixtures (Fix #37) — single-region by default;
+      # multi-region drill is owned by Continuum DR controllers + the
+      # cnpg-pair-controller. Override the *Region knobs once cross-
+      # region NodePort filtering is resolved (incidents.md §"Hetzner
+      # cross-region NodePort 32379 filtered").
+      cnpgPrimaryClusterName: ${QA_CNPG_PRIMARY_CLUSTER:-cluster-primary}
+      cnpgReplicaClusterName: ${QA_CNPG_REPLICA_CLUSTER:-cluster-replica}
+      cnpgPrimaryRegion: ${QA_CNPG_PRIMARY_REGION:-hz-fsn-rtz-prod}
+      cnpgReplicaRegion: ${QA_CNPG_REPLICA_REGION:-hz-fsn-rtz-prod}
+      cnpgImage: ${QA_CNPG_IMAGE:-ghcr.io/cloudnative-pg/postgresql:16.4-1}
+      cnpgStorageClass: ${QA_CNPG_STORAGE_CLASS:-local-path}
+      cnpgStorageSize: ${QA_CNPG_STORAGE_SIZE:-1Gi}
+      # Kyverno baseline policies (Fix #37). disallow-privileged-containers
+      # ships in Enforce mode; the other 18 baseline policies in Audit so
+      # the matrix sees ClusterPolicyReports without blocking platform
+      # pods. Soft-launch by setting Audit on a fresh Sovereign.
+      kyvernoEnforceMode: ${QA_KYVERNO_ENFORCE_MODE:-Enforce}

clusters/_template/bootstrap-kit/14-crossplane-claims.yaml

@@ -30,7 +30,6 @@ metadata:
   namespace: flux-system
 spec:
   interval: 15m
-  timeout: 15m
   releaseName: crossplane-claims
   targetNamespace: crossplane-system
   # bp-crossplane installs the apiextensions.crossplane.io/v1 CRDs
@@ -50,9 +49,15 @@ spec:
         kind: HelmRepository
         name: bp-crossplane-claims
         namespace: flux-system
+  # Event-driven install: Helm completes when manifests apply, not when the
+  # XRD-backed CRs reach Ready. dependsOn on bp-crossplane already gates this
+  # HR on the upstream CRDs being live; disableWait replaces PR #221's
+  # blanket spec.timeout: 15m band-aid.
   install:
+    disableWait: true
     remediation:
       retries: 3
   upgrade:
+    disableWait: true
     remediation:
       retries: 3

clusters/_template/bootstrap-kit/15-external-secrets.yaml

@@ -57,7 +57,7 @@ spec:
   chart:
     spec:
       chart: bp-external-secrets
-      version: 1.0.0
+      version: 1.1.0
       sourceRef:
         kind: HelmRepository
         name: bp-external-secrets

clusters/_template/bootstrap-kit/15a-external-secrets-stores.yaml

@@ -0,0 +1,65 @@
+# bp-external-secrets-stores — Catalyst bootstrap-kit Blueprint, slot 15a
+# (sub-slot of 15, follows bp-external-secrets controller).
+#
+# Owns the default ClusterSecretStore CR(s) wiring ESO to bp-openbao.
+# Split from bp-external-secrets@1.0.0 (issue #331) to resolve the
+# CRD-ordering deadlock — Helm's `before-hook-creation` delete policy on
+# the in-line ClusterSecretStore hook ran a kubectl-style lookup of the
+# CR before the upstream chart's CRDs finished registering, deadlocking
+# the install with `no matches for kind ClusterSecretStore` (incident on
+# otech.omani.works 2026-04-30).
+#
+# Mirrors the bp-crossplane (controller) ↔ bp-crossplane-claims (CRs)
+# split shape from PR #247.
+#
+# Wrapper chart: platform/external-secrets-stores/chart/
+# Catalyst-curated values: platform/external-secrets-stores/chart/values.yaml
+# Reconciled by: Flux on the new Sovereign's k3s control plane.
+
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: bp-external-secrets-stores
+  namespace: flux-system
+spec:
+  type: oci
+  interval: 15m
+  url: oci://ghcr.io/openova-io
+  secretRef:
+    name: ghcr-pull
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: bp-external-secrets-stores
+  namespace: flux-system
+spec:
+  interval: 15m
+  releaseName: external-secrets-stores
+  targetNamespace: external-secrets-system
+  # Order — Flux will not start install until bp-external-secrets reaches
+  # Ready=True (which means: upstream ESO controller running AND CRDs
+  # registered) AND bp-openbao reaches Ready (the secret backend the
+  # ClusterSecretStore points at).
+  dependsOn:
+    - name: bp-external-secrets
+    - name: bp-openbao
+  chart:
+    spec:
+      chart: bp-external-secrets-stores
+      version: 1.0.0
+      sourceRef:
+        kind: HelmRepository
+        name: bp-external-secrets-stores
+        namespace: flux-system
+  # Event-driven install per docs/INVIOLABLE-PRINCIPLES.md #3 (Flux
+  # dependsOn is the gate, not Helm timeout).
+  install:
+    disableWait: true
+    remediation:
+      retries: 3
+  upgrade:
+    disableWait: true
+    remediation:
+      retries: 3

clusters/_template/bootstrap-kit/18-seaweedfs.yaml

@@ -55,7 +55,7 @@ spec:
   chart:
     spec:
       chart: bp-seaweedfs
-      version: 1.0.0
+      version: 1.1.1
       sourceRef:
         kind: HelmRepository
         name: bp-seaweedfs

clusters/_template/bootstrap-kit/19-harbor.yaml

@@ -3,15 +3,44 @@
 # container images so the Sovereign isn't dependent on ghcr.io for
 # day-2 image pulls; also hosts Org-private images per Application.
 #
+# Per ADR-0001 §13 (S3-aware app rule) + docs/omantel-handover-wbs.md
+# §3 + §3a, on Hetzner Sovereigns Harbor writes its blob backend
+# DIRECTLY to Hetzner Object Storage — NOT SeaweedFS, which is
+# reserved as a POSIX→S3 buffer for legacy POSIX-only writers and is
+# not in the minimal Sovereign set.
+#
+# Wrapper chart: platform/harbor/chart/ (umbrella over upstream
+# goharbor/harbor chart, Catalyst-curated values under the `harbor:`
+# key + a vendor-AGNOSTIC `objectStorage.s3.*` section that ships the
+# harbor-namespace credentials Secret in
+# REGISTRY_STORAGE_S3_{ACCESSKEY,SECRETKEY} envFrom shape).
+# Reconciled by: Flux on the new Sovereign's k3s control plane.
+#
+# Object Storage credential pattern (issue #371, vendor-agnostic since
+# #425, applied to bp-harbor in #383):
+#   - cloud-init writes flux-system/object-storage Secret with 5 keys:
+#     s3-endpoint / s3-region / s3-bucket / s3-access-key /
+#     s3-secret-key (operator-issued in the Hetzner Console; Hetzner
+#     exposes no Cloud API to mint S3 credentials. Future AWS / Azure /
+#     GCP / OCI Sovereigns provision the same Secret name + same keys
+#     via their respective `infra/<provider>/` Tofu modules — the seam
+#     is vendor-agnostic by name).
+#   - This HelmRelease references that Secret via Flux `valuesFrom`,
+#     pulling each key into the appropriate Helm value path. The
+#     umbrella chart's templates/objectstorage-credentials.yaml then
+#     synthesises a harbor-namespace Secret with
+#     REGISTRY_STORAGE_S3_ACCESSKEY / REGISTRY_STORAGE_S3_SECRETKEY
+#     keys, referenced via persistence.imageChartStorage.s3.existingSecret.
+#
+# dependsOn: bp-cnpg + bp-cert-manager. The earlier dependency on
+# bp-seaweedfs is REMOVED in 1.1.0 (cloud-direct architecture rule;
+# SeaweedFS is no longer a Harbor prerequisite on Sovereigns).
+#
 # Per docs/BOOTSTRAP-KIT-EXPANSION-PLAN.md §6.7 — Harbor sits in the
 # storage cohort (W2.K1) rather than apps cohort because it is a
-# consumer of CNPG (registry metadata DB) and SeaweedFS (blob backend),
+# consumer of CNPG (registry metadata DB), and its presence gates
-# and its presence gates Cosign signing in bp-sigstore (slot 32) and
+# Cosign signing in bp-sigstore (slot 32) and image pinning across
-# image pinning across all later HRs.
+# all later HRs.
-#
-# Wrapper chart: platform/harbor/chart/
-# Catalyst-curated values: platform/harbor/chart/values.yaml
-# Reconciled by: Flux on the new Sovereign's k3s control plane.
 
 ---
 apiVersion: v1
@@ -19,7 +48,7 @@ kind: Namespace
 metadata:
   name: harbor
   labels:
-    catalyst.openova.io/sovereign: SOVEREIGN_FQDN_PLACEHOLDER
+    catalyst.openova.io/sovereign: ${SOVEREIGN_FQDN}
 ---
 apiVersion: source.toolkit.fluxcd.io/v1beta2
 kind: HelmRepository
@@ -44,16 +73,33 @@ spec:
   targetNamespace: harbor
   # Harbor depends on:
   #   - bp-cnpg(16): registry metadata DB (postgresql.cnpg.io/v1.Cluster).
-  #   - bp-seaweedfs(18): registry blob backend (S3-compatible).
   #   - bp-cert-manager(02): registry endpoint TLS via ClusterIssuer.
+  # bp-seaweedfs dependency REMOVED per ADR-0001 §13 (cloud-direct).
   dependsOn:
     - name: bp-cnpg
-    - name: bp-seaweedfs
     - name: bp-cert-manager
+    # bp-gateway-api (issue #503): chart ships an HTTPRoute template;
+    # gateway.networking.k8s.io/v1 CRDs must be registered first.
+    - name: bp-gateway-api
   chart:
     spec:
       chart: bp-harbor
-      version: 1.0.0
+      # 1.2.15: hot-fix for issue #949 — admin-secret.yaml duplicate
+      # label keys (app.kubernetes.io/name, catalyst.openova.io/
+      # component) made Helm's strict YAML post-render reject the
+      # rendered manifest, blocking the upgrade chain on otech113.
+      # Labels in admin-secret.yaml are now inlined verbatim instead
+      # of `include "bp-harbor.labels"` + override, eliminating the
+      # collision.
+      # 1.2.14: Catalyst-curated `harbor-admin` Secret with Reflector
+      # mirror annotations into `catalyst` ns so the
+      # bp-self-sovereign-cutover Step 02 (harbor-projects) Job in
+      # `catalyst` can read HARBOR_ADMIN_PASSWORD via secretKeyRef
+      # without the cross-namespace forbiddance K8s enforces. Caught
+      # live on otech113 2026-05-05 (issue #935 Bug 1) — Step 02 was
+      # in CreateContainerConfigError for 11+ retries, blocking
+      # cutover indefinitely.
+      version: 1.2.15
       sourceRef:
         kind: HelmRepository
         name: bp-harbor
@@ -67,3 +113,59 @@ spec:
     disableWait: true
     remediation:
       retries: 3
+  # ── Vendor-agnostic Object Storage backend wiring (issue #383 / #425) ──
+  #
+  # Each entry below pulls a single key from the canonical
+  # flux-system/object-storage Secret (shipped by cloud-init in
+  # infra/<provider>/cloudinit-control-plane.tftpl) into the matching
+  # value path in the umbrella chart. Flux dereferences `valuesFrom` at
+  # HelmRelease apply time, so plaintext credentials never appear in
+  # this committed manifest.
+  #
+  # NOTE: targetPath uses dot notation; keys are required by default
+  # (`optional: false` is the implicit default).
+  valuesFrom:
+    - kind: Secret
+      name: object-storage
+      valuesKey: s3-bucket
+      targetPath: harbor.persistence.imageChartStorage.s3.bucket
+    - kind: Secret
+      name: object-storage
+      valuesKey: s3-region
+      targetPath: harbor.persistence.imageChartStorage.s3.region
+    - kind: Secret
+      name: object-storage
+      valuesKey: s3-endpoint
+      targetPath: harbor.persistence.imageChartStorage.s3.regionendpoint
+    - kind: Secret
+      name: object-storage
+      valuesKey: s3-access-key
+      targetPath: objectStorage.s3.accessKey
+    - kind: Secret
+      name: object-storage
+      valuesKey: s3-secret-key
+      targetPath: objectStorage.s3.secretKey
+  # Per-Sovereign overrides — issue #387 + #383:
+  # - gateway.host wires the per-Sovereign hostname into the HTTPRoute.
+  # - objectStorage.enabled: true engages the cloud-direct S3 backend
+  #   (Hetzner Object Storage on Hetzner Sovereigns).
+  # - harbor.persistence.imageChartStorage.type: s3 flips upstream chart
+  #   off the default filesystem mode.
+  # - harbor.persistence.imageChartStorage.s3.existingSecret matches the
+  #   credentials Secret name templated by the umbrella chart.
+  values:
+    gateway:
+      host: registry.${SOVEREIGN_FQDN}
+    objectStorage:
+      enabled: true
+      useExistingSecret: false
+      credentialsSecretName: harbor-objectstorage-credentials
+    harbor:
+      persistence:
+        imageChartStorage:
+          type: s3
+          s3:
+            existingSecret: harbor-objectstorage-credentials
+            v4auth: true
+            secure: true
+            storageclass: STANDARD

clusters/_template/bootstrap-kit/21-alloy.yaml

@@ -56,7 +56,7 @@ spec:
   chart:
     spec:
       chart: bp-alloy
-      version: 1.0.0
+      version: 1.0.1
       sourceRef:
         kind: HelmRepository
         name: bp-alloy

clusters/_template/bootstrap-kit/23-mimir.yaml

@@ -53,7 +53,7 @@ spec:
   chart:
     spec:
       chart: bp-mimir
-      version: 1.0.0
+      version: 1.0.2
       sourceRef:
         kind: HelmRepository
         name: bp-mimir

clusters/_template/bootstrap-kit/25-grafana.yaml

@@ -57,6 +57,9 @@ spec:
     - name: bp-mimir
     - name: bp-tempo
     - name: bp-keycloak
+    # bp-gateway-api (issue #503): chart ships an HTTPRoute template;
+    # gateway.networking.k8s.io/v1 CRDs must be registered first.
+    - name: bp-gateway-api
   chart:
     spec:
       chart: bp-grafana
@@ -73,3 +76,10 @@ spec:
     disableWait: true
     remediation:
       retries: 3
+  # Per-Sovereign overrides — issue #387:
+  # Wire the per-Sovereign hostname into the HTTPRoute template
+  # (platform/grafana/chart/templates/httproute.yaml). The HTTPRoute
+  # attaches to cilium-gateway/kube-system installed by 01-cilium.yaml.
+  values:
+    gateway:
+      host: grafana.${SOVEREIGN_FQDN}

clusters/_template/bootstrap-kit/26-langfuse.yaml

@@ -1,84 +0,0 @@
-# bp-langfuse — Catalyst Blueprint #26 (W2.K2 Observability batch).
-# Langfuse — LLM observability platform (traces, evaluations, prompt
-# management, cost attribution). Hooks into the Catalyst LLM gateway
-# (slot 40) once W2.K4 lands. CNPG-backed Postgres; Keycloak OIDC SSO.
-#
-# Wrapper chart:  platform/langfuse/chart/
-# Reconciled by:  Flux on the new Sovereign's k3s control plane, AFTER
-#                 bp-cnpg, bp-keycloak, bp-cert-manager are all Ready.
-#
-# dependsOn:
-#   - bp-cnpg (slot 16)         — Postgres backend for Langfuse traces /
-#                                 prompts / evaluations.
-#   - bp-keycloak (slot 09)     — OIDC IdP for SSO.
-#   - bp-cert-manager (slot 02) — TLS for the Langfuse Ingress.
-#
-# disableWait: Langfuse waits for its CNPG-managed `langfuse-app` Secret
-# and for upstream Bitnami subcharts to be filtered out at template time
-# (the chart sets `postgresql.deploy=false`, `redis.deploy=false`,
-# `clickhouse.deploy=false`, `s3.deploy=false` to route to bp-cnpg /
-# bp-valkey / bp-clickhouse / bp-seaweedfs respectively). Helm `--wait`
-# would block on the Deployment rollout, which the HelmRelease cannot
-# influence.
-#
-# Forward-prep notice — issue #215 (bp-langfuse:1.0.0 GHCR publish 500):
-#   At the time this HR file was authored, bp-langfuse:1.0.0 had not
-#   published to oci://ghcr.io/openova-io due to a Helm v3.16 + GHCR
-#   manifest interaction with langfuse's nested OCI subchart deps. W1.G
-#   is the concurrent track fixing the publish path. Until that lands,
-#   this HelmRelease will fail to install with a chart-pull error; this
-#   is expected and tracked in #215. The HR file is committed now so
-#   the moment the artifact is published, Flux reconciles the SeaweedFS-
-#   /CNPG-/Keycloak-Ready Sovereign to bring Langfuse online without a
-#   second deploy gate.
-
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: langfuse
-  labels:
-    catalyst.openova.io/sovereign: ${SOVEREIGN_FQDN}
----
-apiVersion: source.toolkit.fluxcd.io/v1beta2
-kind: HelmRepository
-metadata:
-  name: bp-langfuse
-  namespace: flux-system
-spec:
-  type: oci
-  interval: 15m
-  url: oci://ghcr.io/openova-io
-  secretRef:
-    name: ghcr-pull
----
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: bp-langfuse
-  namespace: flux-system
-spec:
-  interval: 15m
-  timeout: 15m
-  releaseName: langfuse
-  targetNamespace: langfuse
-  dependsOn:
-    - name: bp-cnpg
-    - name: bp-keycloak
-    - name: bp-cert-manager
-  chart:
-    spec:
-      chart: bp-langfuse
-      version: 1.0.0
-      sourceRef:
-        kind: HelmRepository
-        name: bp-langfuse
-        namespace: flux-system
-  install:
-    disableWait: true
-    remediation:
-      retries: 3
-  upgrade:
-    disableWait: true
-    remediation:
-      retries: 3

clusters/_template/bootstrap-kit/29-vpa.yaml

@@ -48,7 +48,7 @@ spec:
   chart:
     spec:
       chart: bp-vpa
-      version: 1.0.0
+      version: 1.0.1
       sourceRef:
         kind: HelmRepository
         name: bp-vpa

clusters/_template/bootstrap-kit/30-trivy.yaml

@@ -51,7 +51,7 @@ spec:
   chart:
     spec:
       chart: bp-trivy
-      version: 1.0.0
+      version: 1.0.3
       sourceRef:
         kind: HelmRepository
         name: bp-trivy

clusters/_template/bootstrap-kit/31-falco.yaml

@@ -49,7 +49,7 @@ spec:
   chart:
     spec:
       chart: bp-falco
-      version: 1.0.0
+      version: 1.0.1
       sourceRef:
         kind: HelmRepository
         name: bp-falco

clusters/_template/bootstrap-kit/34-velero.yaml

@@ -1,22 +1,37 @@
 # bp-velero — Catalyst bootstrap-kit Blueprint #34 (W2.K3, Tier 7 — Security/Policy).
-# Per-host-cluster backup engine. Catalyst-Zero pins backups to SeaweedFS
+#
-# (the unified S3 layer, slot 18) so backup data never leaves the
+# Per-host-cluster backup engine. Per ADR-0001 §13 (S3-aware app rule)
-# Sovereign at install time; per-Sovereign archival to a cloud backend
+# + docs/omantel-handover-wbs.md §3 + §3a, on Hetzner Sovereigns Velero
-# is wired in post-bootstrap via Crossplane.
+# writes its backups DIRECTLY to Hetzner Object Storage — NOT SeaweedFS,
+# which is reserved as a POSIX→S3 buffer for legacy POSIX-only writers
+# and is not in the minimal Sovereign set.
 #
 # Wrapper chart: platform/velero/chart/ (umbrella over upstream
 # vmware-tanzu/velero chart, Catalyst-curated values under the `velero:`
-# key — `seaweedfs` BackupStorageLocation provider, no cloud plugin
+# key + a vendor-AGNOSTIC `objectStorage.s3.*` section that ships the
-# pinned at install time).
+# velero-namespace credentials Secret in AWS-CLI INI format).
 # Reconciled by: Flux on the new Sovereign's k3s control plane.
 #
-# dependsOn:
+# Object Storage credential pattern (issue #371, vendor-agnostic since
-#   - bp-seaweedfs — Velero's BackupStorageLocation points at the
+# #425):
-#     in-cluster SeaweedFS S3 endpoint (`seaweedfs.seaweedfs.svc:8333`)
+#   - cloud-init writes flux-system/object-storage Secret with 5 keys:
-#     and reads the `seaweedfs-s3-credentials` Secret SeaweedFS renders
+#     s3-endpoint / s3-region / s3-bucket / s3-access-key /
-#     during install. Without bp-seaweedfs Ready, the BSL Phase sits
+#     s3-secret-key (operator-issued in the Hetzner Console; Hetzner
-#     `Unavailable` and Velero's first reconcile fails — every backup
+#     exposes no Cloud API to mint S3 credentials. Future AWS / Azure /
-#     CR queues with the same error until the dep lands.
+#     GCP / OCI Sovereigns provision the same Secret name + same keys
+#     via their respective `infra/<provider>/` Tofu modules — the seam
+#     is vendor-agnostic by name).
+#   - This HelmRelease references that Secret via Flux `valuesFrom`,
+#     pulling each key into the appropriate Helm value path. The
+#     umbrella chart's templates/objectstorage-credentials.yaml then
+#     synthesises a velero-namespace Secret with a `cloud` key in the
+#     AWS-CLI INI format upstream Velero expects (mounted at
+#     /credentials/cloud).
+#
+# dependsOn: none — Velero is independent of all other minimal-set
+# blueprints. Earlier revisions of this slot dependsOn'd bp-seaweedfs;
+# that dependency is REMOVED per the cloud-direct architecture rule
+# (SeaweedFS is no longer a Velero prerequisite on Sovereigns).
 
 ---
 apiVersion: v1
@@ -47,12 +62,10 @@ spec:
   interval: 15m
   releaseName: velero
   targetNamespace: velero
-  dependsOn:
-    - name: bp-seaweedfs
   chart:
     spec:
       chart: bp-velero
-      version: 1.0.0
+      version: 1.2.0
       sourceRef:
         kind: HelmRepository
         name: bp-velero
@@ -70,3 +83,61 @@ spec:
     disableWait: true
     remediation:
       retries: 3
+  # ── Vendor-agnostic Object Storage backend wiring (issue #425) ──────
+  #
+  # Each entry below pulls a single key from the canonical
+  # flux-system/object-storage Secret (shipped by cloud-init in
+  # infra/<provider>/cloudinit-control-plane.tftpl) into the matching
+  # value path in the umbrella chart. Flux dereferences `valuesFrom` at
+  # HelmRelease apply time, so plaintext credentials never appear in
+  # this committed manifest.
+  #
+  # NOTE: targetPath uses dot notation; array indices use [N]. Keys are
+  # required by default (`optional: false` is the implicit default).
+  valuesFrom:
+    - kind: Secret
+      name: object-storage
+      valuesKey: s3-bucket
+      targetPath: velero.configuration.backupStorageLocation[0].bucket
+    - kind: Secret
+      name: object-storage
+      valuesKey: s3-region
+      targetPath: velero.configuration.backupStorageLocation[0].config.region
+    - kind: Secret
+      name: object-storage
+      valuesKey: s3-endpoint
+      targetPath: velero.configuration.backupStorageLocation[0].config.s3Url
+    - kind: Secret
+      name: object-storage
+      valuesKey: s3-access-key
+      targetPath: objectStorage.s3.accessKey
+    - kind: Secret
+      name: object-storage
+      valuesKey: s3-secret-key
+      targetPath: objectStorage.s3.secretKey
+  # Baseline values supplied by the bootstrap-kit slot. Per-Sovereign
+  # overlays in clusters/<sovereign>/bootstrap-kit/34-velero.yaml MAY
+  # override any of these (e.g. a different bucket-name strategy, a
+  # different credentials Secret name, or `deployNodeAgent: true` for
+  # file-system backup) without changing this template.
+  values:
+    objectStorage:
+      enabled: true
+      useExistingSecret: false
+      credentialsSecretName: velero-objectstorage-credentials
+    velero:
+      backupsEnabled: true
+      credentials:
+        useSecret: true
+        existingSecret: velero-objectstorage-credentials
+      configuration:
+        backupStorageLocation:
+          - name: default
+            provider: aws
+            default: true
+            accessMode: ReadWrite
+            credential:
+              name: velero-objectstorage-credentials
+              key: cloud
+            config:
+              s3ForcePathStyle: "true"

clusters/_template/bootstrap-kit/49-bp-cert-manager-powerdns-webhook.yaml

@@ -0,0 +1,131 @@
+# bp-cert-manager-powerdns-webhook — Catalyst bootstrap-kit Blueprint #49.
+# (Slot 36 was reserved in the W2.K0 forward-declared DAG for `bp-stunner`;
+# this Phase-2 webhook lands at slot 49 — first free slot after the W2.K4
+# forward declarations end at 48. Source of truth: scripts/expected-
+# bootstrap-deps.yaml.)
+# DNS-01 ACME solver against contabo's central PowerDNS (authoritative for
+# omani.works) for wildcard TLS on *.${SOVEREIGN_FQDN}. Supersedes
+# bp-cert-manager-dynadot-webhook (slot 49b, dropped in this PR).
+# Closes openova#373.
+#
+# ──────────────────────────────────────────────────────────────────────────
+# Why this slot exists
+# ──────────────────────────────────────────────────────────────────────────
+# The per-Sovereign Gateway in 01-cilium.yaml requests a wildcard
+# Certificate covering `*.${SOVEREIGN_FQDN}` — e.g. `*.otechN.omani.works`.
+# omani.works itself is registered at Dynadot but is delegated to
+# ns1/2/3.openova.io which run on contabo's PowerDNS in the
+# openova-system namespace. Dynadot is NOT the API-level authority for
+# omani.works subdomains; contabo PowerDNS is.
+#
+# When Let's Encrypt validates a DNS-01 challenge for `*.otechN.omani.works`,
+# its resolvers walk the public DNS chain: Dynadot → ns1/2/3.openova.io
+# (contabo PowerDNS). Until pool-domain-manager has committed the per-
+# Sovereign NS delegation into contabo PowerDNS (and that delegation has
+# propagated), the Sovereign's own PowerDNS is INVISIBLE on the public
+# chain — LE queries contabo, gets NXDOMAIN, and the cert never issues.
+#
+# Caught live on otech43–46: manual workaround was to seed the challenge
+# TXT record directly in contabo PowerDNS. This blueprint automates that
+# write path: every Sovereign's cert-manager webhook calls contabo's
+# PowerDNS API at https://pdns.openova.io to PATCH the challenge TXT
+# record, regardless of whether the Sovereign's own DNS delegation has
+# sealed yet.
+#
+# ──────────────────────────────────────────────────────────────────────────
+# Wiring
+# ──────────────────────────────────────────────────────────────────────────
+# Wrapper chart: platform/cert-manager-powerdns-webhook/chart/
+# Catalyst-curated values: platform/cert-manager-powerdns-webhook/chart/values.yaml
+# Reconciled by: Flux on the new Sovereign's k3s control plane.
+#
+# dependsOn:
+#   - bp-cert-manager — provides the cert-manager.io CRDs + controllers.
+#                       Without this the ClusterIssuer + Certificate
+#                       resources templated by this blueprint can't apply.
+#
+# Note: this slot does NOT depend on bp-powerdns. The webhook calls
+# contabo's central PowerDNS (https://pdns.openova.io) — an out-of-cluster
+# endpoint — not the Sovereign's local PowerDNS. The Sovereign's
+# bp-powerdns slot (11) is still installed (it backs the Sovereign's own
+# subzone for app-level records via bp-external-dns), but it is NOT in
+# the cert-issuance path.
+#
+# Credentials: the chart's apiKeySecretRef points at a Secret named
+# `powerdns-api-credentials` in the cert-manager namespace. That Secret's
+# `api-key` value MUST match the API key configured on contabo's central
+# PowerDNS. It is provisioned onto every Sovereign by cloud-init at
+# control-plane boot time (mirrors the dynadot-api-credentials seeding
+# pattern; see infra/hetzner/cloudinit-control-plane.tftpl).
+#
+# Per docs/INVIOLABLE-PRINCIPLES.md #4 ("never hardcode") every URL/zone
+# is operator-overridable. ${SOVEREIGN_FQDN} is substituted by Flux
+# envsubst at the per-Sovereign apply time; contabo's bootstrap path
+# does NOT include this template (per ADR-0001 §9.4 contabo stays on
+# the legacy Traefik + per-host HTTP-01 stack).
+
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: bp-cert-manager-powerdns-webhook
+  namespace: flux-system
+spec:
+  type: oci
+  interval: 15m
+  url: oci://ghcr.io/openova-io
+  secretRef:
+    name: ghcr-pull
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: bp-cert-manager-powerdns-webhook
+  namespace: flux-system
+spec:
+  interval: 15m
+  releaseName: cert-manager-powerdns-webhook
+  # Co-located with cert-manager so the webhook's serving Certificate
+  # (issued by the chart's selfSigned + CA Issuers) and APIService
+  # caBundle injection live in the same namespace cert-manager itself
+  # watches. Mirrors upstream chart convention.
+  targetNamespace: cert-manager
+  dependsOn:
+    - name: bp-cert-manager
+  chart:
+    spec:
+      chart: bp-cert-manager-powerdns-webhook
+      version: 1.0.4
+      sourceRef:
+        kind: HelmRepository
+        name: bp-cert-manager-powerdns-webhook
+        namespace: flux-system
+  # Event-driven install: the chart's ClusterIssuer template uses a
+  # post-install Helm hook that runs AFTER cert-manager's CRDs land,
+  # so blocking on Helm `--wait` for the leaf Certificate to reach
+  # Ready is unnecessary. Replaces blanket spec.timeout band-aids.
+  install:
+    disableWait: true
+    remediation:
+      retries: 3
+  upgrade:
+    disableWait: true
+    remediation:
+      retries: 3
+  values:
+    # ─── PowerDNS API endpoint ──────────────────────────────────────────
+    # The chart's default value (https://pdns.openova.io — contabo's
+    # central PowerDNS, authoritative for omani.works) is correct for
+    # every Sovereign in the omani.works pool, so no override is needed
+    # here. Operators provisioning a Sovereign in a non-omani.works pool
+    # add a `powerdns: { host: "https://pdns.<other-pool>" }` override
+    # in their per-cluster overlay.
+
+    # ─── Paired ClusterIssuer ───────────────────────────────────────────
+    # Operator opts in here; the chart's default render skips this
+    # resource (skip-render pattern, lesson from #387 follow-up #402).
+    clusterIssuer:
+      enabled: true
+      name: letsencrypt-dns01-prod-powerdns
+      email: "ops@${SOVEREIGN_FQDN}"
+      acmeServer: "https://acme-v02.api.letsencrypt.org/directory"

clusters/_template/bootstrap-kit/50-cluster-autoscaler.yaml

@@ -0,0 +1,148 @@
+# bp-cluster-autoscaler-hcloud — Catalyst bootstrap-kit Blueprint #50
+# (Tier 5 — Scaling/Resilience). Slot 40 was already forward-declared
+# for bp-llm-gateway in scripts/expected-bootstrap-deps.yaml; this
+# blueprint lands at slot 50 — after the W2.K4 cohort + slot 49
+# (bp-cert-manager-powerdns-webhook) — to preserve the existing
+# numbering plan.
+#
+# Adds and removes Hetzner Cloud worker nodes on demand in response to
+# `FailedScheduling` events on the Sovereign's k3s cluster. Bounded by
+# the `min`/`max` node-group config the operator picked at launch.
+#
+# Live evidence motivating this blueprint (issue #767):
+#   otech92 — 2× cpx32 workers couldn't fit external-secrets-webhook
+#   because the bootstrap-kit's RAM aggregate (~14 GB across 35
+#   HelmReleases) exceeded the 2× 8 GB pool the operator chose. With
+#   cluster-autoscaler the Sovereign would have grown the pool to a
+#   third worker automatically.
+#
+# Wrapper chart: platform/cluster-autoscaler-hcloud/chart/ — umbrella
+# over upstream kubernetes/autoscaler cluster-autoscaler chart 9.46.6
+# (appVersion 1.32.0). Catalyst-curated values flow under the
+# `cluster-autoscaler:` key + a vendor-agnostic
+# `clusterAutoscalerHcloud.*` block that ships the namespace-local
+# Hetzner-API-token Secret (`hcloud-token`).
+#
+# Reconciled by: Flux on the new Sovereign's k3s control plane.
+#
+# Hetzner-token wiring (mirrors the velero/harbor object-storage pattern
+# in 19-harbor.yaml + 34-velero.yaml):
+#   - cloud-init writes `flux-system/cloud-credentials` Secret with the
+#     `hcloud-token` key (see infra/hetzner/cloudinit-control-plane.tftpl
+#     §"cloud-credentials-secret"). That Secret is the canonical Hetzner-
+#     API-token holder for every Day-2 mutation seam (Crossplane provider-
+#     hcloud, this autoscaler, future hcloud Floating-IP claims).
+#   - This HelmRelease lifts the `hcloud-token` value into the umbrella
+#     chart's `clusterAutoscalerHcloud.hcloudToken` value via Flux
+#     `valuesFrom`. The umbrella chart then synthesises a namespace-local
+#     `cluster-autoscaler/hcloud-token` Secret (templates/hetzner-token-
+#     secret.yaml) the upstream chart's `extraEnvSecrets.HCLOUD_TOKEN`
+#     wiring binds as the deployment's HCLOUD_TOKEN env var.
+#
+# dependsOn: (none) — cluster-autoscaler is independent of every other
+# bootstrap-kit blueprint at install time. The cloud-credentials Secret
+# is provisioned by cloud-init BEFORE Flux installs anything.
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cluster-autoscaler
+  labels:
+    catalyst.openova.io/sovereign: ${SOVEREIGN_FQDN}
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: bp-cluster-autoscaler-hcloud
+  namespace: flux-system
+spec:
+  type: oci
+  interval: 15m
+  url: oci://ghcr.io/openova-io
+  secretRef:
+    name: ghcr-pull
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: bp-cluster-autoscaler-hcloud
+  namespace: flux-system
+spec:
+  interval: 15m
+  releaseName: cluster-autoscaler
+  targetNamespace: cluster-autoscaler
+  chart:
+    spec:
+      chart: bp-cluster-autoscaler-hcloud
+      version: 1.0.0
+      sourceRef:
+        kind: HelmRepository
+        name: bp-cluster-autoscaler-hcloud
+        namespace: flux-system
+  # Event-driven install: cluster-autoscaler is a single Deployment +
+  # ServiceAccount + RBAC. Helm install completes when manifests apply;
+  # the binary's Hetzner-API connectivity check is a runtime concern,
+  # not a Helm-wait concern. disableWait keeps Flux's Ready signal
+  # aligned with manifest apply.
+  install:
+    disableWait: true
+    remediation:
+      retries: 3
+  upgrade:
+    disableWait: true
+    remediation:
+      retries: 3
+  # ── Hetzner-token + node-bootstrap wiring (issue #921) ─────────────
+  # Pulls keys from the canonical `flux-system/cloud-credentials`
+  # Secret cloud-init writes at Phase 0
+  # (infra/hetzner/cloudinit-control-plane.tftpl §"cloud-credentials-
+  # secret"):
+  #   - hcloud-token       → API token (mandatory)
+  #   - hcloud-cloud-init  → base64(cloud-init.yaml) — the autoscaler-
+  #                          spawned worker's bootstrap, identical to the
+  #                          Phase-0 worker user_data. Required by
+  #                          cluster-autoscaler 1.32.x's Hetzner provider
+  #                          (HCLOUD_CLOUD_INIT env var) — without it the
+  #                          autoscaler Pod exits at startup with FATAL
+  #                          "HCLOUD_CLUSTER_CONFIG or HCLOUD_CLOUD_INIT
+  #                          is not specified".
+  # Flux dereferences `valuesFrom` at HelmRelease apply time, so the
+  # plaintext payloads never appear in this committed manifest.
+  #
+  # The chart's templates/hetzner-node-config-secret.yaml renders these
+  # values into a namespace-local `cluster-autoscaler/hetzner-node-config`
+  # Secret which the upstream chart's `extraEnvSecrets.HCLOUD_CLOUD_INIT`
+  # binding lifts onto the deployment's env.
+  valuesFrom:
+    - kind: Secret
+      name: cloud-credentials
+      valuesKey: hcloud-token
+      targetPath: clusterAutoscalerHcloud.hcloudToken
+    - kind: Secret
+      name: cloud-credentials
+      valuesKey: hcloud-cloud-init
+      targetPath: clusterAutoscalerHcloud.cloudInit
+      # When older Sovereigns provisioned BEFORE issue #921 lack the
+      # hcloud-cloud-init key, Flux skips this entry rather than failing
+      # the entire HelmRelease — the chart's empty-string default keeps
+      # the upstream Deployment shape valid (the autoscaler will still
+      # FATAL at startup, surfacing the missing-cloud-init in Pod logs;
+      # operators rotate by re-running cloud-init or by patching
+      # cloud-credentials directly).
+      optional: true
+  # Per-Sovereign baseline values. clusters/<sovereign>/bootstrap-kit/
+  # 40-cluster-autoscaler.yaml MAY override `autoscalingGroups` to set
+  # the actual instanceType + region + min/max + name the Tofu module
+  # provisioned at Phase 0. The defaults below match the canonical
+  # otechN topology (cpx32 in fsn1, min 2 / max 10) so a vanilla
+  # Sovereign that forgets to patch this still gets a sensible
+  # autoscaler.
+  values:
+    cluster-autoscaler:
+      autoscalingGroups:
+        - name: workers
+          instanceType: cpx32
+          region: fsn1
+          minSize: 2
+          maxSize: 10

clusters/_template/bootstrap-kit/80-newapi.yaml

@@ -0,0 +1,212 @@
+# bp-newapi — Catalyst Application Blueprint, bootstrap-kit slot 80.
+# Multi-tenant LLM marketplace gateway. Ships in backend-only mode: the
+# OpenAI-compatible API at api.<sovereign-fqdn>/v1/* is customer-facing;
+# the upstream's portal UI is disabled at ingress; Catalyst replaces it
+# as the customer surface; NewAPI's admin UI at admin.<sovereign-fqdn>
+# is exposed only to ops staff (Keycloak-gated).
+#
+# This slot enables the SME-tenant turnkey experience (epic #795). The
+# Catalyst signup hook (delivered by unified-rbac in #802 against the
+# contract recorded in ADR-0003) reads the `catalyst-newapi-admin-token`
+# Secret rendered by this chart's ExternalSecret to issue per-user API
+# keys against NewAPI's admin API at `http://newapi.newapi.svc`.
+#
+# Wrapper chart: platform/newapi/chart/
+# Catalyst-curated values: platform/newapi/chart/values.yaml
+# Reconciled by: Flux on the new Sovereign's k3s control plane.
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: newapi
+  labels:
+    catalyst.openova.io/sovereign: ${SOVEREIGN_FQDN}
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: bp-newapi
+  namespace: flux-system
+spec:
+  type: oci
+  interval: 15m
+  url: oci://ghcr.io/openova-io
+  secretRef:
+    name: ghcr-pull
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: bp-newapi
+  namespace: flux-system
+spec:
+  interval: 15m
+  releaseName: newapi
+  targetNamespace: newapi
+  # bp-newapi depends on:
+  #   - bp-openbao(08): the secret backend the chart's ExternalSecret
+  #     pulls `ADMIN_API_TOKEN` from. Without OpenBao Ready, the
+  #     ExternalSecret never resolves and the Catalyst signup hook can't
+  #     reach the NewAPI admin API.
+  #   - bp-keycloak(09): the OIDC issuer for the ops-staff admin UI at
+  #     admin.<sovereign-fqdn>. Without Keycloak Ready, the OIDC
+  #     middleware can't redirect ops-staff requests.
+  #   - bp-cnpg(16): operator provisions the Postgres cluster for users,
+  #     credits, channels, and audit log via a Crossplane
+  #     PostgresqlInstance claim once cnpg is Ready. The DSN is mounted
+  #     into NewAPI via `database.existingSecret` (operator-set).
+  dependsOn:
+    - name: bp-openbao
+    - name: bp-keycloak
+    - name: bp-cnpg
+  chart:
+    spec:
+      chart: bp-newapi
+      # 1.4.0 (issue #943, 2026-05-05): auto-provision CNPG-backed
+      # Postgres + chart-emitted SESSION_SECRET/CRYPTO_SECRET so a
+      # Sovereign install lands a real Pod without operator intervention.
+      # Pre-#943 the Deployment silently skipped render whenever
+      # database.existingSecret OR credentials.existingSecret was
+      # empty (the bootstrap-kit overlay supplies neither), so NewAPI
+      # never came up and alice signup gate 5 (LLM) timed out. Both
+      # auto-provisions are capability-gated on bp-cnpg's CRD and
+      # operator-overridable per Inviolable Principle #4.
+      # 1.3.0: defaultChannels.qwenBankDhofar (channel #1 = Qwen3.6 @
+      # https://llm-api.omtd.bankdhofar.com) + post-install/post-upgrade
+      # `channel-seed` Helm hook Job that idempotently POSTs default
+      # channels into NewAPI's admin API. Issue #915 (epic SME tenant
+      # integration DoD: alice → OpenClaw → NewAPI → Qwen3.6@BankDhofar
+      # end-to-end).
+      # 1.2.0: Traefik Middleware gated behind ingress.middleware.enabled.
+      # 1.4.1 (issue #952, 2026-05-05): Pod imagePullSecrets templated +
+      # default to `[{name: ghcr-pull}]` so kubelet authenticates pulls
+      # of the PRIVATE newapi-mirror + metering-sidecar images. Paired
+      # with cloud-init adding `newapi` to flux-system/ghcr-pull's
+      # reflector auto-namespaces list.
+      version: 1.4.1
+      sourceRef:
+        kind: HelmRepository
+        name: bp-newapi
+        namespace: flux-system
+  # Event-driven install per docs/INVIOLABLE-PRINCIPLES.md #3 (Flux
+  # dependsOn is the gate, not Helm timeout). NewAPI itself starts in
+  # ~10 s once the Postgres DSN Secret is present; the long pole is
+  # waiting for the operator's Crossplane claim to materialise the DB.
+  install:
+    disableWait: true
+    remediation:
+      retries: 3
+  upgrade:
+    disableWait: true
+    remediation:
+      retries: 3
+  # Per-Sovereign overrides — the operator MUST supply at install time:
+  #   - ingress.host           = api.${SOVEREIGN_FQDN}
+  #   - ingress.adminHost      = admin.${SOVEREIGN_FQDN}
+  #   - auth.adminUI.keycloak.issuer = https://auth.${SOVEREIGN_FQDN}/realms/ops
+  #   - database.existingSecret      = Postgres DSN Secret (from the
+  #                                    Crossplane PostgresqlInstance claim)
+  #   - credentials.existingSecret   = SESSION_SECRET + CRYPTO_SECRET
+  #                                    (rotated via OpenBao)
+  #   - catalystIntegration.externalSecret.remoteRef.key
+  #                                  = sovereign/${SOVEREIGN_FQDN}/newapi/admin-token
+  #   - defaultChannels.vllm.enabled = true (first-otech)
+  #   - defaultChannels.vllm.endpoint
+  #     + defaultChannels.vllm.attestation.owner
+  #
+  # Defaults below wire the first-otech provider channel to the same
+  # upstream the OpenOva marketing site uses (Qwen via Axon →
+  # `llm-api.omtd.bankdhofar.com`, model `qwen3-coder`); the operator
+  # overlay overrides any of these by setting them in this HelmRelease's
+  # spec.values.
+  values:
+    sovereignFQDN: ${SOVEREIGN_FQDN}
+    ingress:
+      host: api.${SOVEREIGN_FQDN}
+      adminHost: admin.${SOVEREIGN_FQDN}
+      tls:
+        enabled: true
+        issuer: letsencrypt-prod
+    auth:
+      adminUI:
+        mode: keycloak
+        keycloak:
+          issuer: https://auth.${SOVEREIGN_FQDN}/realms/ops
+          clientId: newapi-admin
+          existingSecret: newapi-oidc
+      customerAPI:
+        keyIssuer: catalyst
+    catalystIntegration:
+      enabled: true
+      existingSecret: catalyst-newapi-admin-token
+      externalSecret:
+        enabled: true
+        refreshInterval: "1h"
+        secretStoreRef:
+          kind: ClusterSecretStore
+          name: vault-region1
+        remoteRef:
+          # Canonical OpenBao path per docs/INVIOLABLE-PRINCIPLES.md #4.
+          # Under the `vault-region1` store's `secret/` mount the full
+          # path is `secret/sovereign/<fqdn>/newapi/admin-token`.
+          key: sovereign/${SOVEREIGN_FQDN}/newapi/admin-token
+          property: ADMIN_API_TOKEN
+    # Default channels — chart-side composition (channel #1 first).
+    #
+    # `qwenBankDhofar` (issue #915) is the canonical first channel:
+    # Qwen3.6 hosted at BankDhofar (https://llm-api.omtd.bankdhofar.com,
+    # model `qwen3-coder` / alias `qwen3.6`) — the SAME relay the
+    # OpenOva marketing site's Axon helmrelease consumes
+    # (openova-private/clusters/contabo-mkt/apps/axon/helmrelease.yaml).
+    # Disabled in the template so a fresh Sovereign does not silently
+    # wire customers to a third-party endpoint; per-Sovereign overlays
+    # (clusters/<sovereign>/bootstrap-kit/80-newapi.yaml) enable this
+    # block and supply:
+    #   - defaultChannels.qwenBankDhofar.enabled = true
+    #   - defaultChannels.qwenBankDhofar.endpoint = https://llm-api.omtd.bankdhofar.com
+    #   - defaultChannels.qwenBankDhofar.attestation.accountId   (legal-team-owned)
+    #   - defaultChannels.qwenBankDhofar.attestation.contractRef (legal-team-owned)
+    #   - the Secret `newapi-channel-qwen-bankdhofar` containing the
+    #     upstream API key under key `API_KEY` (or an ExternalSecret
+    #     pulling from OpenBao at
+    #     `sovereign/<sovereign-fqdn>/newapi/channel-qwen-bankdhofar`)
+    #   - auth.adminUI.masterKeySecret = name of a Secret carrying
+    #     `MASTER_KEY` (NewAPI bootstrap admin auth) — required for
+    #     the channel-seed Helm hook Job to POST against the admin API
+    #     ONCE at install time. Operator may rotate the master key out
+    #     post-bootstrap; channels persist in Postgres.
+    #
+    # When the operator flips `qwenBankDhofar.enabled: true`, the
+    # chart's post-install/post-upgrade `channel-seed` Job probes
+    # NewAPI's admin API (`/api/channel/?keyword=<name>`) and POSTs
+    # the channel definition idempotently. Re-runs after upgrades
+    # are no-ops once the channel exists.
+    #
+    # The legacy `vllm` slot (in-cluster vLLM fallback) remains for
+    # operators that run their own bp-vllm + open-weight model in-
+    # cluster; it composes after `qwenBankDhofar` and any operator
+    # `.Values.channels`.
+    defaultChannels:
+      qwenBankDhofar:
+        enabled: false
+        name: qwen3.6-bankdhofar
+        endpoint: ""
+        models:
+          - qwen3.6
+          - qwen3-coder
+        existingSecret: newapi-channel-qwen-bankdhofar
+        existingSecretKey: API_KEY
+        attestation:
+          kind: commercial-contract
+          accountId: ""
+          contractRef: ""
+      vllm:
+        enabled: false
+        name: qwen
+        endpoint: ""
+        models:
+          - qwen3-coder
+        attestation:
+          kind: in-cluster
+          owner: ${SOVEREIGN_FQDN}

clusters/_template/bootstrap-kit/kustomization.yaml

@@ -6,11 +6,12 @@ kind: Kustomization
 # Phase 0 sequence per SOVEREIGN-PROVISIONING.md §3.
 resources:
   - 01-cilium.yaml
+  - 01a-gateway-api.yaml
   - 02-cert-manager.yaml
   - 03-flux.yaml
   - 04-crossplane.yaml
   - 05-sealed-secrets.yaml
-  - 06-spire.yaml
+  - 05a-reflector.yaml
   - 07-nats-jetstream.yaml
   - 08-openbao.yaml
   - 09-keycloak.yaml
@@ -20,17 +21,23 @@ resources:
   - 13-bp-catalyst-platform.yaml
   - 14-crossplane-claims.yaml
   - 15-external-secrets.yaml
+  - 15a-external-secrets-stores.yaml
   - 16-cnpg.yaml
   - 17-valkey.yaml
   - 18-seaweedfs.yaml
   - 19-harbor.yaml
+  # 06a — Post-handover Self-Sovereignty Cutover (issue #791). Filename
+  # carries the 06a prefix to colocate cohorts visually, but the slot's
+  # dependsOn pins actual install order to AFTER bp-gitea (slot 10) and
+  # bp-harbor (slot 19). Chart installs DORMANT — catalyst-api stamps
+  # Jobs only on operator-driven cutover trigger.
+  - 06a-bp-self-sovereign-cutover.yaml
   - 20-opentelemetry.yaml
   - 21-alloy.yaml
   - 22-loki.yaml
   - 23-mimir.yaml
   - 24-tempo.yaml
   - 25-grafana.yaml
-  - 26-langfuse.yaml
   - 27-kyverno.yaml
   - 28-reloader.yaml
   - 29-vpa.yaml
@@ -40,3 +47,22 @@ resources:
   - 33-syft-grype.yaml
   - 34-velero.yaml
   - 35-coraza.yaml
+  - 49-bp-cert-manager-powerdns-webhook.yaml
+  - 50-cluster-autoscaler.yaml
+  # bp-newapi (slot 80) — multi-tenant LLM marketplace gateway. Sequenced
+  # after the W2.K1 dependency wave (cnpg/keycloak/openbao Ready) so
+  # NewAPI's ExternalSecret + DSN dependencies resolve on first reconcile.
+  # See clusters/_template/bootstrap-kit/80-newapi.yaml for full
+  # dependsOn rationale and per-Sovereign override surface.
+  - 80-newapi.yaml
+  # bp-stalwart-sovereign (slot 95) — REMOVED 2026-05-05.
+  # Phase-2 Sovereign-local mail (per-Sovereign Stalwart for Console
+  # PIN/magic-link delivery, umbrella #924) is OUT OF SCOPE for the
+  # current Phase-1 cutover. The Phase-1 design is mothership SMTP
+  # relay (mail.openova.io:587) — see products/catalyst/chart/values.yaml
+  # `sovereign.smtp.*` and the catalyst-api `sovereign_smtp_seed.go`
+  # path. The chart's post-install Job was timing out on otech113 and
+  # blocking the bootstrap-kit Kustomization. Re-introduce this slot
+  # only when Phase-2 is explicitly in scope and the chart's readiness
+  # gate is reliable. See platform/stalwart-sovereign/ for the chart
+  # itself (kept in-tree for future Phase-2 work).

clusters/_template/sovereign-tls/cilium-gateway-cert.yaml

@@ -0,0 +1,68 @@
+# Wildcard TLS Certificate for the Cilium Gateway listener.
+#
+# Split from clusters/_template/bootstrap-kit/01-cilium.yaml in
+# fix/cilium-cert-split-from-bootstrap-kit (Phase-8a bug #13). The
+# Cert lives in its OWN Flux Kustomization (`sovereign-tls`) which
+# depends on bootstrap-kit being Ready — i.e. cert-manager + the
+# powerdns-webhook are both installed and their CRDs registered.
+#
+# Without this split, Flux's server-side dry-run on the bootstrap-kit
+# Kustomization fails with `no matches for kind "Certificate" in
+# version "cert-manager.io/v1"` because the validation runs BEFORE any
+# HelmRelease has installed the cert-manager CRDs — and a single
+# dry-run failure aborts the entire Kustomization apply, leaving the
+# Sovereign with zero HRs reconciled.
+#
+# The Gateway resource stays in 01-cilium.yaml: Gateway.networking.k8s.io
+# CRDs ship with Cilium itself (gatewayAPI.enabled=true) and dry-run
+# against them only requires the Gateway API CRD bundle which Cilium
+# pre-installs at chart-time. The Certificate is the ONLY resource
+# whose CRD is provided by a HelmRelease in the same Kustomization
+# that needs to validate it.
+#
+# Issuer: `letsencrypt-dns01-prod-powerdns` is shipped by
+# bp-cert-manager-powerdns-webhook (bootstrap-kit slot 49). It writes
+# the ACME challenge TXT record to contabo's central PowerDNS at
+# https://pdns.openova.io (authoritative for omani.works) so Let's
+# Encrypt validation succeeds even before the Sovereign's own NS
+# delegation has propagated. Replaces the previous letsencrypt-dns01-prod
+# (dynadot-webhook-backed) — Dynadot is not the API-level authority for
+# omani.works subdomains. Caught live on otech43–46.
+#
+# ──────────────────────────────────────────────────────────────────────────
+# Multi-zone Sovereign (issue #827, parent epic #825) coexistence note
+# ──────────────────────────────────────────────────────────────────────────
+# bp-catalyst-platform 1.4.0+ ships templates/sovereign-wildcard-certs.yaml
+# which renders one Certificate PER ENTRY in `.Values.parentZones`, each
+# named `sovereign-wildcard-tls-<sanitised-zone>` (e.g.
+# `sovereign-wildcard-tls-omani-trade`). Those resource names are DISTINCT
+# from this file's `sovereign-wildcard-tls` so the two paths never collide:
+#   - Single-zone Sovereigns (parentZones empty) — this file owns the only
+#     wildcard cert.
+#   - Multi-zone Sovereigns (parentZones populated) — this file STILL owns
+#     `sovereign-wildcard-tls` (covering the operator's primary parent
+#     zone) AND the chart adds N additional zone-specific certs. The
+#     Cilium Gateway listener is updated in the per-cluster overlay to
+#     reference the appropriate Secret per zone listener.
+#
+# Once issue #831 lands a multi-listener Gateway template in
+# bp-catalyst-platform itself, this file becomes redundant and is
+# deletable.
+
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: sovereign-wildcard-tls
+  namespace: kube-system
+  labels:
+    catalyst.openova.io/sovereign: ${SOVEREIGN_FQDN}
+    catalyst.openova.io/component: cilium-gateway
+spec:
+  secretName: sovereign-wildcard-tls
+  issuerRef:
+    name: letsencrypt-dns01-prod-powerdns
+    kind: ClusterIssuer
+  commonName: "*.${SOVEREIGN_FQDN}"
+  dnsNames:
+    - "*.${SOVEREIGN_FQDN}"
+    - "${SOVEREIGN_FQDN}"

clusters/_template/sovereign-tls/cilium-gateway.yaml

@@ -0,0 +1,54 @@
+# Cilium Gateway (Phase-8a bug #14 follow-up to #484).
+# Moved out of bootstrap-kit/01-cilium.yaml because gateway.networking.k8s.io/v1
+# CRDs are installed by the Cilium HelmRelease itself; Flux dry-runs the
+# whole Kustomization before applying any HR, so Gateway dry-run fails on
+# a fresh cluster. The sovereign-tls Kustomization dependsOn bootstrap-kit
+# Ready, so by the time Gateway is applied here, Cilium has installed.
+
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: cilium-gateway
+  namespace: kube-system
+  labels:
+    catalyst.openova.io/sovereign: ${SOVEREIGN_FQDN}
+    catalyst.openova.io/component: cilium-gateway
+spec:
+  gatewayClassName: cilium
+  # NOTE: ports 30080/30443 (not 80/443) — even with hostNetwork=true,
+  # cilium-envoy refuses to bind privileged ports because cilium-agent
+  # gates that bind through its `envoy-keep-cap-netbindservice` flag and
+  # the resulting bind() syscall is intercepted by the agent's BPF
+  # socket-LB program. Setting privileged: true on the cilium-envoy
+  # DaemonSet + adding NET_BIND_SERVICE + flipping the configmap flag
+  # all failed to lift the bind() rejection (verified live on otech45,
+  # otech46, otech47).
+  #
+  # High-port (>1024) bind succeeds without NET_BIND_SERVICE. The
+  # Hetzner LB does the public-facing port translation: HCLB listens on
+  # 80→forwards to CP node:30080; HCLB listens on 443→forwards to CP
+  # node:30443. Browsers hit the canonical URL (`https://console.<fqdn>/`)
+  # so port 30443 is never visible externally.
+  #
+  # See infra/hetzner/main.tf hcloud_load_balancer_service.{http,https}
+  # destination_port settings — they MUST match these listener ports.
+  listeners:
+    - name: https
+      port: 30443
+      protocol: HTTPS
+      hostname: "*.${SOVEREIGN_FQDN}"
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - kind: Secret
+            name: sovereign-wildcard-tls
+      allowedRoutes:
+        namespaces:
+          from: All
+    - name: http
+      port: 30080
+      protocol: HTTP
+      hostname: "*.${SOVEREIGN_FQDN}"
+      allowedRoutes:
+        namespaces:
+          from: All

clusters/_template/sovereign-tls/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - cilium-gateway-cert.yaml
+  - cilium-gateway.yaml

clusters/contabo-mkt/tenants/bakkal/apps/app-stalwart-mail.yaml

@@ -1,46 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: stalwart-mail
-  namespace: apps
-  labels:
-    app: stalwart-mail
-    openova.io/tenant: "bakkal"
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: stalwart-mail
-  template:
-    metadata:
-      labels:
-        app: stalwart-mail
-        openova.io/tenant: "bakkal"
-    spec:
-      containers:
-        - name: stalwart-mail
-          image: 
-          ports:
-            - containerPort: 0
-          env:
-          resources:
-            requests:
-              cpu: 
-              memory: 
-            limits:
-              cpu: 500m
-              memory: 512Mi
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: stalwart-mail
-  namespace: apps
-spec:
-  selector:
-    app: stalwart-mail
-  ports:
-    - port: 80
-      targetPort: 0

clusters/contabo-mkt/tenants/bakkal/ingress.yaml

@@ -1,59 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: tenant-ingress
-  namespace: tenant-bakkal
-  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-prod
-spec:
-  ingressClassName: traefik
-  rules:
-    - host: bakkal.omani.rest
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: nextcloud-x-tenant-bakkal-x-vcluster
-                port:
-                  number: 80
-          - path: /nextcloud
-            pathType: Prefix
-            backend:
-              service:
-                name: nextcloud-x-tenant-bakkal-x-vcluster
-                port:
-                  number: 80
-          - path: /bookstack
-            pathType: Prefix
-            backend:
-              service:
-                name: bookstack-x-tenant-bakkal-x-vcluster
-                port:
-                  number: 80
-          - path: /vaultwarden
-            pathType: Prefix
-            backend:
-              service:
-                name: vaultwarden-x-tenant-bakkal-x-vcluster
-                port:
-                  number: 80
-          - path: /cal-com
-            pathType: Prefix
-            backend:
-              service:
-                name: cal-com-x-tenant-bakkal-x-vcluster
-                port:
-                  number: 80
-          - path: /stalwart-mail
-            pathType: Prefix
-            backend:
-              service:
-                name: stalwart-mail-x-tenant-bakkal-x-vcluster
-                port:
-                  number: 80
-  tls:
-    - hosts:
-        - bakkal.omani.rest
-      secretName: tenant-bakkal-tls

clusters/contabo-mkt/tenants/bbb/apps-sync.yaml

@@ -1,7 +1,7 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: tenant-bakkal-apps
+  name: tenant-bbb-apps
   namespace: flux-system
 spec:
   interval: 5m
@@ -9,13 +9,13 @@ spec:
   timeout: 5m
   prune: true
   wait: true
-  targetNamespace: tenant-bakkal
+  targetNamespace: tenant-bbb
   sourceRef:
     kind: GitRepository
     name: flux-system
     namespace: flux-system
-  path: ./clusters/contabo-mkt/tenants/bakkal/apps
+  path: ./clusters/contabo-mkt/tenants/bbb/apps
   kubeConfig:
     secretRef:
-      name: tenant-bakkal-kubeconfig
+      name: tenant-bbb-kubeconfig
       key: config

clusters/contabo-mkt/tenants/bbb/apps/app-bookstack.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: apps
   labels:
     app: bookstack
-    openova.io/tenant: "bakkal"
+    openova.io/tenant: "bbb"
 spec:
   replicas: 1
   strategy:
@@ -17,7 +17,7 @@ spec:
     metadata:
       labels:
         app: bookstack
-        openova.io/tenant: "bakkal"
+        openova.io/tenant: "bbb"
     spec:
       containers:
         - name: bookstack
@@ -30,7 +30,7 @@ spec:
             - name: WORDPRESS_DB_USER
               value: "app"
             - name: WORDPRESS_DB_PASSWORD
-              value: "1b556de942f5df2a1458fdb8b19dec0b"
+              value: "bbaa187122d88da6b0e38b8de814c133"
             - name: WORDPRESS_DB_NAME
               value: "db_bookstack"
             - name: MYSQL_HOST
@@ -38,7 +38,7 @@ spec:
             - name: MYSQL_USER
               value: "app"
             - name: MYSQL_PASSWORD
-              value: "1b556de942f5df2a1458fdb8b19dec0b"
+              value: "bbaa187122d88da6b0e38b8de814c133"
             - name: MYSQL_DATABASE
               value: "db_bookstack"
           resources:

clusters/contabo-mkt/tenants/bbb/apps/app-cal-com.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: apps
   labels:
     app: cal-com
-    openova.io/tenant: "bakkal"
+    openova.io/tenant: "bbb"
 spec:
   replicas: 1
   strategy:
@@ -17,7 +17,7 @@ spec:
     metadata:
       labels:
         app: cal-com
-        openova.io/tenant: "bakkal"
+        openova.io/tenant: "bbb"
     spec:
       containers:
         - name: cal-com
@@ -26,11 +26,11 @@ spec:
             - containerPort: 3000
           env:
             - name: NEXTAUTH_URL
-              value: "https://bakkal.omani.rest/calcom"
+              value: "https://bbb.omani.rest/calcom"
             - name: NEXT_PUBLIC_WEBAPP_URL
-              value: "https://bakkal.omani.rest/calcom"
+              value: "https://bbb.omani.rest/calcom"
             - name: DATABASE_URL
-              value: "postgresql://app:1b556de942f5df2a1458fdb8b19dec0b@postgres:5432/db_cal-com"
+              value: "postgresql://app:bbaa187122d88da6b0e38b8de814c133@postgres:5432/db_cal-com"
             - name: POSTGRES_HOST
               value: "postgres"
             - name: POSTGRES_PORT
@@ -40,7 +40,7 @@ spec:
             - name: POSTGRES_USERNAME
               value: "app"
             - name: POSTGRES_PASSWORD
-              value: "1b556de942f5df2a1458fdb8b19dec0b"
+              value: "bbaa187122d88da6b0e38b8de814c133"
           resources:
             requests:
               cpu: 100m

clusters/contabo-mkt/tenants/bbb/apps/app-gitea.yaml

@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gitea
+  namespace: apps
+  labels:
+    app: gitea
+    openova.io/tenant: "bbb"
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: gitea
+  template:
+    metadata:
+      labels:
+        app: gitea
+        openova.io/tenant: "bbb"
+    spec:
+      containers:
+        - name: gitea
+          image: gitea/gitea:1-rootless
+          ports:
+            - containerPort: 3000
+          env:
+            - name: DATABASE_URL
+              value: "postgresql://app:bbaa187122d88da6b0e38b8de814c133@postgres:5432/db_gitea"
+            - name: POSTGRES_HOST
+              value: "postgres"
+            - name: POSTGRES_PORT
+              value: "5432"
+            - name: POSTGRES_DATABASE
+              value: "db_gitea"
+            - name: POSTGRES_USERNAME
+              value: "app"
+            - name: POSTGRES_PASSWORD
+              value: "bbaa187122d88da6b0e38b8de814c133"
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: gitea
+  namespace: apps
+spec:
+  selector:
+    app: gitea
+  ports:
+    - port: 80
+      targetPort: 3000

clusters/contabo-mkt/tenants/bbb/apps/db-mysql.yaml

@@ -5,9 +5,9 @@ metadata:
   namespace: apps
 type: Opaque
 stringData:
-  MYSQL_ROOT_PASSWORD: "1b556de942f5df2a1458fdb8b19dec0b"
+  MYSQL_ROOT_PASSWORD: "bbaa187122d88da6b0e38b8de814c133"
   MYSQL_USER: app
-  MYSQL_PASSWORD: "1b556de942f5df2a1458fdb8b19dec0b"
+  MYSQL_PASSWORD: "bbaa187122d88da6b0e38b8de814c133"
   MYSQL_DATABASE: db_bookstack
 ---
 apiVersion: v1

clusters/contabo-mkt/tenants/bbb/apps/db-postgres.yaml

@@ -6,7 +6,7 @@ metadata:
 type: Opaque
 stringData:
   POSTGRES_USER: app
-  POSTGRES_PASSWORD: "1b556de942f5df2a1458fdb8b19dec0b"
+  POSTGRES_PASSWORD: "bbaa187122d88da6b0e38b8de814c133"
   POSTGRES_DB: db_cal-com
 ---
 apiVersion: v1
@@ -17,8 +17,8 @@ metadata:
 data:
   init.sql: |
     -- per-app database bootstrap (postgres)
-    CREATE DATABASE db_nextcloud;
+    CREATE DATABASE db_gitea;
-    GRANT ALL PRIVILEGES ON DATABASE db_nextcloud TO app;
+    GRANT ALL PRIVILEGES ON DATABASE db_gitea TO app;
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim

clusters/contabo-mkt/tenants/bbb/apps/kustomization.yaml

@@ -4,9 +4,7 @@ namespace: apps
 resources:
   - app-bookstack.yaml
   - app-cal-com.yaml
-  - app-nextcloud.yaml
+  - app-gitea.yaml
-  - app-stalwart-mail.yaml
-  - app-vaultwarden.yaml
   - db-mysql.yaml
   - db-postgres.yaml
   - namespace.yaml

clusters/contabo-mkt/tenants/bbb/ingress.yaml

@@ -0,0 +1,45 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: tenant-ingress
+  namespace: tenant-bbb
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: bbb.omani.rest
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: bookstack-x-tenant-bbb-x-vcluster
+                port:
+                  number: 80
+          - path: /bookstack
+            pathType: Prefix
+            backend:
+              service:
+                name: bookstack-x-tenant-bbb-x-vcluster
+                port:
+                  number: 80
+          - path: /cal-com
+            pathType: Prefix
+            backend:
+              service:
+                name: cal-com-x-tenant-bbb-x-vcluster
+                port:
+                  number: 80
+          - path: /gitea
+            pathType: Prefix
+            backend:
+              service:
+                name: gitea-x-tenant-bbb-x-vcluster
+                port:
+                  number: 80
+  tls:
+    - hosts:
+        - bbb.omani.rest
+      secretName: tenant-bbb-tls

clusters/contabo-mkt/tenants/bbb/namespace.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: tenant-bakkal
+  name: tenant-bbb
   labels:
-    openova.io/tenant: "bakkal"
+    openova.io/tenant: "bbb"
     openova.io/managed-by: provisioning

clusters/contabo-mkt/tenants/bbb/provisioning-rbac.yaml

@@ -2,7 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   name: provisioning-tenant
-  namespace: tenant-bakkal
+  namespace: tenant-bbb
   labels:
     openova.io/managed-by: provisioning
 rules:
@@ -45,7 +45,7 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: provisioning-tenant
-  namespace: tenant-bakkal
+  namespace: tenant-bbb
   labels:
     openova.io/managed-by: provisioning
 roleRef:

clusters/contabo-mkt/tenants/bbb/vcluster.yaml

@@ -2,7 +2,7 @@ apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
   name: vcluster
-  namespace: tenant-bakkal
+  namespace: tenant-bbb
 spec:
   interval: 10m
   chart:
@@ -42,11 +42,11 @@ spec:
           type: ClusterIP
     exportKubeConfig:
       context: vcluster
-      server: https://vcluster.tenant-bakkal:443
+      server: https://vcluster.tenant-bbb:443
       insecure: false
       additionalSecrets:
         - name: vc-vcluster
-          server: https://vcluster.tenant-bakkal:443
+          server: https://vcluster.tenant-bbb:443
           insecure: false
           context: vcluster
     sync:

clusters/contabo-mkt/tenants/e2e-wp-test/apps-sync.yaml

@@ -0,0 +1,21 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: tenant-e2e-wp-test-apps
+  namespace: flux-system
+spec:
+  interval: 5m
+  retryInterval: 1m
+  timeout: 5m
+  prune: true
+  wait: true
+  targetNamespace: tenant-e2e-wp-test
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  path: ./clusters/contabo-mkt/tenants/e2e-wp-test/apps
+  kubeConfig:
+    secretRef:
+      name: tenant-e2e-wp-test-kubeconfig
+      key: config

clusters/contabo-mkt/tenants/e2e-wp-test/apps/app-wordpress.yaml

@@ -0,0 +1,62 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: wordpress
+  namespace: apps
+  labels:
+    app: wordpress
+    openova.io/tenant: "e2e-wp-test"
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: wordpress
+  template:
+    metadata:
+      labels:
+        app: wordpress
+        openova.io/tenant: "e2e-wp-test"
+    spec:
+      containers:
+        - name: wordpress
+          image: wordpress:6-apache
+          ports:
+            - containerPort: 80
+          env:
+            - name: WORDPRESS_DB_HOST
+              value: "mysql"
+            - name: WORDPRESS_DB_USER
+              value: "app"
+            - name: WORDPRESS_DB_PASSWORD
+              value: "0c6cd48ebb3991570bd15d9223d06a89"
+            - name: WORDPRESS_DB_NAME
+              value: "db_wordpress"
+            - name: MYSQL_HOST
+              value: "mysql"
+            - name: MYSQL_USER
+              value: "app"
+            - name: MYSQL_PASSWORD
+              value: "0c6cd48ebb3991570bd15d9223d06a89"
+            - name: MYSQL_DATABASE
+              value: "db_wordpress"
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: wordpress
+  namespace: apps
+spec:
+  selector:
+    app: wordpress
+  ports:
+    - port: 80
+      targetPort: 80

clusters/contabo-mkt/tenants/e2e-wp-test/apps/db-mysql.yaml

@@ -0,0 +1,88 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: mysql-credentials
+  namespace: apps
+type: Opaque
+stringData:
+  MYSQL_ROOT_PASSWORD: "0c6cd48ebb3991570bd15d9223d06a89"
+  MYSQL_USER: app
+  MYSQL_PASSWORD: "0c6cd48ebb3991570bd15d9223d06a89"
+  MYSQL_DATABASE: db_wordpress
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mysql-initdb
+  namespace: apps
+data:
+  init.sql: |
+    FLUSH PRIVILEGES;
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: mysql-data
+  namespace: apps
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: 2Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mysql
+  namespace: apps
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: mysql
+  template:
+    metadata:
+      labels:
+        app: mysql
+    spec:
+      containers:
+        - name: mysql
+          image: mariadb:11
+          ports:
+            - containerPort: 3306
+          envFrom:
+            - secretRef:
+                name: mysql-credentials
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 256Mi
+          volumeMounts:
+            - name: mysqldata
+              mountPath: /var/lib/mysql
+            - name: initdb
+              mountPath: /docker-entrypoint-initdb.d
+      volumes:
+        - name: mysqldata
+          persistentVolumeClaim:
+            claimName: mysql-data
+        - name: initdb
+          configMap:
+            name: mysql-initdb
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql
+  namespace: apps
+spec:
+  selector:
+    app: mysql
+  ports:
+    - port: 3306
+      targetPort: 3306

clusters/contabo-mkt/tenants/e2e-wp-test/apps/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: apps
+resources:
+  - app-wordpress.yaml
+  - db-mysql.yaml
+  - namespace.yaml

clusters/contabo-mkt/tenants/e2e-wp-test/apps/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: apps

clusters/contabo-mkt/tenants/e2e-wp-test/ingress.yaml

@@ -0,0 +1,31 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: tenant-ingress
+  namespace: tenant-e2e-wp-test
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: e2e-wp-test.omani.rest
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: wordpress-x-tenant-e2e-wp-test-x-vcluster
+                port:
+                  number: 80
+          - path: /wordpress
+            pathType: Prefix
+            backend:
+              service:
+                name: wordpress-x-tenant-e2e-wp-test-x-vcluster
+                port:
+                  number: 80
+  tls:
+    - hosts:
+        - e2e-wp-test.omani.rest
+      secretName: tenant-e2e-wp-test-tls

clusters/contabo-mkt/tenants/e2e-wp-test/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - apps-sync.yaml
+  - ingress.yaml
+  - namespace.yaml
+  - provisioning-rbac.yaml
+  - vcluster.yaml

clusters/contabo-mkt/tenants/e2e-wp-test/namespace.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: tenant-e2e-wp-test
+  labels:
+    openova.io/tenant: "e2e-wp-test"
+    openova.io/managed-by: provisioning

clusters/contabo-mkt/tenants/e2e-wp-test/provisioning-rbac.yaml

@@ -0,0 +1,58 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: provisioning-tenant
+  namespace: tenant-e2e-wp-test
+  labels:
+    openova.io/managed-by: provisioning
+rules:
+  - apiGroups: ["helm.toolkit.fluxcd.io"]
+    resources: ["helmreleases"]
+    verbs: ["get", "list", "watch", "patch", "delete"]
+  - apiGroups: ["kustomize.toolkit.fluxcd.io"]
+    resources: ["kustomizations"]
+    verbs: ["get", "list", "watch", "patch", "delete"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    # delete needed so waitForVclusterDNSOrKick can bounce vcluster-0 when
+    # the syncer's initial DNS reconciliation doesn't publish the
+    # kube-dns-x-kube-system-x-vcluster service. Issues #103, #105.
+    resources: ["pods"]
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: [""]
+    # services verb needed for waitForVclusterDNSOrKick to read the synced
+    # kube-dns-x-kube-system-x-vcluster Service to know DNS is live.
+    # Without this, the DNS probe returns 403 → we think DNS isn't synced
+    # → we kick vcluster-0 unnecessarily → 150s wasted per tenant.
+    # Also used by pod-truth reconciler to verify tenant apps are healthy
+    # regardless of provision-record freshness. Issue #115.
+    resources: ["services"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates", "certificaterequests"]
+    # patch needed so stripCertificateFinalizers can drop
+    # finalizer.cert-manager.io/certificate-secret-binding at teardown;
+    # without it the tenant NS can't GC because cert-manager can't
+    # reconcile the delete inside a Terminating NS. Issue #86.
+    verbs: ["get", "list", "watch", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: provisioning-tenant
+  namespace: tenant-e2e-wp-test
+  labels:
+    openova.io/managed-by: provisioning
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: provisioning-tenant
+subjects:
+  - kind: ServiceAccount
+    name: provisioning
+    namespace: sme

clusters/contabo-mkt/tenants/e2e-wp-test/vcluster.yaml

@@ -0,0 +1,60 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: vcluster
+  namespace: tenant-e2e-wp-test
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: vcluster
+      version: "0.33.*"
+      sourceRef:
+        kind: HelmRepository
+        name: loft
+        namespace: vcluster-system
+  values:
+    controlPlane:
+      distro:
+        k8s:
+          enabled: true
+      backingStore:
+        database:
+          embedded:
+            enabled: true
+      statefulSet:
+        image:
+          registry: ghcr.io
+          repository: loft-sh/vcluster-oss
+        resources:
+          requests:
+            cpu: 100m
+            memory: 192Mi
+          limits:
+            cpu: 2000m
+            memory: 2Gi
+        persistence:
+          volumeClaim:
+            size: 5Gi
+      service:
+        enabled: true
+        spec:
+          type: ClusterIP
+    exportKubeConfig:
+      context: vcluster
+      server: https://vcluster.tenant-e2e-wp-test:443
+      insecure: false
+      additionalSecrets:
+        - name: vc-vcluster
+          server: https://vcluster.tenant-e2e-wp-test:443
+          insecure: false
+          context: vcluster
+    sync:
+      toHost:
+        services:
+          enabled: true
+        ingresses:
+          enabled: false
+      fromHost:
+        ingressClasses:
+          enabled: true

clusters/contabo-mkt/tenants/kustomization.yaml

@@ -1,4 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - bakkal
+  - bbb
+  - test12-2
+  - e2e-wp-test

clusters/contabo-mkt/tenants/test/apps-sync.yaml

@@ -0,0 +1,21 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: tenant-test-apps
+  namespace: flux-system
+spec:
+  interval: 5m
+  retryInterval: 1m
+  timeout: 5m
+  prune: true
+  wait: true
+  targetNamespace: tenant-test
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  path: ./clusters/contabo-mkt/tenants/test/apps
+  kubeConfig:
+    secretRef:
+      name: tenant-test-kubeconfig
+      key: config

clusters/contabo-mkt/tenants/test/apps/app-bookstack.yaml

@@ -0,0 +1,62 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: bookstack
+  namespace: apps
+  labels:
+    app: bookstack
+    openova.io/tenant: "test"
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: bookstack
+  template:
+    metadata:
+      labels:
+        app: bookstack
+        openova.io/tenant: "test"
+    spec:
+      containers:
+        - name: bookstack
+          image: lscr.io/linuxserver/bookstack:latest
+          ports:
+            - containerPort: 80
+          env:
+            - name: WORDPRESS_DB_HOST
+              value: "mysql"
+            - name: WORDPRESS_DB_USER
+              value: "app"
+            - name: WORDPRESS_DB_PASSWORD
+              value: "a75d5d4bc534619c0ed8f16e0602f492"
+            - name: WORDPRESS_DB_NAME
+              value: "db_bookstack"
+            - name: MYSQL_HOST
+              value: "mysql"
+            - name: MYSQL_USER
+              value: "app"
+            - name: MYSQL_PASSWORD
+              value: "a75d5d4bc534619c0ed8f16e0602f492"
+            - name: MYSQL_DATABASE
+              value: "db_bookstack"
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: bookstack
+  namespace: apps
+spec:
+  selector:
+    app: bookstack
+  ports:
+    - port: 80
+      targetPort: 80

clusters/contabo-mkt/tenants/test/apps/db-mysql.yaml

@@ -0,0 +1,88 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: mysql-credentials
+  namespace: apps
+type: Opaque
+stringData:
+  MYSQL_ROOT_PASSWORD: "a75d5d4bc534619c0ed8f16e0602f492"
+  MYSQL_USER: app
+  MYSQL_PASSWORD: "a75d5d4bc534619c0ed8f16e0602f492"
+  MYSQL_DATABASE: db_bookstack
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mysql-initdb
+  namespace: apps
+data:
+  init.sql: |
+    FLUSH PRIVILEGES;
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: mysql-data
+  namespace: apps
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: 2Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mysql
+  namespace: apps
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: mysql
+  template:
+    metadata:
+      labels:
+        app: mysql
+    spec:
+      containers:
+        - name: mysql
+          image: mariadb:11
+          ports:
+            - containerPort: 3306
+          envFrom:
+            - secretRef:
+                name: mysql-credentials
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 256Mi
+          volumeMounts:
+            - name: mysqldata
+              mountPath: /var/lib/mysql
+            - name: initdb
+              mountPath: /docker-entrypoint-initdb.d
+      volumes:
+        - name: mysqldata
+          persistentVolumeClaim:
+            claimName: mysql-data
+        - name: initdb
+          configMap:
+            name: mysql-initdb
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql
+  namespace: apps
+spec:
+  selector:
+    app: mysql
+  ports:
+    - port: 3306
+      targetPort: 3306

clusters/contabo-mkt/tenants/test/apps/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: apps
+resources:
+  - app-bookstack.yaml
+  - db-mysql.yaml
+  - namespace.yaml

clusters/contabo-mkt/tenants/test/apps/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: apps

clusters/contabo-mkt/tenants/test/ingress.yaml

@@ -0,0 +1,31 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: tenant-ingress
+  namespace: tenant-test
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: test.omani.rest
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: bookstack-x-tenant-test-x-vcluster
+                port:
+                  number: 80
+          - path: /bookstack
+            pathType: Prefix
+            backend:
+              service:
+                name: bookstack-x-tenant-test-x-vcluster
+                port:
+                  number: 80
+  tls:
+    - hosts:
+        - test.omani.rest
+      secretName: tenant-test-tls

clusters/contabo-mkt/tenants/test/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - apps-sync.yaml
+  - ingress.yaml
+  - namespace.yaml
+  - provisioning-rbac.yaml
+  - vcluster.yaml

clusters/contabo-mkt/tenants/test/namespace.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: tenant-test
+  labels:
+    openova.io/tenant: "test"
+    openova.io/managed-by: provisioning

clusters/contabo-mkt/tenants/test/provisioning-rbac.yaml

@@ -0,0 +1,58 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: provisioning-tenant
+  namespace: tenant-test
+  labels:
+    openova.io/managed-by: provisioning
+rules:
+  - apiGroups: ["helm.toolkit.fluxcd.io"]
+    resources: ["helmreleases"]
+    verbs: ["get", "list", "watch", "patch", "delete"]
+  - apiGroups: ["kustomize.toolkit.fluxcd.io"]
+    resources: ["kustomizations"]
+    verbs: ["get", "list", "watch", "patch", "delete"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    # delete needed so waitForVclusterDNSOrKick can bounce vcluster-0 when
+    # the syncer's initial DNS reconciliation doesn't publish the
+    # kube-dns-x-kube-system-x-vcluster service. Issues #103, #105.
+    resources: ["pods"]
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: [""]
+    # services verb needed for waitForVclusterDNSOrKick to read the synced
+    # kube-dns-x-kube-system-x-vcluster Service to know DNS is live.
+    # Without this, the DNS probe returns 403 → we think DNS isn't synced
+    # → we kick vcluster-0 unnecessarily → 150s wasted per tenant.
+    # Also used by pod-truth reconciler to verify tenant apps are healthy
+    # regardless of provision-record freshness. Issue #115.
+    resources: ["services"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates", "certificaterequests"]
+    # patch needed so stripCertificateFinalizers can drop
+    # finalizer.cert-manager.io/certificate-secret-binding at teardown;
+    # without it the tenant NS can't GC because cert-manager can't
+    # reconcile the delete inside a Terminating NS. Issue #86.
+    verbs: ["get", "list", "watch", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: provisioning-tenant
+  namespace: tenant-test
+  labels:
+    openova.io/managed-by: provisioning
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: provisioning-tenant
+subjects:
+  - kind: ServiceAccount
+    name: provisioning
+    namespace: sme

clusters/contabo-mkt/tenants/test/vcluster.yaml

@@ -0,0 +1,60 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: vcluster
+  namespace: tenant-test
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: vcluster
+      version: "0.33.*"
+      sourceRef:
+        kind: HelmRepository
+        name: loft
+        namespace: vcluster-system
+  values:
+    controlPlane:
+      distro:
+        k8s:
+          enabled: true
+      backingStore:
+        database:
+          embedded:
+            enabled: true
+      statefulSet:
+        image:
+          registry: ghcr.io
+          repository: loft-sh/vcluster-oss
+        resources:
+          requests:
+            cpu: 100m
+            memory: 192Mi
+          limits:
+            cpu: 2000m
+            memory: 2Gi
+        persistence:
+          volumeClaim:
+            size: 5Gi
+      service:
+        enabled: true
+        spec:
+          type: ClusterIP
+    exportKubeConfig:
+      context: vcluster
+      server: https://vcluster.tenant-test:443
+      insecure: false
+      additionalSecrets:
+        - name: vc-vcluster
+          server: https://vcluster.tenant-test:443
+          insecure: false
+          context: vcluster
+    sync:
+      toHost:
+        services:
+          enabled: true
+        ingresses:
+          enabled: false
+      fromHost:
+        ingressClasses:
+          enabled: true

clusters/contabo-mkt/tenants/test12-2/apps-sync.yaml

@@ -0,0 +1,21 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: tenant-test12-2-apps
+  namespace: flux-system
+spec:
+  interval: 5m
+  retryInterval: 1m
+  timeout: 5m
+  prune: true
+  wait: true
+  targetNamespace: tenant-test12-2
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  path: ./clusters/contabo-mkt/tenants/test12-2/apps
+  kubeConfig:
+    secretRef:
+      name: tenant-test12-2-kubeconfig
+      key: config

clusters/contabo-mkt/tenants/test12-2/apps/app-nextcloud.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: apps
   labels:
     app: nextcloud
-    openova.io/tenant: "bakkal"
+    openova.io/tenant: "test12-2"
 spec:
   replicas: 1
   strategy:
@@ -17,7 +17,7 @@ spec:
     metadata:
       labels:
         app: nextcloud
-        openova.io/tenant: "bakkal"
+        openova.io/tenant: "test12-2"
     spec:
       containers:
         - name: nextcloud
@@ -26,7 +26,7 @@ spec:
             - containerPort: 80
           env:
             - name: DATABASE_URL
-              value: "postgresql://app:1b556de942f5df2a1458fdb8b19dec0b@postgres:5432/db_nextcloud"
+              value: "postgresql://app:e16cde7aeb535edc96b435d7a1523cd5@postgres:5432/db_nextcloud"
             - name: POSTGRES_HOST
               value: "postgres"
             - name: POSTGRES_PORT
@@ -36,7 +36,7 @@ spec:
             - name: POSTGRES_USERNAME
               value: "app"
             - name: POSTGRES_PASSWORD
-              value: "1b556de942f5df2a1458fdb8b19dec0b"
+              value: "e16cde7aeb535edc96b435d7a1523cd5"
           resources:
             requests:
               cpu: 100m

clusters/contabo-mkt/tenants/test12-2/apps/app-vaultwarden.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: apps
   labels:
     app: vaultwarden
-    openova.io/tenant: "bakkal"
+    openova.io/tenant: "test12-2"
 spec:
   replicas: 1
   strategy:
@@ -17,7 +17,7 @@ spec:
     metadata:
       labels:
         app: vaultwarden
-        openova.io/tenant: "bakkal"
+        openova.io/tenant: "test12-2"
     spec:
       containers:
         - name: vaultwarden

clusters/contabo-mkt/tenants/test12-2/apps/db-postgres.yaml

@@ -0,0 +1,87 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgres-credentials
+  namespace: apps
+type: Opaque
+stringData:
+  POSTGRES_USER: app
+  POSTGRES_PASSWORD: "e16cde7aeb535edc96b435d7a1523cd5"
+  POSTGRES_DB: db_nextcloud
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: postgres-initdb
+  namespace: apps
+data:
+  init.sql: |
+    -- per-app database bootstrap (postgres)
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: postgres-data
+  namespace: apps
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: 2Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postgres
+  namespace: apps
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: postgres
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      containers:
+        - name: postgres
+          image: postgres:16-alpine
+          ports:
+            - containerPort: 5432
+          envFrom:
+            - secretRef:
+                name: postgres-credentials
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 256Mi
+          volumeMounts:
+            - name: pgdata
+              mountPath: /var/lib/postgresql/data
+            - name: initdb
+              mountPath: /docker-entrypoint-initdb.d
+      volumes:
+        - name: pgdata
+          persistentVolumeClaim:
+            claimName: postgres-data
+        - name: initdb
+          configMap:
+            name: postgres-initdb
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+  namespace: apps
+spec:
+  selector:
+    app: postgres
+  ports:
+    - port: 5432
+      targetPort: 5432

clusters/contabo-mkt/tenants/test12-2/apps/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: apps
+resources:
+  - app-nextcloud.yaml
+  - app-vaultwarden.yaml
+  - db-postgres.yaml
+  - namespace.yaml

clusters/contabo-mkt/tenants/test12-2/apps/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: apps