openova/infra/hetzner/cloudinit-control-plane.tftpl

#cloud-config
# Catalyst Sovereign control-plane bootstrap.
# Sovereign: ${sovereign_fqdn}
# Provisioned by: catalyst-provisioner (https://console.openova.io/sovereign)
#
# This script:
#   1. Installs OS hardening (SSH password-auth off, fail2ban, unattended-upgrades).
#   2. Installs k3s with --flannel-backend=none (Cilium replaces it).
#   3. Installs Flux + bootstraps the GitRepository pointing at the shared
#      clusters/_template/ tree in the public OpenOva monorepo. The
#      Sovereign's FQDN is interpolated into the template manifests via
#      Flux postBuild.substitute (${SOVEREIGN_FQDN}) at apply time, so
#      no per-Sovereign directory needs to be committed before
#      provisioning. From this point Flux is the GitOps reconciler and
#      installs the 11-component bootstrap kit (Cilium → cert-manager →
#      Crossplane → ... → bp-catalyst-platform) in dependency order via
#      Kustomizations the _template directory ships.
#   4. Touches /var/lib/catalyst/cloud-init-complete so the catalyst-api
#      provisioner can detect cloud-init has finished.

package_update: true
package_upgrade: false
packages:
  - curl
  - iptables
  - jq
  - ca-certificates
  - git
%{ if enable_fail2ban ~}
  - fail2ban
%{ endif ~}
%{ if enable_unattended_upgrades ~}
  - unattended-upgrades
  - apt-listchanges
%{ endif ~}

write_files:
  - path: /var/lib/catalyst/sovereign.json
    permissions: '0644'
    content: |
      {
        "sovereignFQDN": "${sovereign_fqdn}",
        "sovereignSubdomain": "${sovereign_subdomain}",
        "orgName": ${jsonencode(org_name)},
        "orgEmail": ${jsonencode(org_email)},
        "region": "${region}",
        "haEnabled": ${ha_enabled},
        "workerCount": ${worker_count},
        "k3sVersion": "${k3s_version}",
        "gitopsRepoUrl": "${gitops_repo_url}",
        "gitopsBranch": "${gitops_branch}"
      }

  # ── OS hardening: SSH daemon ──────────────────────────────────────────
  # Drop-in overrides /etc/ssh/sshd_config defaults. Per Catalyst's threat
  # model the operator's only valid path in is the Hetzner-project SSH key
  # injected via cloud-init authorized_keys. Password auth, KbdInteractive,
  # and root password login are all off.
  - path: /etc/ssh/sshd_config.d/99-catalyst-hardening.conf
    permissions: '0644'
    content: |
      # Managed by Catalyst Sovereign cloud-init — do not edit by hand.
      PasswordAuthentication no
      KbdInteractiveAuthentication no
      ChallengeResponseAuthentication no
      PermitRootLogin prohibit-password
      PermitEmptyPasswords no
      UsePAM yes
      X11Forwarding no
      AllowAgentForwarding no
      AllowTcpForwarding no
      ClientAliveInterval 300
      ClientAliveCountMax 2
      MaxAuthTries 3
      LoginGraceTime 30

%{ if enable_unattended_upgrades ~}
  # ── Unattended security upgrades ──────────────────────────────────────
  # Ubuntu's stock unattended-upgrades, restricted to the security pocket.
  # Runs daily, reboots automatically at 02:30 if a kernel upgrade requires
  # it (k3s tolerates single-node restarts on a solo Sovereign within the
  # ~60s window the Hetzner LB health-check covers).
  - path: /etc/apt/apt.conf.d/20auto-upgrades
    permissions: '0644'
    content: |
      APT::Periodic::Update-Package-Lists "1";
      APT::Periodic::Unattended-Upgrade "1";
      APT::Periodic::AutocleanInterval "7";
  - path: /etc/apt/apt.conf.d/52unattended-upgrades-catalyst
    permissions: '0644'
    content: |
      Unattended-Upgrade::Allowed-Origins {
        "$${distro_id}:$${distro_codename}-security";
        "$${distro_id}ESMApps:$${distro_codename}-apps-security";
        "$${distro_id}ESM:$${distro_codename}-infra-security";
      };
      Unattended-Upgrade::Automatic-Reboot "true";
      Unattended-Upgrade::Automatic-Reboot-Time "02:30";
      Unattended-Upgrade::Remove-Unused-Kernel-Packages "true";
      Unattended-Upgrade::Remove-Unused-Dependencies "true";
%{ endif ~}

%{ if enable_fail2ban ~}
  # ── fail2ban: sshd jail ───────────────────────────────────────────────
  # Even though SSH is firewalled to ssh_allowed_cidrs (or fully closed at
  # the firewall), fail2ban remains a defence-in-depth layer for the case
  # where the firewall rule is widened by an operator post-bootstrap.
  - path: /etc/fail2ban/jail.d/catalyst-sshd.local
    permissions: '0644'
    content: |
      [sshd]
      enabled = true
      port = ssh
      filter = sshd
      maxretry = 5
      findtime = 10m
      bantime = 1h
      backend = systemd
%{ endif ~}

  # ── flux-system/ghcr-pull Secret ─────────────────────────────────────
  #
  # Every HelmRepository CR in clusters/_template/bootstrap-kit/
  # references `secretRef: name: ghcr-pull` because the bp-* OCI artifacts
  # at `ghcr.io/openova-io/` are PRIVATE. Without this Secret, the
  # source-controller logs:
  #
  #   failed to get authentication secret 'flux-system/ghcr-pull':
  #     secrets "ghcr-pull" not found
  #
  # …and Phase 1 stalls at bp-cilium. The operator workaround (kubectl
  # apply the Secret by hand after Flux installs) is not durable across
  # re-provisioning of the same Sovereign — every fresh control-plane
  # boots without the Secret.
  #
  # We write the Secret into flux-system at cloud-init time, BEFORE
  # /var/lib/catalyst/flux-bootstrap.yaml is applied, so the GitRepository +
  # Kustomization land into a cluster that already has working GHCR creds.
  # The apply step is in runcmd: below; the manifest itself lives here.
  #
  # Token rotation policy: yearly, stored in 1Password under
  # "Catalyst — GHCR pull token (catalyst-ghcr-pull-token)". See
  # docs/SECRET-ROTATION.md. The token NEVER lives in git.
  - path: /var/lib/catalyst/ghcr-pull-secret.yaml
    permissions: '0600'
    content: |
      apiVersion: v1
      kind: Secret
      metadata:
        name: ghcr-pull
        namespace: flux-system
      type: kubernetes.io/dockerconfigjson
      data:
        .dockerconfigjson: ${base64encode(jsonencode({
          auths = {
            "ghcr.io" = {
              username = ghcr_pull_username
              password = ghcr_pull_token
              auth     = ghcr_pull_auth_b64
            }
          }
        }))}

  # ── flux-system/object-storage Secret (issue #371, vendor-agnostic since #425) ─
  #
  # The Sovereign's per-cluster S3 credentials, materialised as a stock
  # Kubernetes Secret in the `flux-system` namespace. Harbor (#383) and
  # Velero (#384) consume this Secret via the canonical `secretRef` field
  # in their respective HelmRelease values blocks, e.g.
  #
  #   harbor:
  #     persistence:
  #       imageChartStorage:
  #         type: s3
  #         s3:
  #           existingSecret: object-storage
  #
  # Per #425 the Secret name is vendor-AGNOSTIC (`object-storage`, no
  # `hetzner-` prefix). A future AWS / Azure / GCP / OCI Sovereign
  # provisions the same Secret name with the same key set via its own
  # `infra/<provider>/` Tofu module — every existing chart Just Works
  # without renaming.
  #
  # The Secret is namespace-bound to flux-system so the helm-controller can
  # rewrite it into the workload namespaces at chart install time — that's
  # the same boundary `ghcr-pull` already uses, so the apply ordering in
  # runcmd: below stays a single sequenced step.
  #
  # Why pre-populated by cloud-init rather than a SealedSecret committed to
  # git: ADR-0001 §9.2 forbids bespoke cloud-API calls and Hetzner exposes
  # NO Cloud API for S3 credential issuance — they're operator-issued in
  # the Hetzner Console exactly once. Therefore catalyst-api receives the
  # plaintext from the wizard, validates it, and forwards it to the new
  # Sovereign via the same encrypted-PVC + cloud-init channel as the GHCR
  # pull token. The credentials never land in git; the only durable copies
  # are the per-deployment OpenTofu workdir (mode 0600, wiped on tofu
  # destroy) and inside the new Sovereign's etcd (encrypted at rest by
  # k3s default).
  #
  # Token rotation policy: per Hetzner's docs, the secret half is shown
  # exactly once at issue time. To rotate, the operator issues a fresh
  # credential pair in the Hetzner Console, updates the wizard payload
  # for the next provisioning, OR for an existing Sovereign uses a
  # day-2 Crossplane XRC write (the Provider+ProviderConfig planted
  # below makes this possible without a Tofu re-run; out of scope for
  # the initial bootstrap).
  - path: /var/lib/catalyst/object-storage-secret.yaml
    permissions: '0600'
    content: |
      apiVersion: v1
      kind: Secret
      metadata:
        name: object-storage
        namespace: flux-system
      type: Opaque
      stringData:
        # S3 endpoint URL — composed from object_storage_region in main.tf.
        # Format: https://<region>.your-objectstorage.com per Hetzner docs.
        s3-endpoint: ${object_storage_endpoint}
        # S3 region — fsn1 / nbg1 / hel1 (Object Storage availability is
        # European-only as of 2026-04). NOT the same as compute region.
        s3-region: ${object_storage_region}
        # Bucket name — deterministic per-Sovereign identifier composed by
        # the catalyst-api from the Sovereign's FQDN slug. Created (or
        # adopted if already present) by the minio_s3_bucket resource in
        # main.tf earlier in this same `tofu apply`.
        s3-bucket: ${object_storage_bucket_name}
        # Operator-issued S3 access key + secret key. Hetzner's docs note
        # the secret half is shown exactly once at credential creation
        # time and is irretrievable thereafter — losing it means rotating.
        s3-access-key: ${object_storage_access_key}
        s3-secret-key: ${object_storage_secret_key}

  # ── flux-system/cloud-credentials Secret (issue #425, OpenTofu→Crossplane) ─
  #
  # Bootstrap of the Crossplane Hetzner Cloud provider (planted further
  # below in this cloud-init). Carries the operator's hcloud API token —
  # the same token Tofu used to provision Phase 0 — under a single key
  # `hcloud-token`. Per ADR-0001 §11.3 + INVIOLABLE-PRINCIPLES #3,
  # Day-2 cloud-resource changes (additional Floating IPs, additional
  # buckets, scaling LoadBalancers, firewall rule edits, ...) flow
  # through Crossplane XRC writes against this Provider — NEVER through
  # bespoke Go cloud-API calls in catalyst-api, NEVER through manual
  # Tofu re-runs.
  #
  # The Secret name is vendor-agnostic (`cloud-credentials`); the
  # `hcloud-token` key name encodes the cloud-specific shape of the
  # credential. A future AWS Sovereign would write
  # `aws-access-key-id`/`aws-secret-access-key` keys into the same
  # Secret name; the matching Crossplane Provider/ProviderConfig
  # (added in the same Tofu module's cloud-init) reads them.
  - path: /var/lib/catalyst/cloud-credentials-secret.yaml
    permissions: '0600'
    content: |
      apiVersion: v1
      kind: Secret
      metadata:
        name: cloud-credentials
        namespace: flux-system
      type: Opaque
      stringData:
        hcloud-token: ${hcloud_token}

  # ── Crossplane provider-hcloud + ProviderConfig (issue #425) ────────
  #
  # Phase 0→Day-2 handover. After Flux installs Crossplane core (via
  # bp-crossplane in the bootstrap-kit), this Provider package + its
  # ProviderConfig come up in the cluster and become the seam for ALL
  # subsequent Hetzner Cloud mutations.
  #
  # Per ADR-0001 §11.3 + INVIOLABLE-PRINCIPLES #3:
  #   - OpenTofu provisions Phase 0 EXACTLY ONCE per Sovereign.
  #   - Crossplane is the only Day-2 cloud-API mutation seam.
  #   - Flux is the only GitOps reconciler.
  #   - Blueprints (`bp-<name>:<semver>` OCI) are the only install unit.
  #   - NEVER bespoke Go cloud-API calls. NEVER `exec.Command("helm",
  #     ...)`. NEVER direct `kubectl apply` of production manifests.
  #
  # Once `provider-hcloud` reaches `Healthy=True` (event the Catalyst
  # control plane observes via the Crossplane status conditions), the
  # catalyst-api's bespoke Hetzner-API calls for any RUNTIME-scaling
  # concern (additional Floating IPs, additional buckets, scaling
  # LoadBalancers, ...) MUST be retired in favour of XRC writes against
  # this Provider. Provisioning Phase 0 (the very first server, network,
  # firewall, LB, bucket) stays in this Tofu module by design — that's
  # the bootstrap exception that lets the Provider exist in the first
  # place.
  #
  # Package version pin: v0.4.0 of `crossplane-contrib/provider-hcloud`
  # is the latest stable as of 2026-05. Per INVIOLABLE-PRINCIPLES #4
  # (never hardcode), the version is operator-bumpable via PR; future
  # rotations land here in the same commit that bumps the
  # `bp-crossplane-claims` Composition referencing the new Provider
  # types.
  - path: /var/lib/catalyst/crossplane-provider-hcloud.yaml
    permissions: '0644'
    content: |
      ---
      apiVersion: pkg.crossplane.io/v1
      kind: Provider
      metadata:
        name: provider-hcloud
        labels:
          catalyst.openova.io/sovereign: ${sovereign_fqdn}
      spec:
        package: xpkg.upbound.io/crossplane-contrib/provider-hcloud:v0.4.0
        packagePullPolicy: IfNotPresent
      ---
      apiVersion: hcloud.crossplane.io/v1beta1
      kind: ProviderConfig
      metadata:
        name: default
      spec:
        credentials:
          source: Secret
          secretRef:
            namespace: flux-system
            name: cloud-credentials
            key: hcloud-token

  # Flux GitRepository + Kustomizations that take over after k3s is up.
  #
  # ── Per-Sovereign tree vs. shared _template (issue #218) ─────────────
  #
  # Earlier revisions of this template selected a per-FQDN cluster tree
  # (`!/clusters/${sovereign_fqdn}`) and pointed the Kustomization
  # `spec.path` at `./clusters/${sovereign_fqdn}/bootstrap-kit`. That
  # required a per-Sovereign directory to be committed to the public
  # openova repo BEFORE provisioning, which the wizard does NOT do —
  # only `clusters/_template/` is canonical. Result on every fresh
  # Sovereign was Phase-1 stall:
  #   kustomization path not found:
  #     stat /tmp/kustomization-…/clusters/<fqdn>/bootstrap-kit:
  #     no such file or directory
  # (live evidence: otech.omani.works deployment ce476aaf80731a46.)
  #
  # Canonical fix: GitRepository selects the shared `_template/` tree,
  # Kustomization paths point at `clusters/_template/{bootstrap-kit,
  # infrastructure}`, and Flux's `postBuild.substitute` interpolates
  # `${SOVEREIGN_FQDN}` into the template manifests at apply time. The
  # per-FQDN copy that prior provisioning depended on becomes a no-op:
  # one shared tree serves every Sovereign, with the Sovereign's FQDN
  # injected by Flux on the cluster instead of by sed in the repo.
  - path: /var/lib/catalyst/flux-bootstrap.yaml
    permissions: '0644'
    content: |
      apiVersion: source.toolkit.fluxcd.io/v1
      kind: GitRepository
      metadata:
        name: openova
        namespace: flux-system
      spec:
        interval: 1m
        url: ${gitops_repo_url}
        ref:
          branch: ${gitops_branch}
        ignore: |
          /*
          !/clusters/_template
          !/platform
          !/products
      ---
      # Two Flux Kustomizations with dependsOn so Crossplane CRDs land
      # before any resource that uses them is dry-run-applied.
      #
      # bootstrap-kit installs the 11 HelmReleases (Cilium, cert-manager,
      # Flux, Crossplane core, sealed-secrets, SPIRE, NATS-JetStream,
      # OpenBao, Keycloak, Gitea, bp-catalyst-platform). bp-crossplane
      # registers the Crossplane core CRDs (Provider, ProviderConfig…)
      # AND the bp-catalyst-platform umbrella reconciles the rest.
      #
      # infrastructure-config applies the cluster's Provider package +
      # ProviderConfig + Compositions. Because it dependsOn bootstrap-kit
      # AND uses wait: true, Flux waits until bootstrap-kit's HelmReleases
      # are Ready (Crossplane core + provider-hcloud installed,
      # hcloud.crossplane.io/v1beta1 CRDs registered) before dry-running
      # ProviderConfig — which is the exact ordering the prior single-
      # Kustomization model tripped over with:
      #   no matches for kind "ProviderConfig" in version
      #   "hcloud.crossplane.io/v1beta1"
      #
      # postBuild.substitute (issue #218): Flux's envsubst runs over the
      # rendered manifests after kustomize build, replacing ${SOVEREIGN_FQDN}
      # with the Sovereign's FQDN that this cloud-init was rendered for.
      # The template manifests in clusters/_template/bootstrap-kit/*.yaml
      # use ${SOVEREIGN_FQDN} as the substitution token.
      apiVersion: kustomize.toolkit.fluxcd.io/v1
      kind: Kustomization
      metadata:
        name: bootstrap-kit
        namespace: flux-system
      spec:
        interval: 5m
        path: ./clusters/_template/bootstrap-kit
        prune: true
        sourceRef:
          kind: GitRepository
          name: openova
        wait: true
        timeout: 30m
        postBuild:
          substitute:
            SOVEREIGN_FQDN: ${sovereign_fqdn}
      ---
      apiVersion: kustomize.toolkit.fluxcd.io/v1
      kind: Kustomization
      metadata:
        name: infrastructure-config
        namespace: flux-system
      spec:
        interval: 5m
        path: ./clusters/_template/infrastructure
        prune: true
        sourceRef:
          kind: GitRepository
          name: openova
        dependsOn:
          - name: bootstrap-kit
        wait: true
        timeout: 30m
        postBuild:
          substitute:
            SOVEREIGN_FQDN: ${sovereign_fqdn}

runcmd:
  - swapoff -a
  - sed -i '/swap/d' /etc/fstab
  - update-alternatives --set iptables /usr/sbin/iptables-legacy || true
  - update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy || true

  # Activate hardened sshd config (cloud-init may have written authorized_keys
  # already from Hetzner ssh_keys[]; we never touch that file).
  - systemctl reload ssh || systemctl reload sshd || true
%{ if enable_fail2ban ~}
  - systemctl enable --now fail2ban
%{ endif ~}
%{ if enable_unattended_upgrades ~}
  - systemctl enable --now unattended-upgrades
%{ endif ~}

  # k3s control-plane. Flags per docs/SOVEREIGN-PROVISIONING.md §3 and
  # docs/PLATFORM-TECH-STACK.md §8.1:
  #   --cluster-init                Initialise embedded etcd (HA-ready).
  #   --flannel-backend=none        Cilium replaces flannel.
  #   --disable=traefik             Cilium Gateway replaces traefik.
  #   --disable=servicelb           Hetzner LB handles ingress.
  #   --disable-network-policy      Cilium handles NetworkPolicy.
  #   --tls-san=${sovereign_fqdn}   API server cert valid for the sovereign FQDN.
  #
  # NOTE: --disable=local-storage is intentionally NOT passed. k3s ships a
  # built-in local-path-provisioner (Rancher) and registers a `local-path`
  # StorageClass. That is the canonical solo-Sovereign StorageClass:
  # PVCs (bp-spire data dir, bp-keycloak postgres, bp-openbao raft store,
  # bp-nats-jetstream, bp-gitea, bp-catalyst-platform postgres) bind to
  # node-local storage on the single CPX21/CPX31 control-plane node and
  # come up immediately. Operators upgrading to multi-node migrate to
  # hcloud-csi (Hetzner Cloud Volumes) as a separate, deliberate step —
  # see docs/RUNBOOK-PROVISIONING.md §"StorageClass missing".
  #
  # Architectural background: the prior version of this template passed
  # `--disable=local-storage` with the intent that Crossplane would
  # install hcloud-csi day-2 and register the StorageClass after
  # bp-crossplane reconciled. That created a circular dependency: the
  # 11-component bootstrap kit (bp-spire / bp-keycloak / bp-openbao / …)
  # all carry PVCs whose bind step blocks waiting for a StorageClass that
  # would only exist AFTER bp-crossplane had finished installing AND
  # provisioned hcloud-csi. Result on a fresh Sovereign: every PVC stuck
  # Pending forever, bootstrap-kit deadlocked. Keeping local-path solves
  # the circularity by giving the cluster a default StorageClass at boot.
  - 'curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${k3s_version} K3S_TOKEN=${k3s_token} INSTALL_K3S_EXEC="server --cluster-init --flannel-backend=none --disable-network-policy --disable=traefik --disable=servicelb --tls-san=${sovereign_fqdn} --node-label catalyst.openova.io/role=control-plane --write-kubeconfig-mode=0644" sh -'

  # Wait for the API server to be reachable. Cilium needs to come up before
  # nodes Ready, so we wait specifically for the API endpoint.
  - 'until kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml get --raw /healthz; do sleep 5; done'

  # ── Default StorageClass: local-path (k3s built-in) ─────────────────────
  #
  # k3s ships local-path-provisioner (deployment in kube-system,
  # `app=local-path-provisioner`) and registers a `local-path`
  # StorageClass on first boot. We need the StorageClass to exist AND
  # be marked default BEFORE Flux applies the bootstrap-kit Kustomization
  # below — otherwise the 11-component bootstrap kit (bp-spire,
  # bp-keycloak postgres, bp-openbao, bp-nats-jetstream, bp-gitea,
  # bp-catalyst-platform postgres) ships HelmReleases with PVCs that
  # have no `storageClassName` set, expecting the cluster default to
  # take over. Without a default, every one of those PVCs sits Pending
  # waiting on a class that nobody nominates, and the bootstrap-kit
  # Kustomization deadlocks.
  #
  # Sequence (#207 — fix the circular wait that blocked every fresh provision):
  #   1. Poll until the `local-path` StorageClass object is registered by
  #      k3s. We CANNOT wait for the local-path-provisioner POD to be
  #      Ready here — k3s runs with --flannel-backend=none so the node
  #      stays Ready=False until Cilium installs (further down). Waiting
  #      on the Pod creates a circular deadlock and 60s timeout. The SC
  #      object itself is registered by k3s manifests independently of CNI
  #      (verified live: SC creationTimestamp 3s after k3s start).
  #   2. Patch the `local-path` StorageClass with the
  #      `storageclass.kubernetes.io/is-default-class: "true"` annotation.
  #   3. Verify (the poll already implies presence; the explicit grep stays
  #      as defensive belt-and-braces, identical exit semantics).
  - 'until kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml get sc local-path >/dev/null 2>&1; do sleep 2; done'
  - 'kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml patch storageclass local-path -p ''{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'''
  - 'kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml get sc -o name | grep -q "^storageclass.storage.k8s.io/local-path$" || { echo "FATAL: local-path StorageClass missing after k3s install — see docs/RUNBOOK-PROVISIONING.md StorageClass missing section" >&2; exit 1; }'

%{ if deployment_id != "" && kubeconfig_bearer_token != "" && catalyst_api_url != "" ~}
  # ── Cloud-init kubeconfig postback (issue #183, Option D) ───────────────
  #
  # The k3s install above wrote /etc/rancher/k3s/k3s.yaml with the API
  # server URL pinned to https://127.0.0.1:6443 — kubectl's default for a
  # local single-node install. catalyst-api lives off-cluster (Catalyst-Zero
  # franchise console on contabo-mkt) and cannot reach 127.0.0.1 on this
  # node, so we MUST rewrite that field before sending the kubeconfig
  # back. The Hetzner load balancer at $${load_balancer_ipv4} forwards
  # 6443 to the control plane's 6443 (firewall rule above), so a kubeconfig
  # pointing at the LB's public IPv4 is reachable from anywhere.
  #
  # Plaintext: we read from /etc/rancher/k3s/k3s.yaml (mode 0644 written
  # by k3s), apply the rewrite via sed, write the result to
  # /etc/rancher/k3s/k3s.yaml.public (mode 0600 explicitly), then
  # curl --data-binary the file content to catalyst-api with the bearer
  # token. The .public file is removed at the end of the runcmd block
  # so the bearer-protected kubeconfig only lives on this node for the
  # few seconds it takes to PUT.
  #
  # --retry 60 --retry-delay 10 --retry-all-errors handles the case
  # where catalyst-api is briefly unreachable (image roll, ingress
  # reconciliation) — the cloud-init runcmd budget is bounded by the
  # systemd cloud-final timeout (~30 minutes).
  - install -m 0600 /dev/null /etc/rancher/k3s/k3s.yaml.public
  - sed 's|https://127.0.0.1:6443|https://${load_balancer_ipv4}:6443|g' /etc/rancher/k3s/k3s.yaml > /etc/rancher/k3s/k3s.yaml.public
  - chmod 0600 /etc/rancher/k3s/k3s.yaml.public
  - |
    curl -fsSL --retry 60 --retry-delay 10 --retry-all-errors \
      -X PUT \
      -H "Authorization: Bearer ${kubeconfig_bearer_token}" \
      -H "Content-Type: application/x-yaml" \
      --data-binary @/etc/rancher/k3s/k3s.yaml.public \
      ${catalyst_api_url}/api/v1/deployments/${deployment_id}/kubeconfig
  - rm -f /etc/rancher/k3s/k3s.yaml.public
%{ endif ~}

  # ── Cilium FIRST (before Flux) ───────────────────────────────────────────
  #
  # k3s started with --flannel-backend=none, so the cluster has NO CNI yet.
  # If we apply Flux install.yaml at this point, the Flux controller pods
  # stay Pending forever — kubelet rejects them with
  #   "container runtime network not ready: cni plugin not initialized"
  # Flux is then unable to reconcile bp-cilium, so Cilium is never
  # installed → bootstrap deadlock that we hit in production at
  # omantel.omani.works deployment 5cd1bceaaacb71f6 (25 min stuck Pending).
  #
  # Bootstrap chicken-and-egg: Cilium IS the install unit (bp-cilium), but
  # Flux needs a CNI to run, and Cilium IS the CNI. Resolution: install
  # Cilium ONCE here via Helm with the same chart + values bp-cilium would
  # apply later. When Flux reconciles bp-cilium, it adopts the existing
  # release (Helm release-name match), so there is no churn.
  #
  # Per INVIOLABLE-PRINCIPLES.md #3 the GitOps engine is Flux — this Helm
  # install is the one-shot bootstrap exception explicitly authorised by
  # the same principle's "everything ELSE" qualifier. The chart version
  # matches platform/cilium/blueprint.yaml's chartVersion to keep the
  # bootstrap install and the reconciled HelmRelease byte-identical.
  - 'curl -sSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash'
  - 'helm repo add cilium https://helm.cilium.io/'
  - 'helm repo update'
  - |
    KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm install cilium cilium/cilium \
      --version 1.16.5 \
      --namespace kube-system \
      --set kubeProxyReplacement=true \
      --set k8sServiceHost=127.0.0.1 \
      --set k8sServicePort=6443 \
      --set ipam.mode=kubernetes \
      --set tunnelProtocol=vxlan \
      --set bpf.masquerade=true
  - 'kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml -n kube-system rollout status ds/cilium --timeout=240s'

  # Install Flux core. Cilium is now the cluster's CNI, so Flux pods will
  # actually start. Flux then reconciles clusters/_template/ (with
  # SOVEREIGN_FQDN substituted via postBuild — issue #218) which
  # adopts the Helm release above as bp-cilium and continues with
  # bp-cert-manager, bp-flux (which ADOPTS this Flux install rather than
  # reinstalls — see version-pin invariant below), bp-crossplane, etc.
  #
  # CRITICAL VERSION-PIN INVARIANT — DO NOT CHANGE IN ISOLATION
  # -----------------------------------------------------------
  # The version pinned in the URL below MUST match the upstream Flux
  # release that `platform/flux/chart/Chart.yaml`'s `flux2` subchart
  # bundles, otherwise bp-flux's HelmRelease runs `helm install` on top
  # of THIS Flux installation with a different upstream version, the
  # CRD `status.storedVersions` mismatches, Helm install fails, rollback
  # fires, and rollback DELETES the running Flux controllers — leaving
  # the cluster with no GitOps engine, unrecoverable in-place.
  #
  # Live verified on omantel.omani.works on 2026-04-29 — every Sovereign
  # provisioned without this pin in sync was destroyed minutes after
  # bp-flux's first reconcile. See docs/RUNBOOK-PROVISIONING.md
  # §"bp-flux double-install".
  #
  # Mapping (cloud-init install.yaml -> chart subchart -> appVersion):
  #   v2.4.0  ->  flux2 2.14.1  ->  appVersion 2.4.0  <- CURRENT
  #   v2.3.0  ->  flux2 2.13.0  ->  appVersion 2.3.0
  #
  # CI gate `platform/flux/chart/tests/version-pin-replay.sh` rejects
  # divergence between this URL's version and the chart's subchart pin.
  - 'curl -fsSL https://github.com/fluxcd/flux2/releases/download/v2.4.0/install.yaml | kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml apply -f -'
  - 'kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml -n flux-system wait --for=condition=Available --timeout=300s deployment --all'

  # ── flux-system/ghcr-pull Secret (applied BEFORE GitRepository) ──────
  #
  # Apply the docker-registry pull secret rendered above. This MUST land
  # before the GitRepository + Kustomization in flux-bootstrap.yaml,
  # because the bootstrap-kit Kustomization includes HelmRepository CRs
  # that reference this Secret by name; the source-controller resolves
  # them on its first reconciliation tick and a missing Secret propagates
  # as a Ready=False/AuthError state that has been observed to persist
  # for 5+ minutes even after the Secret is later applied.
  #
  # Idempotent: `kubectl apply` against an existing Secret is a no-op
  # when the manifest's bytes match. A reprovision (same Sovereign FQDN)
  # rewrites this with the same content; a token rotation propagates
  # through here on the next cloud-init render.
  - 'kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml apply -f /var/lib/catalyst/ghcr-pull-secret.yaml'

  # ── OpenBao auto-unseal seed Secret (issue #316) ─────────────────────
  #
  # Generate a one-shot 32-byte recovery seed during cloud-init and
  # write it to a K8s Secret `openbao-recovery-seed` in the `openbao`
  # namespace. The bp-openbao chart (v1.2.0+) renders a post-install
  # Job (templates/init-job.yaml, Helm hook weight 5) that:
  #   1. Reads this seed Secret.
  #   2. Calls `bao operator init -recovery-shares=1 -recovery-threshold=1`.
  #   3. Persists the recovery key inside OpenBao's auto-unseal config
  #      (so subsequent pod restarts unseal automatically).
  #   4. Deletes this seed Secret on success.
  #
  # The seed is single-use — once consumed by the init Job, it never
  # exists again. The recovery key + root token live ONLY inside
  # OpenBao's Raft state (acceptance criterion #6 of issue #316).
  #
  # Why a fresh /dev/urandom value (NOT a value baked into Terraform):
  # the recovery seed must NEVER be readable from outside the
  # control-plane node, NEVER appear in tfstate, NEVER appear in any
  # cloud-init render audit log. Generating it here at provision time
  # means the only window of plaintext exposure is the few seconds
  # between this Secret apply and the Helm post-install Job consuming
  # it — bounded by the bootstrap-kit reconcile cadence (1m max).
  #
  # Why we create the namespace here: the bp-openbao HelmRelease in
  # clusters/_template/bootstrap-kit/08-openbao.yaml ships a Namespace
  # manifest, but Flux applies that Namespace + the HelmRelease
  # together. The Helm post-install hook would race the seed Secret
  # apply if we waited for Flux to create the namespace. Pre-creating
  # the namespace at cloud-init time eliminates the race.
  #
  # Idempotency: `kubectl apply` of the namespace and `kubectl create
  # secret --dry-run=client -o yaml | kubectl apply -f -` of the
  # Secret are both safe to re-run. A re-provision (same Sovereign
  # FQDN) regenerates a fresh seed and re-applies — at which point the
  # init Job has either already consumed the previous seed (so the new
  # one becomes a no-op the next time the Helm hook runs) OR sees
  # OpenBao already initialised and exits idempotently without
  # touching the new seed.
  - 'kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml create namespace openbao --dry-run=client -o yaml | kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml apply -f -'
  - |
    OPENBAO_SEED=$(head -c 32 /dev/urandom | base64 | tr -d '\n')
    kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml -n openbao create secret generic openbao-recovery-seed \
      --from-literal=recovery-seed="$OPENBAO_SEED" \
      --dry-run=client -o yaml \
      | kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml annotate --local -f - \
        openbao.openova.io/single-use=true -o yaml \
      | kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml apply -f -
    unset OPENBAO_SEED

  # ── flux-system/object-storage Secret (issue #371, vendor-agnostic since #425) ─
  #
  # Apply the operator-issued Object Storage credentials so they're in
  # the cluster BEFORE Flux reconciles bp-harbor (#383) and bp-velero
  # (#384). Both Blueprints reference `secretRef: name: object-storage`
  # in their HelmRelease values; without this Secret the install reports
  # NoSuchKey at chart-install probe time and Phase 1 stalls.
  #
  # Same idempotency property as ghcr-pull above — re-running cloud-init
  # against an existing Sovereign overwrites the manifest with the same
  # bytes (or rotated bytes when the operator has issued fresh keys); a
  # missing-bucket scenario is impossible by construction because main.tf's
  # minio_s3_bucket resource creates the bucket in the same `tofu apply`
  # run that renders this user_data.
  - 'kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml apply -f /var/lib/catalyst/object-storage-secret.yaml'

  # ── flux-system/cloud-credentials Secret + Crossplane Provider (issue #425) ─
  #
  # Apply the Hetzner Cloud API token Secret + the Crossplane Provider
  # package + ProviderConfig BEFORE Flux's bootstrap-kit lands
  # bp-crossplane. The Provider package itself is installed by
  # Crossplane core (which bp-crossplane brings up); applying the
  # Provider CR here just registers the package install request — it
  # transitions Healthy=True a few minutes later once the bootstrap-
  # kit's Crossplane core controllers come online. The ProviderConfig
  # sits in waiting state until the Provider's CRDs are registered, at
  # which point it goes Ready=True and the Sovereign is ready to accept
  # Day-2 XRC writes.
  #
  # Per ADR-0001 §11.3 + INVIOLABLE-PRINCIPLES #3 this is the OpenTofu
  # → Crossplane handover seam. Tofu provisions Phase 0 exactly once;
  # everything else flows through XRC writes against this Provider.
  - 'kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml apply -f /var/lib/catalyst/cloud-credentials-secret.yaml'
  - 'kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml apply -f /var/lib/catalyst/crossplane-provider-hcloud.yaml'

  # Apply the Flux bootstrap GitRepository + Kustomization. From here, Flux
  # owns the cluster: pulls clusters/_template/ (with ${SOVEREIGN_FQDN}
  # substituted to ${sovereign_fqdn} via postBuild), installs Cilium
  # via bp-cilium, cert-manager via bp-cert-manager, etc., then bp-catalyst-platform.
  - 'kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml apply -f /var/lib/catalyst/flux-bootstrap.yaml'

  # Marker for the catalyst-api provisioner to detect cloud-init is done.
  - mkdir -p /var/lib/catalyst
  - touch /var/lib/catalyst/cloud-init-complete

final_message: "Catalyst control-plane bootstrap complete after $UPTIME seconds — Flux is now reconciling clusters/_template/ for ${sovereign_fqdn}"