fix: ACME ownership self-heal + daily timer, apply-all force, firewall baseline re-stamp

acme:
- acme.sh chmods its tree to owner-only (700/600) every run, which
  broke the two-user model: a tree left owner-only by one user made
  every acme.sh call of the other exit 2
- normalize_acme_home() reopens group access (sudo chmod g+rwX,
  files only — setgid dirs trip RestrictSUIDSGID); _run_acme_preflight
  is the choke point before every daemon acme.sh call + startup
- acme service now runs as the daemon user; --log persists the raw CA
  transcript; SYS_LOG=6 journals manual issue/renew runs
- timer daily-only: two runs/day landed inside ZeroSSL's 24h
  validation backoff (Retry-After: 86400) — a permanent renewal lockout
- _collect_acme no longer raises on cert-list failure; reports
  status.error (AcmeState.status) so the certs page can surface it

firewall: re-stamp the applied baseline on live zone mutations
(interfaces/services/rich-rules/masquerade/forward-ports) so cancel-all
reverts to post-mutation state, not a stale install-era snapshot;
set_masquerade syncs the declarative config for existing zones;
add_forward_port records toaddr only with toport

status: apply-all accepts {"force": true} (forwarded to the firewall
apply only); ApplyConfirm force checkbox; applyResultToasts() — the
errors map wins over the 200; ActionButton checks errors before the
success toast; dashboard uses ApplyConfirm

system_import: drift re-imports carry the existing apply-meta; first
import stamps the adopted content as applied (it is the running state)
— no phantom pending changes

nginx: get_config only re-saves when migration actually changed the
config (no more owner/mtime churn on every read)

install: repair mis-owned top-level system dirs (tmpfiles
unsafe-path-transition), warn with a full-repair command for deeper
mis-ownership

daemon/server: loop.get_exception_handler() (aiohttp API fix)

tests: 888 pytest + 24 node passing; ruff clean

daemon/handlers/acme.py

@@ -54,6 +54,52 @@ _ACME_ENVIRON = {
 
 _WEBROOT = PROJECT_DIR / "data" / "acme" / "www"
 
+
+def normalize_acme_home() -> None:
+    """Restore group access on the ACME home files around acme.sh runs.
+
+    acme.sh hardens its tree on every run (``chmod 700`` on the config
+    home, ``chmod 600`` on keys and confs, owned by the running user).
+    The daemon reopens group read/write via the sudoers whitelist so
+    the shared two-user model keeps the tree readable. Run BEFORE an
+    acme.sh invocation too: acme.sh dot-sources ``account.conf`` on
+    startup, so a tree left owner-only by another user's run (e.g. a
+    manual debug run as the WebUI user) would make every daemon acme.sh
+    call exit 2 — normalizing first is the only self-heal path, since a
+    post-run normalize is unreachable while acme.sh cannot start.
+
+    Files only: the directories in the tree are setgid (2775, group rwx
+    already), and chmodding a setgid directory issues fchmodat with the
+    S_ISGID bit set, which the unit's ``RestrictSUIDSGID=yes`` seccomp
+    filter rejects with EPERM even for root.
+    """
+    files = [str(p) for p in _ACME_HOME.rglob("*") if p.is_file()]
+    if not files:
+        return
+    result = lib_common.run_proc(
+        ["chmod", "g+rwX", *files],
+        sudo=True,
+        check=False,
+        timeout=10,
+    )
+    if result.returncode != 0:
+        logger.warning(
+            "Could not normalize ACME_HOME permissions: %s",
+            result.stderr.strip() or f"exit code {result.returncode}",
+        )
+
+
+def _run_acme_preflight(args: list[str]) -> str:
+    """Normalize ACME home permissions, then run acme.sh with *args*.
+
+    Single choke point for every daemon acme.sh invocation: the
+    preflight normalize makes the run succeed even if a prior run by
+    another user left the tree owner-only.
+    """
+    normalize_acme_home()
+    return _run_acme(args)
+
+
 # In-memory store for active issuance requests.
 _ISSUANCES: dict[str, "IssueRequest"] = {}
 
@@ -839,14 +885,16 @@ async def _run_issue(req: IssueRequest) -> None:
         args.append("--force")
         # acme.sh is a blocking subprocess — run it off the event loop so
         # polling, WS broadcasts, and other requests keep responding.
-        output = await asyncio.to_thread(_run_acme, args)
+        output = await asyncio.to_thread(_run_acme_preflight, args)
+        normalize_acme_home()
         req.steps[0].status = "done"
         req.steps[0].message = output.strip()[:200]
 
         # Step 2: deploy
         req.steps[1].status = "running"
         await asyncio.to_thread(
-            _run_acme, ["--deploy", "-d", req.domain, "--deploy-hook", _DEPLOY_HOOK]
+            _run_acme_preflight,
+            ["--deploy", "-d", req.domain, "--deploy-hook", _DEPLOY_HOOK],
         )
         req.steps[1].status = "done"
         req.steps[1].message = "Deploy hook registered"
@@ -951,7 +999,9 @@ async def _run_renew(req: IssueRequest, force: bool) -> None:
         args: list[str] = ["--renew", "-d", req.domain]
         if force:
             args.append("--force")
-        output = await asyncio.to_thread(_run_acme, args)
+        output = await asyncio.to_thread(_run_acme_preflight, args)
+        # acme.sh hardens its tree even when it skips — normalize first.
+        normalize_acme_home()
         if "Skipping." in output:
             req.steps[0].status = "done"
             req.steps[0].message = "Renewal not yet due — skipped"
@@ -968,7 +1018,8 @@ async def _run_renew(req: IssueRequest, force: bool) -> None:
         # Step 2: deploy
         req.steps[1].status = "running"
         await asyncio.to_thread(
-            _run_acme, ["--deploy", "-d", req.domain, "--deploy-hook", _DEPLOY_HOOK]
+            _run_acme_preflight,
+            ["--deploy", "-d", req.domain, "--deploy-hook", _DEPLOY_HOOK],
         )
         req.steps[1].status = "done"
         req.steps[1].message = "Deploy hook registered"
@@ -1003,7 +1054,7 @@ def remove_cert(_request: Any, body: dict[str, Any] | None) -> dict[str, Any]:
     domain = body.get("domain", "").strip()
     if not domain:
         raise ValueError("'domain' is required")
-    _run_acme(["--remove", "-d", domain])
+    _run_acme_preflight(["--remove", "-d", domain])
     logger.info("Certificate for %s removed", domain)
     refresh_state(["acme"])
     return {"domain": domain}
@@ -1021,7 +1072,7 @@ def set_email(_request: Any, body: dict[str, Any] | None) -> dict[str, Any]:
     email = body.get("email", "").strip()
     if not email:
         raise ValueError("'email' is required")
-    _run_acme(["--register-account", "-m", email])
+    _run_acme_preflight(["--register-account", "-m", email])
     # Persist to declarative ACME config
     acme_cfg = PROJECT_DIR / "config" / "acme" / "config.json"
     acme_cfg.parent.mkdir(parents=True, exist_ok=True)
@@ -1159,7 +1210,7 @@ def register_account(_request: Any, body: dict[str, Any] | None) -> dict[str, An
         raise ValueError("Invalid email format")
     server = (body.get("server") or "letsencrypt").strip()
 
-    _run_acme(["--register-account", "-m", email, "--server", server])
+    _run_acme_preflight(["--register-account", "-m", email, "--server", server])
 
     acme_cfg = PROJECT_DIR / "config" / "acme" / "config.json"
     acme_cfg.parent.mkdir(parents=True, exist_ok=True)
@@ -1179,7 +1230,7 @@ def register_account(_request: Any, body: dict[str, Any] | None) -> dict[str, An
 def deactivate_account(_request: Any, body: dict[str, Any] | None) -> dict[str, Any]:
     """DELETE /acme/account/deactivate — deactivate the ACME account."""
     try:
-        _run_acme(["--deactivate-account"])
+        _run_acme_preflight(["--deactivate-account"])
     except RuntimeError as exc:
         logger.warning("acme.sh deactivate failed: %s", exc)
     acme_cfg = PROJECT_DIR / "config" / "acme" / "config.json"

daemon/handlers/firewall.py

@@ -856,6 +856,9 @@ def set_zone_interfaces(_request: Any, body: dict[str, Any] | None) -> dict[str,
                 old_zone_cfg["interfaces"] = new_ifaces
             elif "interfaces" in old_zone_cfg:
                 del old_zone_cfg["interfaces"]
+    # This mutation already applied to live firewalld, so re-stamp the applied
+    # baseline: cancel-all must revert to this state, not an older snapshot.
+    stamp_applied(cfg)
     _save_config(cfg)
 
     logger.info("Zone '%s' interfaces set to %s", zone, interfaces)
@@ -927,9 +930,12 @@ def set_zone_services(_request: Any, body: dict[str, Any] | None) -> dict[str, A
     _reload()
 
     # Keep the declarative config in sync so the next apply does not
-    # reconcile the live services back to the stale config value.
+    # reconcile the live services back to the stale config value. The
+    # mutation already applied to live firewalld, so re-stamp the applied
+    # baseline: cancel-all must revert to this state, not an older snapshot.
     cfg = _get_config()
     cfg.setdefault("zones", {}).setdefault(zone, {})["services"] = list(services)
+    stamp_applied(cfg)
     _save_config(cfg)
     logger.info("Zone '%s' services set to %s", zone, services)
     sync_result = bus.emit(
@@ -979,6 +985,7 @@ def add_rich_rule(_request: Any, body: dict[str, Any] | None) -> dict[str, Any]:
     rule_id = uuid4().hex[:8]
     entry = {"id": rule_id, "rule": rule}
     cfg["zones"][zone]["rich_rules"].append(entry)
+    stamp_applied(cfg)
     _save_config(cfg)
     sync_result = bus.emit(
         SyncEvent(
@@ -1035,6 +1042,7 @@ def remove_rich_rule(_request: Any, body: dict[str, Any] | None) -> dict[str, An
     zone_cfg["rich_rules"] = [
         r for r in zone_cfg.get("rich_rules", []) if r.get("id") != rule_id
     ]
+    stamp_applied(cfg)
     _save_config(cfg)
     sync_result = bus.emit(
         SyncEvent(
@@ -1084,6 +1092,10 @@ def list_rich_rules(_request: Any, body: dict[str, Any] | None) -> list[dict[str
 def set_masquerade(_request: Any, body: dict[str, Any] | None) -> dict[str, Any]:
     """Enable/disable masquerade on a zone.
 
+    Also syncs the declarative config (and re-stamps the applied baseline)
+    when the zone exists in the config, so the pending diff and cancel-all
+    stay consistent with the live zone.
+
     Args:
         _request: The incoming HTTP request (unused).
         body: JSON body with ``zone`` and ``enable`` (boolean).
@@ -1108,6 +1120,16 @@ def set_masquerade(_request: Any, body: dict[str, Any] | None) -> dict[str, Any]
     action = "--add-masquerade" if enable else "--remove-masquerade"
     run(["firewall-cmd", f"--zone={zone}", action, "--permanent"], sudo=True)
     _reload()
+    # Keep the declarative config in sync with the live zone so the pending
+    # diff and the cancel-all baseline stay consistent. Only touch zones that
+    # already exist in the config — creating a bare zone entry would
+    # manufacture spurious service/interface diffs on the next poll.
+    cfg = _get_config()
+    zone_cfg = cfg.get("zones", {}).get(zone)
+    if isinstance(zone_cfg, dict):
+        zone_cfg["masquerade"] = bool(enable)
+        stamp_applied(cfg)
+        _save_config(cfg)
     sync_result = bus.emit(
         SyncEvent(
             "firewall", "config_saved", {"action": "masquerade_set", "zone": zone}
@@ -1161,13 +1183,14 @@ def add_forward_port(_request: Any, body: dict[str, Any] | None) -> dict[str, An
     _reload()
     fp_id = uuid4().hex[:8]
     entry: dict[str, Any] = {"id": fp_id, "port": int(port), "proto": proto}
-    if toaddr:
+    if toaddr and toport:
         entry["toaddr"] = toaddr
     if toport:
         entry["toport"] = int(toport)
     cfg = _get_config()
     cfg.setdefault("zones", {}).setdefault(zone, {}).setdefault("forward_ports", [])
     cfg["zones"][zone]["forward_ports"].append(entry)
+    stamp_applied(cfg)
     _save_config(cfg)
     sync_result = bus.emit(
         SyncEvent(
@@ -1233,6 +1256,7 @@ def remove_forward_port(_request: Any, body: dict[str, Any] | None) -> dict[str,
     cfg["zones"][zone]["forward_ports"] = [
         fp for fp in fps if not (fp.get("port") == port and fp.get("proto") == proto)
     ]
+    stamp_applied(cfg)
     _save_config(cfg)
     sync_result = bus.emit(
         SyncEvent(

daemon/handlers/status.py

@@ -128,12 +128,20 @@ def status_apply_all(_request: Any, _body: Any) -> dict[str, Any]:
 
     Order: network -> firewall -> wireguard -> dnsmasq -> nginx.
 
+    Args:
+        _request: The incoming HTTP request (unused).
+        _body: Optional JSON body; ``{"force": true}`` is forwarded to the
+            firewall apply, overriding its management-lockout and
+            interface-coverage guards. Other subsystems ignore it.
+
     Returns:
         Dict with applied subsystems and any errors encountered.
     """
     applied = []
     errors = {}
 
+    force = bool(_body and _body.get("force"))
+
     pending_data = status_pending(None, None)
     fw_pending = pending_data["firewall"]["needs_apply"]
     hash_pending = {
@@ -153,7 +161,10 @@ def status_apply_all(_request: Any, _body: Any) -> dict[str, Any]:
 
         handler = SYS_APPLY[name]
         try:
-            handler(None, None)
+            # Only the firewall apply honors `force` (its lockout and
+            # coverage guards); forward it there, not to other subsystems.
+            body = {"force": True} if (name == "firewall" and force) else None
+            handler(None, body)
             applied.append(name)
         except Exception as exc:
             label = SYS_LABELS.get(name, name)

daemon/server.py

@@ -698,7 +698,7 @@ def main() -> None:
         _stop_polling()
         # Suppress the default exception handler during teardown so that
         # cancelling in-flight tasks does not spew tracebacks on SIGTERM.
-        prev_handler = loop.exception_handler
+        prev_handler = loop.get_exception_handler()
         loop.set_exception_handler(_teardown_exception_handler)
         try:
             # Stop accepting new connections (also waits for open sockets,
@@ -747,6 +747,17 @@ def main() -> None:
     if reconciled:
         logger.info("Reconciled subsystems: %s", ", ".join(reconciled))
 
+    # Reopen group access on the ACME home before the first acme.sh
+    # collection: a tree left owner-only by a prior run (e.g. a manual
+    # run as the WebUI user) would otherwise fail every daemon acme.sh
+    # call until the next issue/renew. Never fatal at startup.
+    try:
+        from daemon.handlers.acme import normalize_acme_home
+
+        normalize_acme_home()
+    except Exception:
+        logger.warning("ACME home normalization failed at startup", exc_info=True)
+
     # Populate state from system (blocking — OK at startup)
     logger.info("Populating system state...")
     state_store.populate()