mteehan/vacuum-wall
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ws: migrate push stream to data streaming

Browse Source 
- daemon: send full snapshot on connect; versions/tick now carry the
  full state of one subsystem (subsystem + data); no legacy
  updated/subsystems payloads; refresh_state and POST /status/refresh
  broadcast per-subsystem versions with data
- client: modelSet() patches models in place; onMessage/topic refresh
  retired; 3s initial-load fallback via new POST /api/status/refresh
- schema: lib/schema.py TypedDicts + hoover/schema.js defaults +
  docs/state-model.md as single source of truth for state shapes
- system: poll at 1s, volatile metrics registered, dashboard uses a
  dedicated system model (status model removed)
- firewall: refuse to strip both https and ssh from the default zone
  (409, force override via UI confirm); set_zone_services persists
  services to the declarative config; collector exposes default_zone
- UI: pages migrate to flat state shapes; post-mutation modelFetch
  refreshes removed (WS delta covers it)
- tests: ws snapshot/delta/broadcast, refresh-state, schema types,
  model-set/js ws handler and reconnect fallback
This commit is contained in:
Mike Teehan
2026-08-20 01:38:00 +00:00
parent 9c9f92ad04
commit 332d14e37d
45 changed files with 2819 additions and 496 deletions
Download Patch File Download Diff File Expand all files Collapse all files
daemon/handlers/firewall.py
+75 -7
View File
@@ -33,7 +33,7 @@ from daemon.iface import (
POST_FIREWALL_ZONES_INTERFACES,
POST_FIREWALL_ZONES_SERVICES,
)
from daemon.server import NotFoundError, refresh_state, registry
from daemon.server import ConflictError, NotFoundError, refresh_state, registry
from lib.common import load_json, run, save_json
from lib.firewall import (
_normalize_target,
@@ -85,6 +85,35 @@ def _reload() -> None:
run(["firewall-cmd", "--reload"], sudo=True)
def _default_zone() -> str:
"""Return the firewalld default zone name.
The default zone is the catch-all for any interface without an explicit
zone assignment (including VPN interfaces), so it normally fronts the WAN.
"""
return run(["firewall-cmd", "--get-default-zone"], sudo=True).strip()
def _would_remove_mgmt(zone: str, services: list[str]) -> bool:
"""Return True if *services* lacks both https and ssh on the default zone.
The default zone fronts unassigned (WAN/VPN) interfaces, so removing both
management access (https via nginx) and remote recovery (ssh) from it
would leave no path back except a physical console.
"""
if "https" in services or "ssh" in services:
return False
try:
default = _default_zone()
except Exception:
logger.warning(
"Could not determine the firewalld default zone; failing closed for %s",
zone,
)
return True
return zone == default
def _fp_to_str(fp: dict[str, Any]) -> str:
"""Convert a forward-port dict to firewall-cmd CLI argument string."""
parts = [f"port={fp['port']}", f"proto={fp['proto']}"]
@@ -106,18 +135,39 @@ def _get_forward_ports(zone_name: str) -> list[str]:
return []
def _config_apply() -> dict[str, Any]:
def _config_apply(force: bool = False) -> dict[str, Any]:
"""Apply saved declarative config to live firewalld.
For each zone in the config, reconciles interfaces, services, target,
masquerade, rich rules, and forward ports by removing old values first,
then adding desired values. Reloads firewalld at the end.
With *force* False (default), a ``ConflictError`` is raised before any
mutation if the config would strip both https and ssh from the default
zone; pass ``force=True`` to override.
"""
from lib.firewall import get_config as _get_lib_config
cfg = _get_lib_config()
cfg_zones = cfg.get("zones", {})
if not force:
default_zone = _default_zone()
lockout_zones = [
zn
for zn, zc in cfg_zones.items()
if zn == default_zone
and "https" not in zc.get("services", [])
and "ssh" not in zc.get("services", [])
]
if lockout_zones:
raise ConflictError(
f"Refusing to remove both https and ssh from default zone(s) "
f"{', '.join(repr(z) for z in lockout_zones)}: management access "
f"and remote recovery would be lost. Add at least one of them "
f'to the zone\'s services, or pass {{"force": true}}.'
)
full_state: dict[str, Any] = {
"active_zones": {},
"interfaces": [],
@@ -556,13 +606,15 @@ def config_apply(_request: Any, _body: Any) -> dict[str, Any]:
Args:
_request: The incoming HTTP request (unused).
_body: The request body (unused).
_body: Optional JSON body; ``{"force": true}`` overrides the
management-lockout guard for the default zone.
Returns:
Dict with ``applied_zones`` (list of zone names), ``backup`` (path),
and ``synced`` (affected subsystems).
"""
result = _config_apply()
force = bool(_body and _body.get("force"))
result = _config_apply(force=force)
logger.info("Firewall config applied: %s", result.get("applied_zones", []))
sync_result = bus.emit(
SyncEvent("firewall", "config_saved", {"action": "config_applied"})
@@ -736,18 +788,21 @@ def set_zone_interfaces(_request: Any, body: dict[str, Any] | None) -> dict[str,
@registry.register(POST_FIREWALL_ZONES_SERVICES)
def set_zone_services(_request: Any, body: dict[str, Any] | None) -> dict[str, Any]:
"""Replace zone services, emitting sync event and refreshing state.
"""Replace zone services, persist to declarative config, and refresh state.
Args:
_request: The incoming HTTP request (unused).
body: JSON body with ``zone`` and ``services`` list.
body: JSON body with ``zone`` and ``services`` list; optional
``force`` (bool) overrides the management-lockout guard.
Returns:
Dict with ``zone`` and ``services`` keys.
Raises:
ValueError: If body is missing ``zone``.
NotFoundError: If the zone does not exist.
NotFoundError: If the specified zone does not exist.
ConflictError: If the change would strip both https and ssh from the
firewalld default zone and ``force`` is not set.
"""
if not body:
raise ValueError("Request body required")
@@ -757,6 +812,12 @@ def set_zone_services(_request: Any, body: dict[str, Any] | None) -> dict[str, A
raise ValueError("'zone' is required")
if zone not in run(["firewall-cmd", "--get-zones"], sudo=True).split():
raise NotFoundError(f"Zone '{zone}' does not exist")
if not body.get("force") and _would_remove_mgmt(zone, list(services)):
raise ConflictError(
f"Refusing to remove both https and ssh from default zone '{zone}': "
f"management access and remote recovery would be lost. Add at least "
f'one of them back, or send "force": true to override.'
)
current = _parse_zone_output(
zone, run(["firewall-cmd", f"--zone={zone}", "--list-all"], sudo=True)
).get("services", [])
@@ -782,6 +843,13 @@ def set_zone_services(_request: Any, body: dict[str, Any] | None) -> dict[str, A
sudo=True,
)
_reload()
# Keep the declarative config in sync so the next apply does not
# reconcile the live services back to the stale config value.
cfg = _get_config()
cfg.setdefault("zones", {}).setdefault(zone, {})["services"] = list(services)
_save_config(cfg)
logger.info("Zone '%s' services set to %s", zone, services)
sync_result = bus.emit(
SyncEvent("firewall", "config_saved", {"action": "services_set", "zone": zone})
)
daemon/server.py
+84 -37
View File
@@ -160,7 +160,14 @@ def refresh_state(subsystems: list[str] | None = None) -> None:
except RuntimeError:
pass
else:
task = asyncio.create_task(broadcast_versions())
# Broadcast each refreshed subsystem individually (gather for parallelism)
async def _broadcast_all():
await asyncio.gather(
*[broadcast_versions(name) for name in targets],
return_exceptions=True,
)
task = asyncio.create_task(_broadcast_all())
task.add_done_callback(_ws_tasks.discard)
_ws_tasks.add(task)
@@ -401,10 +408,12 @@ _last_blacklist_cleanup_lock: asyncio.Lock | None = None
async def _handle_ws(request: web.Request) -> web.Response:
"""WebSocket endpoint for real-time state change notifications.
"""WebSocket endpoint for real-time state streaming.
On connect: sends current versions. On state change: broadcasts
updated subsystem versions. Clients disconnect to unsubscribe.
On connect: sends a full state snapshot of every subsystem
({type: snapshot, data: {subsystem: state, …}}). On state change:
broadcasts a data-carrying per-subsystem delta (versions or tick).
Clients disconnect to unsubscribe.
Authentication: JWT access token passed via:
1. WebSocket subprotocol header — the bundled client sends the raw JWT
@@ -452,7 +461,8 @@ async def _handle_ws(request: web.Request) -> web.Response:
await ws.prepare(request)
_ws_subscribers.add(ws)
await ws.send_json({"type": "init", "versions": state_store.get_versions()})
snapshot = state_store.get_snapshot()
await ws.send_json({"type": "snapshot", "data": snapshot})
try:
async for msg in ws:
@@ -466,35 +476,59 @@ async def _handle_ws(request: web.Request) -> web.Response:
return ws
async def broadcast_versions() -> None:
"""Broadcast updated subsystem versions to all WebSocket clients."""
updated = state_store.get_updated_versions()
if not updated or not _ws_subscribers:
async def _send_all(message: str) -> None:
"""Send a message string to all WS subscribers, removing dead ones."""
dead: set[web.WebSocketResponse] = set()
for ws in _ws_subscribers:
try:
await ws.send_str(message)
except Exception:
dead.add(ws)
_ws_subscribers.difference_update(dead)
if dead:
logger.warning("Removed %d dead WS subscribers", len(dead))
async def broadcast_versions(subsystem: str) -> None:
"""Send {type: versions} + full subsystem data to WS clients.
NOTE: Does NOT call state_store.bump(). Callers who need version bumps
(e.g. refresh_state) call bump themselves. poll_loop bumps before calling.
No legacy `updated` field is emitted (no backward compat) — version
counters still advance but are not sent over the wire.
"""
data = state_store.get(subsystem)
if data is None:
# Collector failed during the triggering populate/refresh — populate()
# cleared this subsystem to None (state.py). Skip the broadcast:
# a null payload would overwrite good client data. The next successful
# poll or mutation broadcasts the real value.
return
data = json.dumps({"type": "versions", "updated": updated})
dead: set[web.WebSocketResponse] = set()
for ws in _ws_subscribers:
try:
await ws.send_str(data)
except Exception:
dead.add(ws)
_ws_subscribers.difference_update(dead)
if dead:
logger.warning("Removed %d dead WS subscribers", len(dead))
message = json.dumps(
{
"type": "versions",
"subsystem": subsystem,
"data": data,
}
)
await _send_all(message)
async def broadcast_tick(subsystems: list[str]) -> None:
"""Broadcast a lightweight tick to WS clients without version payload."""
data = json.dumps({"type": "tick", "subsystems": subsystems})
dead: set[web.WebSocketResponse] = set()
for ws in _ws_subscribers:
try:
await ws.send_str(data)
except Exception:
dead.add(ws)
_ws_subscribers.difference_update(dead)
if dead:
logger.warning("Removed %d dead WS subscribers", len(dead))
async def broadcast_tick(subsystem: str) -> None:
"""Send {type: tick} with the changed subsystem data.
No bump — tick is volatile-only; the version counter only bumps on
structural changes.
"""
data = state_store.get(subsystem)
message = json.dumps(
{
"type": "tick",
"subsystem": subsystem,
"data": data,
}
)
await _send_all(message)
async def _poll_loop(subsystem: str, interval: int) -> None:
@@ -512,9 +546,9 @@ async def _poll_loop(subsystem: str, interval: int) -> None:
structural, volatile = state_store.poll(subsystem)
if structural:
state_store.bump(subsystem)
await broadcast_versions()
await broadcast_versions(subsystem) # now per-subsystem, carries data
elif volatile:
await broadcast_tick([subsystem])
await broadcast_tick(subsystem) # now per-subsystem, carries data
# Periodic blacklist cleanup — coordinated across all poll loops
async with _last_blacklist_cleanup_lock:
now = time.time()
@@ -565,11 +599,24 @@ async def refresh_status(_request: web.Request) -> web.Response:
body = await _request.json()
except (json.JSONDecodeError, ValueError):
body = None
subsystems = None
if body and "subsystems" in body:
subsystems = body["subsystems"]
subsystems = body.get("subsystems") if body else None
state_store.populate(subsystems)
return ok({name: state_store.get(name) for name in state_store.SUBSYSTEMS})
targets = subsystems or state_store.SUBSYSTEMS
snapshot = {name: state_store.get(name) for name in targets}
# Broadcast to all WS clients (fire-and-forget, gather for parallelism).
# Deliberately no version bump — versions advance on structural poll
# diffs and on refresh_state() only.
async def _broadcast_all():
await asyncio.gather(
*[broadcast_versions(name) for name in targets],
return_exceptions=True,
)
task = asyncio.create_task(_broadcast_all())
task.add_done_callback(_ws_tasks.discard)
_ws_tasks.add(task)
return ok(snapshot)
async def _catch_all(request: web.Request) -> web.Response: