Add state management, WebSocket polling, html.js templating, and refactor pages

- lib/state.py: per-subsystem collectors with versioned state store
- daemon/server.py: state refresh on request, batch routing updates
- webui/static/hoover/html.js: new html tag template helper via htm.js
- webui/static/hoover/websocket.js: real-time state change notifications
- webui/static/hoover/vdom.js: VDOM improvements for keyed diff
- All frontend pages refactored to use html templates
- Add tests for state management and polling
- Update docs and AGENTS.md

tests/test_handler_acme.py

@@ -122,7 +122,7 @@ class TestCheckNginxRunning:
             patch("subprocess.run", return_value=mock_result),
             patch.object(Path, "is_file", return_value=False),
         ):
-            passed, msg = _check_nginx_running()
+            passed, _ = _check_nginx_running()
             assert passed is False
 
     def test_via_pid_file(self):
@@ -131,27 +131,27 @@ class TestCheckNginxRunning:
         def run_side_effect(cmd, **kwargs):
             raise FileNotFoundError()
 
-        pid_file = Path("/var/run/nginx.pid")
-        with patch("subprocess.run", side_effect=run_side_effect):
-            with patch.object(Path, "is_file") as mock_is_file:
-                with patch.object(Path, "read_text", return_value="1234\n"):
+        with (
+            patch("subprocess.run", side_effect=run_side_effect),
+            patch.object(Path, "read_text", return_value="1234\n"),
+        ):
 
-                    def fake_is_file(self):
-                        if self == Path("/var/run/nginx.pid"):
-                            return True
-                        if str(self) == "/proc/1234/status":
-                            return True
-                        return Path(self).is_file()
+            def fake_is_file(self):
+                if self == Path("/var/run/nginx.pid"):
+                    return True
+                if str(self) == "/proc/1234/status":
+                    return True
+                return Path(self).is_file()
 
-                    with patch.object(Path, "is_file", fake_is_file):
-                        passed, msg = _check_nginx_running()
-                        assert passed is True
+            with patch.object(Path, "is_file", fake_is_file):
+                passed, _ = _check_nginx_running()
+                assert passed is True
 
 
 class TestCheckNginxConfig:
     def test_valid_config(self):
         with patch("lib.nginx.test_config", return_value=(True, "test passed")):
-            passed, msg = _check_nginx_config()
+            passed, _ = _check_nginx_config()
             assert passed is True
 
     def test_invalid_config(self):
@@ -175,7 +175,7 @@ class TestCheckFirewallPort80:
         with (
             patch("lib.common.run_proc", side_effect=proc_side_effect),
         ):
-            passed, msg = _check_firewall_port_80()
+            passed, _ = _check_firewall_port_80()
             assert passed is True
 
     def test_blocked_by_firewall(self):
@@ -213,7 +213,7 @@ class TestCheckAcmeHomeWritable:
         acme_dir = tmp_path / "acme"
         acme_dir.mkdir()
         with patch("daemon.handlers.acme._ACME_HOME", acme_dir):
-            passed, msg = _check_acme_home_writable()
+            passed, _ = _check_acme_home_writable()
             assert passed is True
 
 
@@ -233,7 +233,7 @@ class TestCheckAcmeHomeWritable_Permissions:
         acme_dir.chmod(0o444)
         try:
             with patch("daemon.handlers.acme._ACME_HOME", acme_dir):
-                passed, msg = _check_acme_home_writable()
+                passed, _ = _check_acme_home_writable()
                 assert passed is False
         finally:
             acme_dir.chmod(0o755)
@@ -254,7 +254,7 @@ class TestCheckOpensslAvailable:
 
     def test_not_found(self):
         with patch("shutil.which", return_value=None):
-            passed, msg = _check_openssl_available()
+            passed, _ = _check_openssl_available()
             assert passed is False
 
 
@@ -268,7 +268,7 @@ class TestCheckPort80Listening:
         mock_sock.__exit__ = MagicMock(return_value=False)
 
         with patch("socket.socket", return_value=mock_sock):
-            passed, msg = _check_port_80_listening()
+            passed, _ = _check_port_80_listening()
             assert passed is True
 
     def test_not_listening(self):
@@ -280,7 +280,7 @@ class TestCheckPort80Listening:
         mock_sock.__exit__ = MagicMock(return_value=False)
 
         with patch("socket.socket", return_value=mock_sock):
-            passed, msg = _check_port_80_listening()
+            passed, _ = _check_port_80_listening()
             assert passed is False
 
 
@@ -301,7 +301,7 @@ class TestCheckAcmeAccount:
             ),
             patch("subprocess.run", return_value=MagicMock(returncode=0, stdout="ok")),
         ):
-            passed, msg = _check_acme_account()
+            passed, _ = _check_acme_account()
             assert passed is True
 
     def test_via_account_conf(self, tmp_path):
@@ -320,7 +320,7 @@ class TestCheckAcmeAccount:
             patch("daemon.handlers.acme._ACME_HOME", acme_dir),
             patch("subprocess.run", side_effect=run_side_effect),
         ):
-            passed, msg = _check_acme_account()
+            passed, _ = _check_acme_account()
             assert passed is True
 
     def test_not_configured(self, tmp_path):
@@ -336,7 +336,7 @@ class TestCheckAcmeAccount:
             patch("daemon.handlers.acme._ACME_HOME", acme_dir),
             patch("subprocess.run", side_effect=run_side_effect),
         ):
-            passed, msg = _check_acme_account()
+            passed, _ = _check_acme_account()
             assert passed is False
 
 
@@ -351,7 +351,7 @@ class TestCheckDnsPublic:
             patch("socket.gethostbyname", return_value="192.168.1.1"),
             patch("subprocess.run", return_value=mock_result),
         ):
-            passed, msg = _check_dns_public("example.com")
+            passed, _ = _check_dns_public("example.com")
             assert passed is True
 
     def test_does_not_resolve(self):
@@ -362,7 +362,7 @@ class TestCheckDnsPublic:
             patch("socket.gethostbyname", return_value="192.168.1.1"),
             patch("subprocess.run", return_value=mock_result),
         ):
-            passed, msg = _check_dns_public("example.com")
+            passed, _ = _check_dns_public("example.com")
             assert passed is False
 
 
@@ -502,7 +502,7 @@ class TestValidate:
             result = _validate("example.com")
 
         assert result["ready"] is False
-        nginx_check = [c for c in result["checks"] if c["name"] == "nginx_running"][0]
+        nginx_check = next(c for c in result["checks"] if c["name"] == "nginx_running")
         assert nginx_check["passed"] is False
         assert nginx_check["blocking"] is True
 
@@ -566,7 +566,7 @@ class TestValidate:
             result = _validate("example.com")
 
         assert result["ready"] is True
-        dns_pub = [c for c in result["checks"] if c["name"] == "dns_public"][0]
+        dns_pub = next(c for c in result["checks"] if c["name"] == "dns_public")
         assert dns_pub["passed"] is False
         assert dns_pub["blocking"] is False
 

tests/test_polling.py

@@ -0,0 +1,64 @@
+"""Tests for daemon/server.py polling functions."""
+
+import asyncio
+import json
+from unittest.mock import AsyncMock, patch
+
+
+class TestPollIntervals:
+    def test_default_intervals_loaded(self):
+        from daemon.server import _POLL_INTERVALS
+
+        assert "firewall" in _POLL_INTERVALS
+        assert _POLL_INTERVALS["firewall"] == 30
+        assert _POLL_INTERVALS["wireguard"] == 10
+        assert _POLL_INTERVALS["dnsmasq"] == 10
+        assert _POLL_INTERVALS["networkd"] == 10
+
+    def test_env_override(self):
+        """VACUUM_WALL_POLL_INTERVALS env var can override values."""
+        import importlib
+
+        with patch.dict(
+            "os.environ", {"VACUUM_WALL_POLL_INTERVALS": "firewall:60,wireguard:5"}
+        ):
+            import daemon.server
+
+            importlib.reload(daemon.server)
+            assert daemon.server._POLL_INTERVALS["firewall"] == 60
+            assert daemon.server._POLL_INTERVALS["wireguard"] == 5
+            assert daemon.server._POLL_INTERVALS["dnsmasq"] == 10
+        importlib.reload(daemon.server)
+
+
+class TestBroadcastTick:
+    def test_sends_tick_message(self):
+        from daemon.server import _ws_subscribers, broadcast_tick
+
+        mock_ws = AsyncMock()
+        mock_ws.send_str = AsyncMock()
+        _ws_subscribers.add(mock_ws)
+        try:
+            asyncio.run(broadcast_tick(["firewall", "wireguard"]))
+            mock_ws.send_str.assert_called_once()
+            sent = json.loads(mock_ws.send_str.call_args[0][0])
+            assert sent["type"] == "tick"
+            assert sent["subsystems"] == ["firewall", "wireguard"]
+        finally:
+            _ws_subscribers.discard(mock_ws)
+
+    def test_prunes_dead_subscribers(self):
+        from daemon.server import _ws_subscribers, broadcast_tick
+
+        mock_ws = AsyncMock()
+        mock_ws.send_str = AsyncMock(side_effect=Exception("broken"))
+        _ws_subscribers.add(mock_ws)
+        asyncio.run(broadcast_tick(["firewall"]))
+        assert mock_ws not in _ws_subscribers
+
+
+class TestPollTasks:
+    def test_poll_tasks_is_a_set(self):
+        from daemon.server import _poll_tasks
+
+        assert isinstance(_poll_tasks, set)

tests/test_state.py

@@ -175,3 +175,236 @@ class TestStateVersions:
         updated = s.get_updated_versions()
         assert updated["firewall"] == 2
         assert updated["dnsmasq"] == 1
+
+
+class TestStripVolatile:
+    def test_list_of_dicts_strips_nested_keys(self):
+        """Wireguard-style: status.peers[].transfer_received gets stripped."""
+        from lib.state import _strip_volatile
+
+        data = {
+            "status": {
+                "peers": [
+                    {
+                        "transfer_received": "100B",
+                        "transfer_sent": "200B",
+                        "latest_handshake": "ago",
+                        "public_key": "abc",
+                    },
+                    {
+                        "transfer_received": "300B",
+                        "transfer_sent": "400B",
+                        "latest_handshake": "now",
+                        "public_key": "def",
+                    },
+                ]
+            }
+        }
+        vol = frozenset(
+            {"status.peers[].transfer_received", "status.peers[].latest_handshake"}
+        )
+        stripped = _strip_volatile(data, vol)
+        assert stripped["status"]["peers"][0]["transfer_received"] is None
+        assert stripped["status"]["peers"][0]["latest_handshake"] is None
+        assert stripped["status"]["peers"][0]["transfer_sent"] == "200B"
+        assert stripped["status"]["peers"][0]["public_key"] == "abc"
+        assert stripped["status"]["peers"][1]["transfer_received"] is None
+
+    def test_scalar_path_strips(self):
+        """Scalar paths get zeroed to None."""
+        from lib.state import _strip_volatile
+
+        data = {"a": {"b": 1, "c": 2}}
+        vol = frozenset({"a.b"})
+        stripped = _strip_volatile(data, vol)
+        assert stripped["a"]["b"] is None
+        assert stripped["a"]["c"] == 2
+
+    def test_empty_volatile_returns_copy(self):
+        from lib.state import _strip_volatile
+
+        data = {"x": 1}
+        stripped = _strip_volatile(data, frozenset())
+        assert stripped == data
+        assert stripped is not data
+
+    def test_firewall_volatile_strips_ips(self):
+        """Firewall-style: interfaces[].ips gets stripped."""
+        from lib.state import _strip_volatile
+
+        data = {
+            "interfaces": [
+                {
+                    "name": "eth0",
+                    "ips": ["10.0.0.1/24"],
+                    "ipv6": ["fe80::1"],
+                    "zone": "internal",
+                },
+            ]
+        }
+        vol = frozenset({"interfaces[].ips", "interfaces[].ipv6"})
+        stripped = _strip_volatile(data, vol)
+        assert stripped["interfaces"][0]["ips"] is None
+        assert stripped["interfaces"][0]["ipv6"] is None
+        assert stripped["interfaces"][0]["name"] == "eth0"
+        assert stripped["interfaces"][0]["zone"] == "internal"
+
+    def test_networkd_volatile_strips_addresses(self):
+        """Networkd-style: interfaces dict keyed by name, addresses stripped via fallback."""
+        from lib.state import _strip_volatile
+
+        data = {
+            "interfaces": {
+                "eth0": {
+                    "addresses": ["10.0.0.1/24", "fe80::1"],
+                    "state": "routable",
+                    "type": "ether",
+                },
+                "lo": {
+                    "addresses": ["127.0.0.1/8"],
+                    "state": "degraded",
+                    "type": "loopback",
+                },
+            }
+        }
+        vol = frozenset({"interfaces[].addresses"})
+        stripped = _strip_volatile(data, vol)
+        assert stripped["interfaces"]["eth0"]["addresses"] is None
+        assert stripped["interfaces"]["eth0"]["state"] == "routable"
+        assert stripped["interfaces"]["eth0"]["type"] == "ether"
+        assert stripped["interfaces"]["lo"]["addresses"] is None
+        assert stripped["interfaces"]["lo"]["state"] == "degraded"
+
+
+class TestDiffLayers:
+    def test_no_change(self):
+        """Identical data (minus timestamp) returns (False, False)."""
+        from lib.state import _diff_layers
+
+        old = {"zones": {"public": {}}, "timestamp": "t1"}
+        new = {"zones": {"public": {}}, "timestamp": "t2"}
+        structural, volatile = _diff_layers(old, new, frozenset())
+        assert structural is False
+        assert volatile is False
+
+    def test_structural_only(self):
+        """Non-volatile change detected as structural."""
+        from lib.state import _diff_layers
+
+        old = {
+            "zones": {"public": {}},
+            "interfaces": [{"ips": None}],
+            "timestamp": "t1",
+        }
+        new = {
+            "zones": {"internal": {}},
+            "interfaces": [{"ips": None}],
+            "timestamp": "t2",
+        }
+        vol = frozenset({"interfaces[].ips"})
+        structural, volatile = _diff_layers(old, new, vol)
+        assert structural is True
+        assert volatile is False
+
+    def test_volatile_only(self):
+        """Only volatile fields changed returns (False, True)."""
+        from lib.state import _diff_layers
+
+        old = {
+            "status": {"peers": [{"transfer_received": "100B", "public_key": "abc"}]},
+            "timestamp": "t1",
+        }
+        new = {
+            "status": {"peers": [{"transfer_received": "200B", "public_key": "abc"}]},
+            "timestamp": "t2",
+        }
+        vol = frozenset({"status.peers[].transfer_received"})
+        structural, volatile = _diff_layers(old, new, vol)
+        assert structural is False
+        assert volatile is True
+
+    def test_neither(self):
+        """No change at all returns (False, False)."""
+        from lib.state import _diff_layers
+
+        old = {
+            "status": {"peers": [{"transfer_received": "100B", "public_key": "abc"}]},
+            "timestamp": "t1",
+        }
+        new = {
+            "status": {"peers": [{"transfer_received": "100B", "public_key": "abc"}]},
+            "timestamp": "t2",
+        }
+        vol = frozenset({"status.peers[].transfer_received"})
+        structural, volatile = _diff_layers(old, new, vol)
+        assert structural is False
+        assert volatile is False
+
+    def test_old_none_returns_both_true(self):
+        """When old is None (first poll), both layers report True."""
+        from lib.state import _diff_layers
+
+        new = {"zones": {}}
+        structural, volatile = _diff_layers(None, new, frozenset())
+        assert structural is True
+        assert volatile is True
+
+
+class TestPoll:
+    def test_poll_no_change(self):
+        """poll() returns (False, False) when collector returns same data."""
+        import uuid
+
+        from lib.state import _COLLECTORS, State
+
+        name = f"test_{uuid.uuid4().hex}"
+        s = State()
+        data = {"value": 1, "timestamp": "t1"}
+        _COLLECTORS[name] = lambda: data
+        s.set(name, data)
+        structural, volatile = s.poll(name)
+        assert structural is False
+        assert volatile is False
+        del _COLLECTORS[name]
+
+    def test_poll_detects_structural(self):
+        """poll() detects structural changes via collector."""
+        import uuid
+
+        from lib.state import _COLLECTORS, _VOLATILE, State
+
+        name = f"test_{uuid.uuid4().hex}"
+        s = State()
+        old = {
+            "zones": {"public": {}},
+            "interfaces": [{"ips": None}],
+            "timestamp": "t1",
+        }
+        new = {
+            "zones": {"internal": {}},
+            "interfaces": [{"ips": None}],
+            "timestamp": "t2",
+        }
+        _COLLECTORS[name] = lambda: new
+        _VOLATILE[name] = frozenset({"interfaces[].ips"})
+        s.set(name, old)
+        structural, _volatile = s.poll(name)
+        assert structural is True
+        del _COLLECTORS[name]
+        del _VOLATILE[name]
+
+    def test_poll_failure_returns_no_broadcast(self):
+        """poll() returns (False, False) and preserves existing data on collector failure."""
+        import uuid
+
+        from lib.state import _COLLECTORS, State
+
+        name = f"test_{uuid.uuid4().hex}"
+        s = State()
+        s.set(name, {"value": 1})
+        _COLLECTORS[name] = lambda: (_ for _ in ()).throw(RuntimeError("fail"))
+        structural, volatile = s.poll(name)
+        assert structural is False
+        assert volatile is False
+        assert s.get(name) == {"value": 1}
+        del _COLLECTORS[name]