feat: add webhook retry queue

This commit is contained in:
2026-06-09 11:32:34 +08:00
parent 81f170924c
commit 8f516fdc01
12 changed files with 940 additions and 74 deletions

View File

@@ -7,7 +7,14 @@ from datetime import datetime, timezone
from pathlib import Path
from cold_display_guard.cases import CaseStore
from cold_display_guard.main import case_sink_path, deliver_runtime_webhooks, persist_case_updates, restore_runtime_state
from cold_display_guard.main import (
case_sink_path,
deliver_runtime_webhooks,
persist_case_updates,
restore_runtime_state,
webhook_retry_sink_path,
)
from cold_display_guard.webhooks import load_retry_snapshots
UTC = timezone.utc
@@ -22,6 +29,14 @@ class RuntimeRestoreTests(unittest.TestCase):
self.assertEqual(path, (root / "logs" / "cases.jsonl").resolve())
def test_webhook_retry_sink_path_uses_default_logs_location(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
path = webhook_retry_sink_path(root, {})
self.assertEqual(path, (root / "logs" / "webhook_retry.jsonl").resolve())
def test_persist_case_updates_writes_case_snapshots(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
path = Path(tmpdir) / "cases.jsonl"
@@ -99,6 +114,61 @@ class RuntimeRestoreTests(unittest.TestCase):
self.assertEqual(deliveries[0][1]["kind"], "batch_event")
self.assertEqual(deliveries[1][1]["kind"], "case_event")
def test_deliver_runtime_webhooks_enqueues_failure_and_drains_due_retry(self) -> None:
attempts = {"count": 0}
def flaky_post(url: str, payload: dict[str, object], timeout: tuple[float, float]) -> tuple[int, str]:
attempts["count"] += 1
if attempts["count"] == 1:
return 503, "down"
return 200, "ok"
with tempfile.TemporaryDirectory() as tmpdir:
audit_path = Path(tmpdir) / "webhook_delivery.jsonl"
retry_path = Path(tmpdir) / "webhook_retry.jsonl"
config = {
"webhooks": {
"enabled": True,
"event_url": "https://example.com/events",
"retry_max_attempts": 3,
"retry_backoff_seconds": 30,
}
}
deliver_runtime_webhooks(
[
{
"event": "time_alarm",
"ts": datetime(2026, 6, 9, 9, 0, tzinfo=UTC).isoformat(),
"batch_id": "batch_000001",
"camera_id": "cam_01",
"zone_id": "1",
"zone_label": "区域 1",
"severity": "alarm",
"state": "alerted",
}
],
[],
config,
audit_path,
retry_path=retry_path,
http_post=flaky_post,
now=datetime(2026, 6, 9, 9, 0, tzinfo=UTC),
)
deliver_runtime_webhooks(
[],
[],
config,
audit_path,
retry_path=retry_path,
http_post=flaky_post,
now=datetime(2026, 6, 9, 9, 1, tzinfo=UTC),
)
retries = load_retry_snapshots(retry_path)
self.assertEqual(attempts["count"], 2)
self.assertEqual(retries[-1]["status"], "delivered")
self.assertEqual(retries[-1]["attempt_count"], 2)
def test_restore_runtime_state_uses_stable_occupancy_when_raw_metrics_flicker(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
diagnostics_path = Path(tmpdir) / "runtime_diagnostics.jsonl"