feat: add webhook retry queue

tests/test_manage_api.py

@@ -7,6 +7,7 @@ import threading
 import unittest
 from http.server import ThreadingHTTPServer
 from pathlib import Path
+from unittest import mock
 
 from cold_display_guard.config import load_config_document, merge_calibration, save_config_document
 from cold_display_guard.manage_api import ManageContext, build_summary, config_payload, create_handler
@@ -390,11 +391,13 @@ class ManageApiTests(unittest.TestCase):
                 config_path,
                 {
                     "case_sink": {"path": "logs/cases.jsonl"},
+                    "webhook_retry_sink": {"path": "logs/webhook_retry.jsonl"},
                     "webhooks": {
                         "enabled": True,
                         "event_url": "https://example.com/events",
                         "case_url": "https://example.com/cases",
                         "callback_token": "secret",
+                        "retry_max_attempts": 4,
                     },
                 },
             )
@@ -402,7 +405,9 @@ class ManageApiTests(unittest.TestCase):
             payload = config_payload(ManageContext(config_path=config_path, project_root=root))
 
         self.assertEqual(payload["case_sink"]["path"], str((root / "logs" / "cases.jsonl").resolve()))
+        self.assertEqual(payload["webhook_retry_sink"]["path"], str((root / "logs" / "webhook_retry.jsonl").resolve()))
         self.assertTrue(payload["webhooks"]["enabled"])
+        self.assertEqual(payload["webhooks"]["retry_max_attempts"], 4)
         self.assertNotIn("callback_token", payload["webhooks"])
 
     def test_cases_endpoint_returns_latest_snapshots(self) -> None:
@@ -560,6 +565,144 @@ class ManageApiTests(unittest.TestCase):
         self.assertEqual(lines[-1]["handled_source"], "manual")
         self.assertEqual(lines[-1]["payload"]["note"], "checked")
 
+    def test_manual_handle_endpoint_enqueues_failed_case_webhook_for_retry(self) -> None:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            config_path = root / "config" / "local.toml"
+            save_config_document(
+                config_path,
+                {
+                    "case_sink": {"path": "logs/cases.jsonl"},
+                    "webhooks": {
+                        "enabled": True,
+                        "case_url": "https://example.com/cases",
+                        "retry_max_attempts": 3,
+                        "retry_backoff_seconds": 30,
+                    },
+                    "layout": {"zone_ids": ["1"]},
+                },
+            )
+            cases_path = root / "logs" / "cases.jsonl"
+            retry_path = root / "logs" / "webhook_retry.jsonl"
+            cases_path.parent.mkdir()
+            cases_path.write_text(
+                json.dumps(
+                    {
+                        "case_id": "case_batch_000001",
+                        "batch_id": "batch_000001",
+                        "camera_id": "cam_01",
+                        "zone_id": "1",
+                        "zone_label": "区域 1",
+                        "case_type": "time_alarm",
+                        "case_status": "open",
+                        "source_event": "time_alarm",
+                        "created_at": "2026-06-09T09:00:00+08:00",
+                        "updated_at": "2026-06-09T09:00:00+08:00",
+                        "payload": {},
+                    }
+                ),
+                encoding="utf-8",
+            )
+            ctx = ManageContext(config_path=config_path, project_root=root)
+            server, thread = self._serve_once(ctx)
+            try:
+                with mock.patch("cold_display_guard.webhooks.post_json", side_effect=OSError("network down")):
+                    status, payload = self._request(
+                        server,
+                        "POST",
+                        "/api/manage/cases/case_batch_000001/handle",
+                        body={"handled_by": "alice"},
+                    )
+            finally:
+                self._stop_server(server, thread)
+
+            retries = [json.loads(line) for line in retry_path.read_text(encoding="utf-8").splitlines()]
+
+        self.assertEqual(status, 200)
+        self.assertEqual(payload["case_status"], "handled")
+        self.assertEqual(retries[-1]["status"], "pending")
+        self.assertEqual(retries[-1]["target"], "case_event")
+        self.assertEqual(retries[-1]["attempt_count"], 1)
+
+    def test_retry_queue_endpoint_returns_pending_items(self) -> None:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            config_path = root / "config" / "local.toml"
+            save_config_document(config_path, {"layout": {"zone_ids": ["1"]}})
+            retry_path = root / "logs" / "webhook_retry.jsonl"
+            retry_path.parent.mkdir()
+            retry_path.write_text(
+                json.dumps(
+                    {
+                        "retry_id": "retry_000001",
+                        "target": "case_event",
+                        "url": "https://example.com/cases",
+                        "status": "pending",
+                        "attempt_count": 1,
+                        "payload": {"kind": "case_event"},
+                        "created_at": "2026-06-09T09:00:00+08:00",
+                        "updated_at": "2026-06-09T09:00:00+08:00",
+                        "next_attempt_at": "2026-06-09T09:01:00+08:00",
+                    }
+                ),
+                encoding="utf-8",
+            )
+            ctx = ManageContext(config_path=config_path, project_root=root)
+            server, thread = self._serve_once(ctx)
+            try:
+                status, payload = self._request(server, "GET", "/api/manage/webhooks/retries?status=pending")
+            finally:
+                self._stop_server(server, thread)
+
+        self.assertEqual(status, 200)
+        self.assertEqual(payload["items"][0]["retry_id"], "retry_000001")
+        self.assertEqual(payload["items"][0]["status"], "pending")
+
+    def test_retry_drain_endpoint_retries_pending_item(self) -> None:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            config_path = root / "config" / "local.toml"
+            save_config_document(
+                config_path,
+                {
+                    "webhooks": {"enabled": True, "retry_max_attempts": 3, "retry_backoff_seconds": 30},
+                    "layout": {"zone_ids": ["1"]},
+                },
+            )
+            retry_path = root / "logs" / "webhook_retry.jsonl"
+            retry_path.parent.mkdir()
+            retry_path.write_text(
+                json.dumps(
+                    {
+                        "retry_id": "retry_000001",
+                        "target": "case_event",
+                        "url": "https://example.com/cases",
+                        "status": "pending",
+                        "attempt_count": 1,
+                        "payload": {"kind": "case_event", "case_id": "case_batch_000001"},
+                        "created_at": "2026-06-09T09:00:00+08:00",
+                        "updated_at": "2026-06-09T09:00:00+08:00",
+                        "next_attempt_at": "2026-06-09T09:01:00+08:00",
+                    }
+                ),
+                encoding="utf-8",
+            )
+            ctx = ManageContext(config_path=config_path, project_root=root)
+            server, thread = self._serve_once(ctx)
+            try:
+                with mock.patch("cold_display_guard.webhooks.post_json", return_value=(200, "ok")):
+                    status, payload = self._request(server, "POST", "/api/manage/webhooks/retries/drain", body={})
+            finally:
+                self._stop_server(server, thread)
+
+            lines = [json.loads(line) for line in retry_path.read_text(encoding="utf-8").splitlines()]
+
+        self.assertEqual(status, 200)
+        self.assertEqual(payload["retried_count"], 1)
+        self.assertEqual(payload["delivered_count"], 1)
+        self.assertEqual(lines[-1]["status"], "delivered")
+        self.assertEqual(lines[-1]["attempt_count"], 2)
+
     def test_callback_endpoint_requires_token_and_handles_case(self) -> None:
         with tempfile.TemporaryDirectory() as tmpdir:
             root = Path(tmpdir)