"""Tests for the gateway platform reconnection watcher.""" import asyncio import time from unittest.mock import AsyncMock, MagicMock, patch import pytest from gateway.config import GatewayConfig, Platform, PlatformConfig from gateway.platforms.base import BasePlatformAdapter, SendResult from gateway.run import GatewayRunner class StubAdapter(BasePlatformAdapter): """Adapter whose connect() result can be controlled.""" def __init__( self, *, platform=Platform.TELEGRAM, succeed=True, fatal_error=None, fatal_retryable=True, ): super().__init__(PlatformConfig(enabled=True, token="test"), platform) self._succeed = succeed self._fatal_error = fatal_error self._fatal_retryable = fatal_retryable # Records the is_reconnect value of every connect() call so tests can # assert that the watcher distinguishes reconnect from cold boot (#46621). self.connect_calls: list[bool] = [] async def connect(self, *, is_reconnect: bool = False): self.connect_calls.append(is_reconnect) if self._fatal_error: self._set_fatal_error("test_error", self._fatal_error, retryable=self._fatal_retryable) return False return self._succeed async def disconnect(self): return None async def send(self, chat_id, content, reply_to=None, metadata=None): return SendResult(success=True, message_id="1") async def send_typing(self, chat_id, metadata=None): return None async def get_chat_info(self, chat_id): return {"id": chat_id} def _make_runner(): """Create a minimal GatewayRunner via object.__new__ to skip __init__.""" runner = object.__new__(GatewayRunner) runner.config = GatewayConfig( platforms={Platform.TELEGRAM: PlatformConfig(enabled=True, token="test")} ) runner._running = True runner._shutdown_event = asyncio.Event() runner._exit_reason = None runner._exit_with_failure = False runner._exit_cleanly = False runner._failed_platforms = {} runner.adapters = {} runner.delivery_router = MagicMock() runner._running_agents = {} runner._pending_messages = {} runner._pending_approvals = {} runner._honcho_managers = {} runner._honcho_configs = {} runner._shutdown_all_gateway_honcho = lambda: None runner.session_store = MagicMock() return runner # --- Startup queueing --- class TestStartupPlatformIsolation: """Verify one blocked platform cannot prevent later platforms from starting.""" @pytest.mark.asyncio async def test_start_continues_after_platform_connect_timeout(self, tmp_path): """A timeout on Telegram should queue it and still connect Feishu.""" runner = _make_runner() runner.config = GatewayConfig( platforms={ Platform.TELEGRAM: PlatformConfig(enabled=True, token="test"), Platform.FEISHU: PlatformConfig(enabled=True, token="test"), }, sessions_dir=tmp_path, ) runner.hooks = MagicMock() runner.hooks.loaded_hooks = [] runner.hooks.emit = AsyncMock() runner._suspend_stuck_loop_sessions = MagicMock(return_value=0) runner._update_runtime_status = MagicMock() runner._update_platform_runtime_status = MagicMock() runner._sync_voice_mode_state_to_adapter = MagicMock() runner._send_update_notification = AsyncMock(return_value=True) runner._send_restart_notification = AsyncMock() adapters = { Platform.TELEGRAM: StubAdapter(platform=Platform.TELEGRAM), Platform.FEISHU: StubAdapter(platform=Platform.FEISHU), } runner._create_adapter = MagicMock( side_effect=lambda platform, _config: adapters[platform] ) runner._connect_adapter_with_timeout = AsyncMock( side_effect=[ TimeoutError("telegram connect timed out after 30s"), True, ] ) def fake_create_task(coro): coro.close() return MagicMock() with patch("gateway.status.write_runtime_status"): with patch("hermes_cli.plugins.discover_plugins"): with patch("hermes_cli.config.load_config", return_value={}): with patch("agent.shell_hooks.register_from_config"): with patch( "tools.process_registry.process_registry.recover_from_checkpoint", return_value=0, ): with patch( "gateway.channel_directory.build_channel_directory", new=AsyncMock(return_value={"platforms": {}}), ): with patch("gateway.run.asyncio.create_task", side_effect=fake_create_task): assert await runner.start() is True assert Platform.TELEGRAM in runner._failed_platforms assert Platform.FEISHU in runner.adapters assert Platform.TELEGRAM not in runner.adapters assert runner._create_adapter.call_count == 2 def test_default_connect_timeout_allows_telegram_polling_readiness( self, monkeypatch ): """Telegram gets a larger default; other platforms stay isolated at 30s.""" runner = _make_runner() monkeypatch.delenv("HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT", raising=False) assert runner._platform_connect_timeout_secs(Platform.TELEGRAM) == 180 assert runner._platform_connect_timeout_secs(Platform.FEISHU) == 30 def test_explicit_connect_timeout_still_applies_to_every_platform( self, monkeypatch ): runner = _make_runner() monkeypatch.setenv("HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT", "90") assert runner._platform_connect_timeout_secs(Platform.TELEGRAM) == 90 assert runner._platform_connect_timeout_secs(Platform.FEISHU) == 90 @pytest.mark.asyncio async def test_connect_adapter_timeout_raises_retryable_exception(self, monkeypatch): """The timeout helper turns a hanging connect into a caught startup error.""" runner = _make_runner() adapter = StubAdapter() async def hang(*, is_reconnect: bool = False): await asyncio.sleep(60) return True adapter.connect = hang monkeypatch.setenv("HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT", "0.001") with pytest.raises(TimeoutError, match="telegram connect timed out"): await runner._connect_adapter_with_timeout(adapter, Platform.TELEGRAM) class TestStartupFailureQueuing: """Verify that failed platforms are queued during startup.""" def test_failed_platform_queued_on_connect_failure(self): """When adapter.connect() returns False without fatal error, queue for retry.""" runner = _make_runner() platform_config = PlatformConfig(enabled=True, token="test") runner._failed_platforms[Platform.TELEGRAM] = { "config": platform_config, "attempts": 1, "next_retry": time.monotonic() + 30, } assert Platform.TELEGRAM in runner._failed_platforms assert runner._failed_platforms[Platform.TELEGRAM]["attempts"] == 1 def test_failed_platform_not_queued_for_nonretryable(self): """Non-retryable errors should not be in the retry queue.""" runner = _make_runner() # Simulate: adapter had a non-retryable error, wasn't queued assert Platform.TELEGRAM not in runner._failed_platforms # --- Reconnect watcher --- class TestPlatformReconnectWatcher: """Test the _platform_reconnect_watcher background task.""" @pytest.mark.asyncio async def test_reconnect_succeeds_on_retry(self): """Watcher should reconnect a failed platform when connect() succeeds.""" runner = _make_runner() runner._sync_voice_mode_state_to_adapter = MagicMock() platform_config = PlatformConfig(enabled=True, token="test") runner._failed_platforms[Platform.TELEGRAM] = { "config": platform_config, "attempts": 1, "next_retry": time.monotonic() - 1, # Already past retry time } succeed_adapter = StubAdapter(succeed=True) real_sleep = asyncio.sleep with patch.object(runner, "_create_adapter", return_value=succeed_adapter): with patch("gateway.run.build_channel_directory", create=True): # Run one iteration of the watcher then stop async def run_one_iteration(): runner._running = True # Patch the sleep to exit after first check call_count = 0 async def fake_sleep(n): nonlocal call_count call_count += 1 if call_count > 1: runner._running = False await real_sleep(0) with patch("asyncio.sleep", side_effect=fake_sleep): await runner._platform_reconnect_watcher() await run_one_iteration() assert Platform.TELEGRAM not in runner._failed_platforms assert Platform.TELEGRAM in runner.adapters @pytest.mark.asyncio async def test_reconnect_passes_is_reconnect_true(self): """The watcher must connect with is_reconnect=True so adapters preserve their server-side update queue across an outage (#46621). Without this, bootstrap start_polling(drop_pending_updates=True) silently dropped every message queued while the bot was offline.""" runner = _make_runner() runner._sync_voice_mode_state_to_adapter = MagicMock() runner._failed_platforms[Platform.TELEGRAM] = { "config": PlatformConfig(enabled=True, token="test"), "attempts": 1, "next_retry": time.monotonic() - 1, } succeed_adapter = StubAdapter(succeed=True) real_sleep = asyncio.sleep with patch.object(runner, "_create_adapter", return_value=succeed_adapter): with patch("gateway.run.build_channel_directory", create=True): runner._running = True call_count = 0 async def fake_sleep(n): nonlocal call_count call_count += 1 if call_count > 1: runner._running = False await real_sleep(0) with patch("asyncio.sleep", side_effect=fake_sleep): await runner._platform_reconnect_watcher() assert succeed_adapter.connect_calls == [True], ( f"watcher must pass is_reconnect=True; got {succeed_adapter.connect_calls!r}" ) assert Platform.TELEGRAM in runner.adapters @pytest.mark.asyncio async def test_cold_connect_defaults_to_is_reconnect_false(self): """The cold-start connect path (_connect_adapter_with_timeout with no is_reconnect arg) must default to False so a first boot still drops any stale queue (#46621).""" runner = _make_runner() adapter = StubAdapter(succeed=True) success = await runner._connect_adapter_with_timeout(adapter, Platform.TELEGRAM) assert success is True assert adapter.connect_calls == [False], ( f"cold-start must default to is_reconnect=False; got {adapter.connect_calls!r}" ) @pytest.mark.asyncio async def test_reconnect_retries_resume_pending_for_platform(self): """A successful reconnect retries the startup auto-resume scoped to that platform. Regression: a platform offline at gateway startup had its restart-interrupted sessions skipped by the one-shot startup pass and never rescheduled, so the documented auto-resume silently dropped until the user sent a fresh message. The watcher now re-runs the platform-scoped auto-resume on reconnect. """ runner = _make_runner() runner._sync_voice_mode_state_to_adapter = MagicMock() runner._schedule_resume_pending_sessions = MagicMock(return_value=1) platform_config = PlatformConfig(enabled=True, token="test") runner._failed_platforms[Platform.TELEGRAM] = { "config": platform_config, "attempts": 1, "next_retry": time.monotonic() - 1, } succeed_adapter = StubAdapter(succeed=True) real_sleep = asyncio.sleep with patch.object(runner, "_create_adapter", return_value=succeed_adapter): with patch("gateway.run.build_channel_directory", create=True): async def run_one_iteration(): runner._running = True call_count = 0 async def fake_sleep(n): nonlocal call_count call_count += 1 if call_count > 1: runner._running = False await real_sleep(0) with patch("asyncio.sleep", side_effect=fake_sleep): await runner._platform_reconnect_watcher() await run_one_iteration() assert Platform.TELEGRAM in runner.adapters runner._schedule_resume_pending_sessions.assert_called_once_with( platform=Platform.TELEGRAM ) @pytest.mark.asyncio async def test_reconnect_nonretryable_removed_from_queue(self): """Non-retryable errors should remove the platform from the retry queue.""" runner = _make_runner() platform_config = PlatformConfig(enabled=True, token="test") runner._failed_platforms[Platform.TELEGRAM] = { "config": platform_config, "attempts": 1, "next_retry": time.monotonic() - 1, } fail_adapter = StubAdapter( succeed=False, fatal_error="bad token", fatal_retryable=False ) real_sleep = asyncio.sleep with patch.object(runner, "_create_adapter", return_value=fail_adapter): async def run_one_iteration(): runner._running = True call_count = 0 async def fake_sleep(n): nonlocal call_count call_count += 1 if call_count > 1: runner._running = False await real_sleep(0) with patch("asyncio.sleep", side_effect=fake_sleep): await runner._platform_reconnect_watcher() await run_one_iteration() assert Platform.TELEGRAM not in runner._failed_platforms assert Platform.TELEGRAM not in runner.adapters @pytest.mark.asyncio async def test_reconnect_retryable_stays_in_queue(self): """Retryable failures should remain in the queue with incremented attempts.""" runner = _make_runner() platform_config = PlatformConfig(enabled=True, token="test") runner._failed_platforms[Platform.TELEGRAM] = { "config": platform_config, "attempts": 1, "next_retry": time.monotonic() - 1, } fail_adapter = StubAdapter( succeed=False, fatal_error="DNS failure", fatal_retryable=True ) real_sleep = asyncio.sleep with patch.object(runner, "_create_adapter", return_value=fail_adapter): async def run_one_iteration(): runner._running = True call_count = 0 async def fake_sleep(n): nonlocal call_count call_count += 1 if call_count > 1: runner._running = False await real_sleep(0) with patch("asyncio.sleep", side_effect=fake_sleep): await runner._platform_reconnect_watcher() await run_one_iteration() assert Platform.TELEGRAM in runner._failed_platforms assert runner._failed_platforms[Platform.TELEGRAM]["attempts"] == 2 @pytest.mark.asyncio async def test_reconnect_never_auto_pauses_retryable_failures(self): """Retryable failures (network/DNS) must keep retrying indefinitely — the watcher must NOT auto-pause them. Auto-pausing a transiently-failed platform left bots silently dead after a DNS blip (#35284). The pause circuit breaker remains available for manual /platform pause only. """ runner = _make_runner() platform_config = PlatformConfig(enabled=True, token="test") # Far past the old circuit-breaker threshold (10): even after many # consecutive retryable failures the platform must stay unpaused. runner._failed_platforms[Platform.TELEGRAM] = { "config": platform_config, "attempts": 25, "next_retry": time.monotonic() - 1, } fail_adapter = StubAdapter( succeed=False, fatal_error="DNS failure", fatal_retryable=True ) real_sleep = asyncio.sleep with patch.object(runner, "_create_adapter", return_value=fail_adapter): async def run_one_iteration(): runner._running = True call_count = 0 async def fake_sleep(n): nonlocal call_count call_count += 1 if call_count > 1: runner._running = False await real_sleep(0) with patch("asyncio.sleep", side_effect=fake_sleep): await runner._platform_reconnect_watcher() await run_one_iteration() # Platform stays in queue and keeps retrying — never auto-paused. assert Platform.TELEGRAM in runner._failed_platforms info = runner._failed_platforms[Platform.TELEGRAM] assert info.get("paused") is not True assert "pause_reason" not in info assert info["attempts"] == 26 # next_retry is pushed out by the backoff (capped at 300s), not inf. assert info["next_retry"] != float("inf") assert info["next_retry"] > time.monotonic() @pytest.mark.asyncio async def test_reconnect_skips_paused_platforms(self): """A paused platform should not be retried by the watcher tick.""" runner = _make_runner() platform_config = PlatformConfig(enabled=True, token="test") runner._failed_platforms[Platform.TELEGRAM] = { "config": platform_config, "attempts": 10, "next_retry": time.monotonic() - 1, # would normally retry now "paused": True, "pause_reason": "paused via /platform pause", } real_sleep = asyncio.sleep with patch.object(runner, "_create_adapter") as mock_create: async def run_one_iteration(): runner._running = True call_count = 0 async def fake_sleep(n): nonlocal call_count call_count += 1 if call_count > 1: runner._running = False await real_sleep(0) with patch("asyncio.sleep", side_effect=fake_sleep): await runner._platform_reconnect_watcher() await run_one_iteration() # Paused platform stays queued and was never touched assert Platform.TELEGRAM in runner._failed_platforms assert runner._failed_platforms[Platform.TELEGRAM]["paused"] is True mock_create.assert_not_called() @pytest.mark.asyncio async def test_reconnect_skips_when_not_time_yet(self): """Watcher should skip platforms whose next_retry is in the future.""" runner = _make_runner() platform_config = PlatformConfig(enabled=True, token="test") runner._failed_platforms[Platform.TELEGRAM] = { "config": platform_config, "attempts": 1, "next_retry": time.monotonic() + 9999, # Far in the future } real_sleep = asyncio.sleep with patch.object(runner, "_create_adapter") as mock_create: async def run_one_iteration(): runner._running = True call_count = 0 async def fake_sleep(n): nonlocal call_count call_count += 1 if call_count > 1: runner._running = False await real_sleep(0) with patch("asyncio.sleep", side_effect=fake_sleep): await runner._platform_reconnect_watcher() await run_one_iteration() assert Platform.TELEGRAM in runner._failed_platforms mock_create.assert_not_called() @pytest.mark.asyncio async def test_no_failed_platforms_watcher_idles(self): """When no platforms are failed, watcher should just idle.""" runner = _make_runner() # No failed platforms real_sleep = asyncio.sleep with patch.object(runner, "_create_adapter") as mock_create: async def run_briefly(): runner._running = True call_count = 0 async def fake_sleep(n): nonlocal call_count call_count += 1 if call_count > 2: runner._running = False await real_sleep(0) with patch("asyncio.sleep", side_effect=fake_sleep): await runner._platform_reconnect_watcher() await run_briefly() mock_create.assert_not_called() @pytest.mark.asyncio async def test_adapter_create_returns_none(self): """If _create_adapter returns None, remove from queue (missing deps).""" runner = _make_runner() platform_config = PlatformConfig(enabled=True, token="test") runner._failed_platforms[Platform.TELEGRAM] = { "config": platform_config, "attempts": 1, "next_retry": time.monotonic() - 1, } real_sleep = asyncio.sleep with patch.object(runner, "_create_adapter", return_value=None): async def run_one_iteration(): runner._running = True call_count = 0 async def fake_sleep(n): nonlocal call_count call_count += 1 if call_count > 1: runner._running = False await real_sleep(0) with patch("asyncio.sleep", side_effect=fake_sleep): await runner._platform_reconnect_watcher() await run_one_iteration() assert Platform.TELEGRAM not in runner._failed_platforms # --- Runtime disconnection queueing --- class TestRuntimeDisconnectQueuing: """Test that _handle_adapter_fatal_error queues retryable disconnections.""" @pytest.mark.asyncio async def test_retryable_runtime_error_queued_for_reconnect(self): """Retryable runtime errors should add the platform to _failed_platforms.""" runner = _make_runner() runner.stop = AsyncMock() adapter = StubAdapter(succeed=True) adapter._set_fatal_error("network_error", "DNS failure", retryable=True) runner.adapters[Platform.TELEGRAM] = adapter await runner._handle_adapter_fatal_error(adapter) assert Platform.TELEGRAM in runner._failed_platforms assert runner._failed_platforms[Platform.TELEGRAM]["attempts"] == 0 @pytest.mark.asyncio async def test_retryable_runtime_error_reconnects_immediately(self): """Runtime failures should not wait for the startup retry delay.""" runner = _make_runner() runner.stop = AsyncMock() adapter = StubAdapter(succeed=True) adapter._set_fatal_error("sidecar_crashed", "bridge exited", retryable=True) runner.adapters[Platform.TELEGRAM] = adapter before = time.monotonic() await runner._handle_adapter_fatal_error(adapter) after = time.monotonic() info = runner._failed_platforms[Platform.TELEGRAM] assert info["attempts"] == 0 assert before <= info["next_retry"] <= after @pytest.mark.asyncio async def test_nonretryable_runtime_error_not_queued(self): """Non-retryable runtime errors should not be queued for reconnection.""" runner = _make_runner() adapter = StubAdapter(succeed=True) adapter._set_fatal_error("auth_error", "bad token", retryable=False) runner.adapters[Platform.TELEGRAM] = adapter # Need to prevent stop() from running fully runner.stop = AsyncMock() await runner._handle_adapter_fatal_error(adapter) assert Platform.TELEGRAM not in runner._failed_platforms @pytest.mark.asyncio async def test_retryable_error_keeps_gateway_alive_when_all_down(self): """When all adapters fail at runtime with retryable errors, the gateway should stay alive and let the reconnect watcher recover them in the background. (Previously this exited-with-failure to trigger a systemd restart — that converted transient outages into infinite restart loops and killed in-process state.) """ runner = _make_runner() runner.stop = AsyncMock() adapter = StubAdapter(succeed=True) adapter._set_fatal_error("network_error", "DNS failure", retryable=True) runner.adapters[Platform.TELEGRAM] = adapter await runner._handle_adapter_fatal_error(adapter) # stop() should NOT be called — gateway stays alive for the watcher runner.stop.assert_not_called() assert runner._exit_with_failure is False assert Platform.TELEGRAM in runner._failed_platforms @pytest.mark.asyncio async def test_retryable_error_no_exit_when_other_adapters_still_connected(self): """Gateway should NOT exit if some adapters are still connected.""" runner = _make_runner() runner.stop = AsyncMock() failing_adapter = StubAdapter(succeed=True) failing_adapter._set_fatal_error("network_error", "DNS failure", retryable=True) runner.adapters[Platform.TELEGRAM] = failing_adapter # Another adapter is still connected healthy_adapter = StubAdapter(succeed=True) runner.adapters[Platform.DISCORD] = healthy_adapter await runner._handle_adapter_fatal_error(failing_adapter) # stop() should NOT have been called — Discord is still up runner.stop.assert_not_called() assert Platform.TELEGRAM in runner._failed_platforms @pytest.mark.asyncio async def test_nonretryable_error_triggers_shutdown(self): """Gateway should shut down when no adapters remain and nothing is queued.""" runner = _make_runner() runner.stop = AsyncMock() adapter = StubAdapter(succeed=True) adapter._set_fatal_error("auth_error", "bad token", retryable=False) runner.adapters[Platform.TELEGRAM] = adapter await runner._handle_adapter_fatal_error(adapter) runner.stop.assert_called_once() # --- Pause / resume circuit breaker --- class TestPauseResume: """Test the per-platform pause/resume helpers and slash command.""" def test_pause_marks_platform_paused(self): runner = _make_runner() runner._failed_platforms[Platform.TELEGRAM] = { "config": PlatformConfig(enabled=True, token="t"), "attempts": 3, "next_retry": time.monotonic() + 30, } runner._pause_failed_platform(Platform.TELEGRAM, reason="manual") info = runner._failed_platforms[Platform.TELEGRAM] assert info["paused"] is True assert info["pause_reason"] == "manual" assert info["next_retry"] == float("inf") def test_pause_is_idempotent(self): runner = _make_runner() runner._failed_platforms[Platform.TELEGRAM] = { "config": PlatformConfig(enabled=True, token="t"), "attempts": 3, "next_retry": time.monotonic() + 30, "paused": True, "pause_reason": "first reason", } runner._pause_failed_platform(Platform.TELEGRAM, reason="second reason") # Reason should not be overwritten on a second pause call. assert ( runner._failed_platforms[Platform.TELEGRAM]["pause_reason"] == "first reason" ) def test_pause_no_op_when_platform_not_queued(self): runner = _make_runner() # No exception even when the platform isn't in _failed_platforms. runner._pause_failed_platform(Platform.TELEGRAM, reason="x") assert Platform.TELEGRAM not in runner._failed_platforms def test_resume_clears_paused_and_resets_attempts(self): runner = _make_runner() runner._failed_platforms[Platform.TELEGRAM] = { "config": PlatformConfig(enabled=True, token="t"), "attempts": 10, "next_retry": float("inf"), "paused": True, "pause_reason": "auto-paused", } assert runner._resume_paused_platform(Platform.TELEGRAM) is True info = runner._failed_platforms[Platform.TELEGRAM] assert info["paused"] is False assert info["attempts"] == 0 assert info["next_retry"] != float("inf") assert "pause_reason" not in info def test_resume_returns_false_when_not_paused(self): runner = _make_runner() runner._failed_platforms[Platform.TELEGRAM] = { "config": PlatformConfig(enabled=True, token="t"), "attempts": 1, "next_retry": time.monotonic() + 30, } assert runner._resume_paused_platform(Platform.TELEGRAM) is False def test_resume_returns_false_when_not_queued(self): runner = _make_runner() assert runner._resume_paused_platform(Platform.TELEGRAM) is False class TestPlatformSlashCommand: """Test the /platform list|pause|resume slash command handler.""" def _make_event(self, content: str): ev = MagicMock() ev.content = content return ev @pytest.mark.asyncio async def test_list_shows_connected_and_paused(self): runner = _make_runner() runner.adapters[Platform.DISCORD] = StubAdapter(platform=Platform.DISCORD) runner._failed_platforms[Platform.WHATSAPP] = { "config": PlatformConfig(enabled=True, token="t"), "attempts": 10, "next_retry": float("inf"), "paused": True, "pause_reason": "not paired", } out = await runner._handle_platform_command(self._make_event("/platform list")) assert "discord" in out assert "whatsapp" in out assert "PAUSED" in out assert "not paired" in out @pytest.mark.asyncio async def test_pause_command_pauses_queued_platform(self): runner = _make_runner() runner._failed_platforms[Platform.WHATSAPP] = { "config": PlatformConfig(enabled=True, token="t"), "attempts": 2, "next_retry": time.monotonic() + 30, } out = await runner._handle_platform_command( self._make_event("/platform pause whatsapp") ) assert "paused" in out.lower() assert runner._failed_platforms[Platform.WHATSAPP]["paused"] is True @pytest.mark.asyncio async def test_pause_rejects_unqueued_platform(self): runner = _make_runner() out = await runner._handle_platform_command( self._make_event("/platform pause whatsapp") ) assert "not in the retry queue" in out @pytest.mark.asyncio async def test_resume_command_resumes_paused_platform(self): runner = _make_runner() runner._failed_platforms[Platform.WHATSAPP] = { "config": PlatformConfig(enabled=True, token="t"), "attempts": 10, "next_retry": float("inf"), "paused": True, "pause_reason": "x", } out = await runner._handle_platform_command( self._make_event("/platform resume whatsapp") ) assert "resumed" in out.lower() assert runner._failed_platforms[Platform.WHATSAPP]["paused"] is False @pytest.mark.asyncio async def test_unknown_platform_name(self): runner = _make_runner() out = await runner._handle_platform_command( self._make_event("/platform pause notarealplatform") ) assert "Unknown platform" in out @pytest.mark.asyncio async def test_bare_platform_shows_usage_with_list(self): # An empty /platform call defaults to "list". runner = _make_runner() out = await runner._handle_platform_command(self._make_event("/platform")) assert "Gateway platforms" in out # --- Supervised task wrapper (_spawn_supervised) --- class TestSpawnSupervised: """Verify the task-level supervision wrapper around watcher launches.""" @pytest.mark.asyncio async def test_clean_synchronous_return_is_not_respawned(self): # A supervised coro that returns immediately (clean exit) must be # invoked EXACTLY ONCE — a clean return means deliberate shutdown or a # gated no-op watcher; respawning it would busy-spin the event loop. runner = _make_runner() calls = {"n": 0} async def _coro(): calls["n"] += 1 return runner._spawn_supervised(lambda: _coro(), "clean_watcher") # Drive the loop so the done-callback fires; if it (wrongly) respawned, # the count would keep climbing across these ticks. for _ in range(50): await asyncio.sleep(0) assert calls["n"] == 1 @pytest.mark.asyncio async def test_exception_restart_bounded_by_ceiling(self, monkeypatch): # A coro that always raises is restarted with backoff, but the restart # chain is capped: initial launch + _MAX_SUPERVISED_RESTARTS respawns. runner = _make_runner() calls = {"n": 0} # Collapse the backoff sleeps to a single loop-yield so the restart # chain converges fast. Bind the real sleep BEFORE patching so the # replacement still yields control (and doesn't recurse into itself). real_sleep = asyncio.sleep async def _instant_sleep(_delay): await real_sleep(0) monkeypatch.setattr("gateway.run.asyncio.sleep", _instant_sleep) async def _coro(): calls["n"] += 1 raise RuntimeError("boom") runner._spawn_supervised(lambda: _coro(), "always_raises") expected = runner._MAX_SUPERVISED_RESTARTS + 1 for _ in range(500): await real_sleep(0) if calls["n"] >= expected: break # A few extra ticks to prove the chain has stopped (no over-restart). for _ in range(20): await real_sleep(0) assert calls["n"] == expected @pytest.mark.asyncio async def test_healthy_run_then_crash_resets_restart_counter(self, monkeypatch): # A watcher that runs HEALTHILY (>= _SUPERVISED_HEALTHY_SECS) before # each crash must NOT be abandoned at the ceiling: every healthy run # resets the consecutive-failure counter, so the daemon keeps # restarting it well past _MAX_SUPERVISED_RESTARTS. This is the # long-lived-launchd-daemon guarantee — a watcher that crashes a # handful of times over days is never permanently dropped. runner = _make_runner() # Treat every run as "healthy": with the floor at 0s, any positive # real elapsed (ran_for >= 0.0) counts as a fresh, isolated failure, # so the effective attempt resets to 0 on each crash. monkeypatch.setattr(runner, "_SUPERVISED_HEALTHY_SECS", 0.0) real_sleep = asyncio.sleep async def _instant_sleep(_delay): await real_sleep(0) monkeypatch.setattr("gateway.run.asyncio.sleep", _instant_sleep) # Crash more times than the cumulative cap would ever allow, then # return cleanly to terminate the chain. crash_budget = runner._MAX_SUPERVISED_RESTARTS + 3 calls = {"n": 0} async def _coro(): calls["n"] += 1 if calls["n"] <= crash_budget: raise RuntimeError("boom") return runner._spawn_supervised(lambda: _coro(), "healthy_then_crash") target = crash_budget + 1 # crash_budget failures + one final clean run for _ in range(2000): await real_sleep(0) if calls["n"] >= target: break for _ in range(20): await real_sleep(0) # Under the OLD cumulative cap this would have stopped at # _MAX_SUPERVISED_RESTARTS + 1; the reset lets it run to completion. assert calls["n"] == target assert calls["n"] > runner._MAX_SUPERVISED_RESTARTS + 1 class TestFatalHandoffCancellationProof: """The fatal-error handoff must survive cancellation of the notifying task, and a retryable platform must never be silently stranded.""" @pytest.mark.asyncio async def test_caller_cancellation_does_not_strand_platform(self): """The fatal notification arrives on the failing adapter's own polling task, and adapter.disconnect() inside the handler can cancel that task mid-teardown. The platform must still reach the reconnect queue (previously the CancelledError killed the handler between the fatal log and the queue, stranding the platform until a manual restart).""" runner = _make_runner() runner.stop = AsyncMock() adapter = StubAdapter(succeed=True) adapter._set_fatal_error("network_error", "DNS failure", retryable=True) runner.adapters[Platform.TELEGRAM] = adapter release = asyncio.Event() async def slow_disconnect(): await release.wait() adapter.disconnect = slow_disconnect # hold the handler mid-teardown caller = asyncio.create_task(runner._handle_adapter_fatal_error(adapter)) for _ in range(5): await asyncio.sleep(0) # let the handler reach the disconnect await caller.cancel() # what disconnect() does to the notifying task with pytest.raises(asyncio.CancelledError): await caller release.set() # teardown completes after the caller has died for _ in range(200): if Platform.TELEGRAM in runner._failed_platforms: break await asyncio.sleep(0.01) assert Platform.TELEGRAM in runner._failed_platforms @pytest.mark.asyncio async def test_stranded_retryable_platform_exits_for_supervisor_restart(self): """If a retryable platform ends up neither reconnected nor queued (e.g. its config entry is gone so queueing is skipped), the gateway must exit with failure so launchd/systemd KeepAlive restarts it, instead of running indefinitely with a dead platform while healthy peers mask the loss (#68693).""" runner = _make_runner() async def _stop(): runner._shutdown_event.set() runner.stop = AsyncMock(side_effect=_stop) runner.config = GatewayConfig(platforms={}) # queueing impossible adapter = StubAdapter(succeed=True) adapter._set_fatal_error("network_error", "DNS failure", retryable=True) runner.adapters[Platform.TELEGRAM] = adapter # A healthy peer keeps self.adapters non-empty, so the existing # "no platforms remain" shutdown branches do not fire. runner.adapters[Platform.FEISHU] = StubAdapter(platform=Platform.FEISHU) await runner._handle_adapter_fatal_error(adapter) assert runner._exit_with_failure is True assert runner.stop.await_count == 1 # ── _ensure_reconnect_watcher_running ────────────────────────────────── class TestEnsureReconnectWatcherRunning: """Verify _ensure_reconnect_watcher_running respawns the watcher when dead.""" @pytest.mark.asyncio async def test_reconnect_watcher_alive_does_nothing(self): """Task is alive => no-op.""" runner = _make_runner() runner._running = True runner._background_tasks = set() async def _dummy(): await asyncio.sleep(3600) runner._reconnect_watcher_task = asyncio.create_task(_dummy()) runner._background_tasks.add(runner._reconnect_watcher_task) old_task = runner._reconnect_watcher_task runner._ensure_reconnect_watcher_running() # Same task, not replaced assert runner._reconnect_watcher_task is old_task assert not runner._reconnect_watcher_task.done() old_task.cancel() try: await old_task except asyncio.CancelledError: pass @pytest.mark.asyncio async def test_reconnect_watcher_dead_respawns(self): """Watcher is done => respawn.""" runner = _make_runner() runner._running = True runner._background_tasks = set() runner._reconnect_watcher_task = asyncio.create_task(asyncio.sleep(0)) await runner._reconnect_watcher_task # let it finish assert runner._reconnect_watcher_task.done() runner._ensure_reconnect_watcher_running() assert runner._reconnect_watcher_task is not None assert not runner._reconnect_watcher_task.done() assert runner._reconnect_watcher_task in runner._background_tasks runner._reconnect_watcher_task.cancel() try: await runner._reconnect_watcher_task except asyncio.CancelledError: pass @pytest.mark.asyncio async def test_reconnect_watcher_not_running_respawns(self): """No task at all => creates one.""" runner = _make_runner() runner._running = True runner._background_tasks = set() runner._reconnect_watcher_task = None runner._ensure_reconnect_watcher_running() assert runner._reconnect_watcher_task is not None assert not runner._reconnect_watcher_task.done() assert runner._reconnect_watcher_task in runner._background_tasks runner._reconnect_watcher_task.cancel() try: await runner._reconnect_watcher_task except asyncio.CancelledError: pass @pytest.mark.asyncio async def test_not_running_noop(self): """_running is False => no-op.""" runner = _make_runner() runner._running = False runner._reconnect_watcher_task = None runner._ensure_reconnect_watcher_running() assert runner._reconnect_watcher_task is None # ── _handle_adapter_fatal_error calls _ensure_reconnect_watcher ──────── class TestReconnectWatcherSelfHeals: """Regression tests for issue #71758: a platform already queued in _failed_platforms when the reconnect watcher task dies from an uncaught exception stayed stranded forever, because _ensure_reconnect_watcher_running() is only called from a NEW fatal-error arrival -- if no other platform ever fails afterward, nothing notices the watcher is dead. The watcher must now be spawned via _spawn_supervised (like other long-lived background tasks), so an exception escaping its OUTER while-loop is caught, logged, and auto-restarted with backoff -- independent of any new fatal-error event. """ @pytest.mark.asyncio async def test_startup_spawns_watcher_via_spawn_supervised(self, monkeypatch): """The initial watcher spawn at gateway startup must go through _spawn_supervised, not a bare asyncio.create_task.""" runner = _make_runner() runner._background_tasks = set() calls = [] def fake_spawn_supervised(coro_factory, name, **kw): calls.append(name) task = asyncio.create_task(coro_factory()) runner._background_tasks.add(task) return task async def _noop_watcher(): await asyncio.sleep(3600) monkeypatch.setattr(runner, "_spawn_supervised", fake_spawn_supervised) monkeypatch.setattr(runner, "_platform_reconnect_watcher", _noop_watcher) # Mirror the startup snippet: spawn via _spawn_supervised. runner._reconnect_watcher_task = runner._spawn_supervised( runner._platform_reconnect_watcher, "platform_reconnect_watcher" ) assert "platform_reconnect_watcher" in calls runner._reconnect_watcher_task.cancel() try: await runner._reconnect_watcher_task except asyncio.CancelledError: pass @pytest.mark.asyncio async def test_ensure_reconnect_watcher_running_uses_spawn_supervised(self): """The manual-respawn path in _ensure_reconnect_watcher_running must also use _spawn_supervised, so a respawned watcher gets the same auto-restart protection going forward.""" runner = _make_runner() runner._running = True runner._background_tasks = set() runner._reconnect_watcher_task = asyncio.create_task(asyncio.sleep(0)) await runner._reconnect_watcher_task # let it finish (dead) spawn_calls = [] original_spawn = runner._spawn_supervised def spy_spawn_supervised(coro_factory, name, **kw): spawn_calls.append(name) return original_spawn(coro_factory, name, **kw) runner._spawn_supervised = spy_spawn_supervised runner._ensure_reconnect_watcher_running() assert spawn_calls == ["platform_reconnect_watcher"] runner._reconnect_watcher_task.cancel() try: await runner._reconnect_watcher_task except asyncio.CancelledError: pass @pytest.mark.asyncio async def test_watcher_self_heals_after_uncaught_exception_with_no_new_fatal_error(self): """The core #71758 regression: a platform sits queued in _failed_platforms. The watcher task dies from an uncaught exception (simulating the KeyError race / any other bug in the outer loop). WITHOUT any new fatal-error event for a different platform, the watcher must still come back on its own via _spawn_supervised's crash-detection callback -- the exact gap that stranded the platform for 17.5h in the reported bug. """ runner = _make_runner() runner._running = True runner._background_tasks = set() runner._SUPERVISED_HEALTHY_SECS = GatewayRunner._SUPERVISED_HEALTHY_SECS runner._MAX_SUPERVISED_RESTARTS = GatewayRunner._MAX_SUPERVISED_RESTARTS runner._spawn_supervised = GatewayRunner._spawn_supervised.__get__(runner) attempt_count = {"n": 0} async def _flaky_watcher(): attempt_count["n"] += 1 if attempt_count["n"] == 1: # Simulate the watcher's outer loop raising -- e.g. the # KeyError race this same fix also hardens against, or any # other bug in code outside the per-platform try/except. raise RuntimeError("simulated watcher crash") await asyncio.sleep(3600) # second run: stays "alive" runner._reconnect_watcher_task = runner._spawn_supervised( _flaky_watcher, "platform_reconnect_watcher" ) # Let the first (crashing) attempt run and die. for _ in range(50): await asyncio.sleep(0) if attempt_count["n"] >= 1 and runner._reconnect_watcher_task.done(): break assert attempt_count["n"] == 1 assert runner._reconnect_watcher_task.done() # The supervised _done callback schedules a respawn after a short # backoff (2**0 = 1s at attempt 0) -- wait for it without a new # fatal-error event ever firing. for _ in range(30): await asyncio.sleep(0.1) if attempt_count["n"] >= 2: break assert attempt_count["n"] >= 2, ( "Watcher must self-heal via _spawn_supervised without any new " "fatal-error event -- this is the exact gap that stranded a " "platform in the reported bug" ) # Cleanup: cancel whatever task is currently tracked. for task in list(runner._background_tasks): task.cancel() await asyncio.sleep(0) class TestReconnectWatcherRaceGuard: """Regression: a platform removed from _failed_platforms concurrently (e.g. a manual /platform resume racing with the watcher's own snapshot-then-lookup) must not raise KeyError and kill the loop iteration -- it should just be skipped for that pass.""" @pytest.mark.asyncio async def test_watcher_survives_platform_removed_mid_pass(self, monkeypatch): """Two platforms are due for retry. Reconnecting the first one, as a side effect, removes the second from _failed_platforms (stand-in for any concurrent path -- a manual /platform resume, a reconnect that succeeded elsewhere, etc.). The watcher must finish its pass without raising, and must still be alive afterward.""" runner = _make_runner() runner._running = True runner._background_tasks = set() runner.session_store = MagicMock() runner._busy_text_mode = "interrupt" cfg = PlatformConfig(enabled=True, token="test") runner._failed_platforms = { Platform.TELEGRAM: {"config": cfg, "attempts": 0, "next_retry": 0.0}, Platform.DISCORD: {"config": cfg, "attempts": 0, "next_retry": 0.0}, } runner._platform_connect_timeout_secs = MagicMock(return_value=5) runner._sync_voice_mode_state_to_adapter = MagicMock() runner._schedule_resume_pending_sessions = MagicMock() runner._make_adapter_auth_check = MagicMock(return_value=lambda *a, **kw: True) runner._recover_telegram_topic_thread_id = MagicMock() runner._handle_message = MagicMock() runner._handle_active_session_busy_message = MagicMock() runner._handle_reaction_event = MagicMock() runner._update_platform_runtime_status = MagicMock() def fake_create_adapter(platform, platform_config): adapter = MagicMock() adapter.platform = platform adapter.has_fatal_error = False adapter.set_message_handler = MagicMock() adapter.set_fatal_error_handler = MagicMock() adapter.set_session_store = MagicMock() adapter.set_busy_session_handler = MagicMock() adapter.set_reaction_handler = MagicMock() adapter.set_topic_recovery_fn = MagicMock() adapter.set_authorization_check = MagicMock() if platform == Platform.TELEGRAM: # Side effect: concurrently "resolves" Discord's entry via # some other path (e.g. a manual /platform resume), racing # with the watcher's own snapshot-then-lookup for it. runner._failed_platforms.pop(Platform.DISCORD, None) return adapter runner._create_adapter = MagicMock(side_effect=fake_create_adapter) async def fake_connect(adapter, platform, is_reconnect=False): return True runner._connect_adapter_with_timeout = fake_connect async def _one_pass(): now = time.monotonic() for platform in list(runner._failed_platforms.keys()): info = runner._failed_platforms.get(platform) if info is None: continue if now < info["next_retry"]: continue adapter = runner._create_adapter(platform, info["config"]) success = await runner._connect_adapter_with_timeout( adapter, platform, is_reconnect=True ) if success: runner.adapters[platform] = adapter runner._failed_platforms.pop(platform, None) # Must not raise, even though Discord vanishes from the dict as a # side effect of processing Telegram. await _one_pass() assert Platform.TELEGRAM in runner.adapters assert Platform.DISCORD not in runner._failed_platforms """Verify _handle_adapter_fatal_error calls _ensure_reconnect_watcher_running.""" @pytest.mark.asyncio async def test_retryable_fatal_error_calls_ensure_watcher(self): """A retryable fatal error queues the platform AND ensures watcher is alive.""" runner = _make_runner() runner._running = True runner._background_tasks = set() runner._failed_platforms = {} runner._fatal_handler_tasks = set() runner._reconnect_watcher_task = asyncio.create_task(asyncio.sleep(0)) # Let the dummy watcher finish so _ensure_reconnect_watcher_running # detects it's dead and respawns. await runner._reconnect_watcher_task platform_config = PlatformConfig(enabled=True, token="test") runner.config = GatewayConfig( platforms={Platform.TELEGRAM: platform_config} ) adapter = StubAdapter( platform=Platform.TELEGRAM, succeed=False, fatal_error="network outage", fatal_retryable=True, ) # Pre-set fatal error attributes so the handler can read them # without going through connect() (#70344). adapter._set_fatal_error( "NETWORK_ERROR", "network outage", retryable=True ) # Populate adapters so the impl pops it and queues for reconnect runner.adapters[Platform.TELEGRAM] = adapter call_count = {"ensure": 0} def tracking_ensure(): call_count["ensure"] += 1 with patch.object( runner, "_ensure_reconnect_watcher_running", side_effect=tracking_ensure, ): await runner._handle_adapter_fatal_error(adapter) assert Platform.TELEGRAM in runner._failed_platforms assert call_count["ensure"] >= 1 @pytest.mark.asyncio async def test_nonretryable_fatal_error_does_not_call_ensure(self): """A non-retryable error must NOT queue the platform or call the watcher.""" runner = _make_runner() runner._running = True runner._background_tasks = set() runner._failed_platforms = {} runner._fatal_handler_tasks = set() runner._reconnect_watcher_task = None platform_config = PlatformConfig(enabled=True, token="test") runner.config = GatewayConfig( platforms={Platform.TELEGRAM: platform_config} ) adapter = StubAdapter( platform=Platform.TELEGRAM, succeed=False, fatal_error="bad token", fatal_retryable=False, ) # Pre-set fatal error attributes so the handler can read them # without going through connect() (#70344). adapter._set_fatal_error( "AUTH_FAILED", "bad token", retryable=False ) runner.adapters[Platform.TELEGRAM] = adapter ensure_called = False def noop_ensure(): nonlocal ensure_called ensure_called = True with patch.object(runner, "_ensure_reconnect_watcher_running", side_effect=noop_ensure): await runner._handle_adapter_fatal_error(adapter) assert Platform.TELEGRAM not in runner._failed_platforms assert not ensure_called # ── _connect_adapter_with_timeout detach-on-timeout ──────────────────── class TestConnectAdapterDetachOnTimeout: """Verify _connect_adapter_with_timeout uses the detach pattern.""" @pytest.mark.asyncio async def test_connect_timed_out_raises_timeouterror(self): """A connect() that never finishes must raise TimeoutError.""" runner = _make_runner() adapter = StubAdapter(succeed=True) async def _slow_connect(**kwargs): await asyncio.sleep(3600) # never finishes with patch.object(adapter, "connect", side_effect=_slow_connect): with patch.object( runner, "_platform_connect_timeout_secs", return_value=0.01 ): with pytest.raises(TimeoutError, match="timed out"): await runner._connect_adapter_with_timeout( adapter, Platform.TELEGRAM ) # After the TimeoutError, the slow connect coroutine should have been # cancelled and detached, so the event loop can move on. await asyncio.sleep(0) @pytest.mark.asyncio async def test_connect_success_returns_true(self): """A successful connect returns True.""" runner = _make_runner() adapter = StubAdapter(succeed=True) with patch.object( runner, "_platform_connect_timeout_secs", return_value=30.0 ): result = await runner._connect_adapter_with_timeout( adapter, Platform.TELEGRAM ) assert result is True class TestReconnectWatcherHandleTracking: """Regression: the supervisor's own backoff respawn must keep ``_reconnect_watcher_task`` pointed at the CURRENT live task. Before the ``on_spawn`` fix, ``_spawn_supervised``'s internal respawn created a new task without updating ``self._reconnect_watcher_task``, so after the reconnect watcher crashed and self-respawned, the tracked handle still pointed at the DEAD task. A later ``_ensure_reconnect_watcher_running()`` then saw ``task.done()`` and spawned a SECOND concurrent watcher — double reconnect attempts against every failed platform. The two supervision mechanisms (auto-restart + ensure-respawn) must compose, not race. """ @pytest.mark.asyncio async def test_startup_spawn_tracks_live_handle(self): """The startup spawn passes an on_spawn callback so the handle is recorded at spawn time (not left None until the lambda in prod).""" runner = _make_runner() runner._background_tasks = set() async def _noop_watcher(): await asyncio.sleep(3600) # Mirror the production startup call: on_spawn records the handle. runner._reconnect_watcher_task = None task = runner._spawn_supervised( _noop_watcher, "platform_reconnect_watcher", on_spawn=lambda t: setattr(runner, "_reconnect_watcher_task", t), ) # on_spawn fired synchronously at spawn time. assert runner._reconnect_watcher_task is task task.cancel() try: await task except asyncio.CancelledError: pass @pytest.mark.asyncio async def test_supervised_respawn_refreshes_tracked_handle(self): """When the supervised watcher crashes and the supervisor respawns it, _reconnect_watcher_task must advance to the NEW task, not stay pinned to the dead one. This is the exact condition _ensure_reconnect_watcher_running checks (task.done()) before deciding to spawn a duplicate.""" runner = _make_runner() runner._running = True runner._background_tasks = set() # Fast, deterministic backoff. runner._MAX_SUPERVISED_RESTARTS = 5 runner._SUPERVISED_HEALTHY_SECS = 300 crashed_once = {"done": False} async def _crash_then_live(): if not crashed_once["done"]: crashed_once["done"] = True raise RuntimeError("boom in outer loop") await asyncio.sleep(3600) runner._reconnect_watcher_task = None first = runner._spawn_supervised( _crash_then_live, "platform_reconnect_watcher", on_spawn=lambda t: setattr(runner, "_reconnect_watcher_task", t), ) assert runner._reconnect_watcher_task is first # Let the first task crash and the supervisor's backoff (2**0 = 1s here, # but _attempt=0 -> min(60, 1)=1) schedule + run the respawn. with patch("asyncio.sleep", new=_instant_sleep): # Give the event loop turns for the _done callback + _respawn task. for _ in range(20): await asyncio.sleep(0) if runner._reconnect_watcher_task is not first: break # The handle must now point at the respawned (live) task, NOT the dead one. assert runner._reconnect_watcher_task is not first assert not runner._reconnect_watcher_task.done() # And _ensure_reconnect_watcher_running must therefore be a no-op — it # must NOT spawn a duplicate, because the tracked handle is alive. spawned = [] original = runner._spawn_supervised runner._spawn_supervised = lambda *a, **k: spawned.append(a) or original(*a, **k) runner._ensure_reconnect_watcher_running() assert spawned == [], "ensure spawned a duplicate watcher despite a live handle" runner._reconnect_watcher_task.cancel() try: await runner._reconnect_watcher_task except asyncio.CancelledError: pass async def _instant_sleep(delay, *a, **k): """asyncio.sleep replacement that yields to the loop but never waits.""" await _REAL_ASYNCIO_SLEEP(0) return None _REAL_ASYNCIO_SLEEP = asyncio.sleep