from __future__ import annotations import asyncio from collections.abc import Callable from datetime import timedelta from types import SimpleNamespace from typing import cast import pytest from fastapi.testclient import TestClient from app.core.errors import ConflictError from app.core.time import to_iso, utc_now from app.rotation.repository import RotationRepository from app.rotation.schedule_service import SchedulerLoop from app.rotation.service import RotationService class RecordingRotationService: def __init__(self, outcomes: dict[str, Exception] | None = None) -> None: self.outcomes = outcomes or {} self.calls: list[tuple[str, str]] = [] self.dispatched = asyncio.Event() def start_group(self, group_id: str, trigger: str = "manual") -> SimpleNamespace: self.calls.append((group_id, trigger)) outcome = self.outcomes.get(group_id) if outcome is not None: raise outcome self.dispatched.set() return SimpleNamespace(id=f"run-{group_id}") class ScriptedRepository: def __init__(self, scans: list[list[str] | Exception]) -> None: self.scans = scans self.scan_count = 0 self.deferred: list[str] = [] def due_group_ids(self) -> list[str]: self.scan_count += 1 result = self.scans.pop(0) if self.scans else [] if isinstance(result, Exception): raise result return result def defer_group(self, group_id: str) -> None: self.deferred.append(group_id) def scheduler_for( repository: RotationRepository | ScriptedRepository, service: RecordingRotationService, ) -> SchedulerLoop: return SchedulerLoop( cast(RotationRepository, repository), cast(RotationService, service), ) async def wait_until(predicate: Callable[[], bool], timeout: float = 0.5) -> None: deadline = asyncio.get_running_loop().time() + timeout while not predicate(): if asyncio.get_running_loop().time() >= deadline: pytest.fail("等待调度器状态变化超时") await asyncio.sleep(0.005) @pytest.mark.asyncio async def test_scheduler_dispatches_due_group_from_sqlite( client: TestClient, managed_instance_payload: dict[str, object], group_payload: dict[str, object], ) -> None: container = client.app.state.container container.fleet_repository.create_instance(managed_instance_payload) group = container.fleet_repository.create_group(group_payload) with container.database.connect() as connection, connection: connection.execute( "UPDATE instance_groups SET next_run_at = ? WHERE id = ?", (to_iso(utc_now() - timedelta(seconds=1)), group.id), ) service = RecordingRotationService() scheduler = scheduler_for(container.rotation_repository, service) scheduler.start(0.01) try: await wait_until(service.dispatched.is_set) status = scheduler.status(enabled=True) finally: await scheduler.stop() assert service.calls[0] == (group.id, "scheduled") assert status["state"] == "running" assert status["last_tick_at"] is not None assert status["last_success_at"] is not None assert status["last_dispatch_at"] is not None @pytest.mark.asyncio async def test_scheduler_recovers_after_scan_error(caplog: pytest.LogCaptureFixture) -> None: repository = ScriptedRepository([RuntimeError("database is temporarily locked"), ["group"]]) service = RecordingRotationService() scheduler = scheduler_for(repository, service) scheduler.start(0.01) try: await wait_until(service.dispatched.is_set) status = scheduler.status(enabled=True) finally: await scheduler.stop() assert repository.scan_count >= 2 assert service.calls == [("group", "scheduled")] assert status["running"] is True assert status["last_error_at"] is not None assert status["consecutive_errors"] == 0 assert "scheduler_tick_failed" in caplog.messages @pytest.mark.asyncio async def test_scheduler_reports_unhealthy_after_repeated_scan_errors() -> None: repository = ScriptedRepository([RuntimeError("database locked") for _ in range(100)]) service = RecordingRotationService() scheduler = scheduler_for(repository, service) scheduler.start(0.01) try: await wait_until(lambda: repository.scan_count >= 3) status = scheduler.status(enabled=True) finally: await scheduler.stop() assert status["running"] is True assert status["healthy"] is False assert int(status["consecutive_errors"]) >= 3 @pytest.mark.asyncio async def test_non_active_conflict_does_not_block_next_due_group() -> None: repository = ScriptedRepository([["disabled-group", "ready-group"]]) service = RecordingRotationService( { "disabled-group": ConflictError( "实例组计划未启用", code="GROUP_SCHEDULE_DISABLED", ) } ) scheduler = scheduler_for(repository, service) scheduler.start(0.01) try: await wait_until(service.dispatched.is_set) finally: await scheduler.stop() assert service.calls == [ ("disabled-group", "scheduled"), ("ready-group", "scheduled"), ] assert repository.deferred == [] @pytest.mark.asyncio async def test_unexpected_scheduler_cancellation_is_reported( caplog: pytest.LogCaptureFixture, ) -> None: repository = ScriptedRepository([[]]) service = RecordingRotationService() scheduler = scheduler_for(repository, service) scheduler.start(1) await wait_until(lambda: repository.scan_count >= 1) task = scheduler._task assert task is not None task.cancel() await asyncio.gather(task, return_exceptions=True) await asyncio.sleep(0) status = scheduler.status(enabled=True) assert status["state"] == "stopped" assert status["healthy"] is False assert status["last_error_at"] is not None assert "scheduler_cancelled_unexpectedly" in caplog.messages def test_readyz_reports_scheduler_disabled(client: TestClient) -> None: response = client.get("/readyz") assert response.status_code == 200 assert response.json()["scheduler"] == { "enabled": False, "running": False, "healthy": True, "state": "disabled", "tick_seconds": None, "started_at": None, "last_tick_at": None, "last_success_at": None, "last_dispatch_at": None, "last_error_at": None, "consecutive_errors": 0, } def test_readyz_is_degraded_when_enabled_scheduler_is_not_running(client: TestClient) -> None: client.app.state.container.settings.scheduler_enabled = True response = client.get("/readyz") assert response.status_code == 503 assert response.json()["status"] == "degraded" assert response.json()["scheduler"]["healthy"] is False assert response.json()["scheduler"]["state"] == "stopped"