FluxIP/tests/test_schedule_service.py

from __future__ import annotations

import asyncio
from collections.abc import Callable
from datetime import timedelta
from types import SimpleNamespace
from typing import cast

import pytest
from fastapi.testclient import TestClient

from app.core.errors import ConflictError
from app.core.time import to_iso, utc_now
from app.rotation.repository import RotationRepository
from app.rotation.schedule_service import SchedulerLoop
from app.rotation.service import RotationService


class RecordingRotationService:
    def __init__(self, outcomes: dict[str, Exception] | None = None) -> None:
        self.outcomes = outcomes or {}
        self.calls: list[tuple[str, str]] = []
        self.dispatched = asyncio.Event()

    def start_group(self, group_id: str, trigger: str = "manual") -> SimpleNamespace:
        self.calls.append((group_id, trigger))
        outcome = self.outcomes.get(group_id)
        if outcome is not None:
            raise outcome
        self.dispatched.set()
        return SimpleNamespace(id=f"run-{group_id}")


class ScriptedRepository:
    def __init__(self, scans: list[list[str] | Exception]) -> None:
        self.scans = scans
        self.scan_count = 0
        self.deferred: list[str] = []

    def due_group_ids(self) -> list[str]:
        self.scan_count += 1
        result = self.scans.pop(0) if self.scans else []
        if isinstance(result, Exception):
            raise result
        return result

    def defer_group(self, group_id: str) -> None:
        self.deferred.append(group_id)


def scheduler_for(
    repository: RotationRepository | ScriptedRepository,
    service: RecordingRotationService,
) -> SchedulerLoop:
    return SchedulerLoop(
        cast(RotationRepository, repository),
        cast(RotationService, service),
    )


async def wait_until(predicate: Callable[[], bool], timeout: float = 0.5) -> None:
    deadline = asyncio.get_running_loop().time() + timeout
    while not predicate():
        if asyncio.get_running_loop().time() >= deadline:
            pytest.fail("等待调度器状态变化超时")
        await asyncio.sleep(0.005)


@pytest.mark.asyncio
async def test_scheduler_dispatches_due_group_from_sqlite(
    client: TestClient,
    managed_instance_payload: dict[str, object],
    group_payload: dict[str, object],
) -> None:
    container = client.app.state.container
    container.fleet_repository.create_instance(managed_instance_payload)
    group = container.fleet_repository.create_group(group_payload)
    with container.database.connect() as connection, connection:
        connection.execute(
            "UPDATE instance_groups SET next_run_at = ? WHERE id = ?",
            (to_iso(utc_now() - timedelta(seconds=1)), group.id),
        )
    service = RecordingRotationService()
    scheduler = scheduler_for(container.rotation_repository, service)

    scheduler.start(0.01)
    try:
        await wait_until(service.dispatched.is_set)
        status = scheduler.status(enabled=True)
    finally:
        await scheduler.stop()

    assert service.calls[0] == (group.id, "scheduled")
    assert status["state"] == "running"
    assert status["last_tick_at"] is not None
    assert status["last_success_at"] is not None
    assert status["last_dispatch_at"] is not None


@pytest.mark.asyncio
async def test_scheduler_recovers_after_scan_error(caplog: pytest.LogCaptureFixture) -> None:
    repository = ScriptedRepository([RuntimeError("database is temporarily locked"), ["group"]])
    service = RecordingRotationService()
    scheduler = scheduler_for(repository, service)

    scheduler.start(0.01)
    try:
        await wait_until(service.dispatched.is_set)
        status = scheduler.status(enabled=True)
    finally:
        await scheduler.stop()

    assert repository.scan_count >= 2
    assert service.calls == [("group", "scheduled")]
    assert status["running"] is True
    assert status["last_error_at"] is not None
    assert status["consecutive_errors"] == 0
    assert "scheduler_tick_failed" in caplog.messages


@pytest.mark.asyncio
async def test_scheduler_reports_unhealthy_after_repeated_scan_errors() -> None:
    repository = ScriptedRepository([RuntimeError("database locked") for _ in range(100)])
    service = RecordingRotationService()
    scheduler = scheduler_for(repository, service)

    scheduler.start(0.01)
    try:
        await wait_until(lambda: repository.scan_count >= 3)
        status = scheduler.status(enabled=True)
    finally:
        await scheduler.stop()

    assert status["running"] is True
    assert status["healthy"] is False
    assert int(status["consecutive_errors"]) >= 3


@pytest.mark.asyncio
async def test_non_active_conflict_does_not_block_next_due_group() -> None:
    repository = ScriptedRepository([["disabled-group", "ready-group"]])
    service = RecordingRotationService(
        {
            "disabled-group": ConflictError(
                "实例组计划未启用",
                code="GROUP_SCHEDULE_DISABLED",
            )
        }
    )
    scheduler = scheduler_for(repository, service)

    scheduler.start(0.01)
    try:
        await wait_until(service.dispatched.is_set)
    finally:
        await scheduler.stop()

    assert service.calls == [
        ("disabled-group", "scheduled"),
        ("ready-group", "scheduled"),
    ]
    assert repository.deferred == []


@pytest.mark.asyncio
async def test_unexpected_scheduler_cancellation_is_reported(
    caplog: pytest.LogCaptureFixture,
) -> None:
    repository = ScriptedRepository([[]])
    service = RecordingRotationService()
    scheduler = scheduler_for(repository, service)
    scheduler.start(1)
    await wait_until(lambda: repository.scan_count >= 1)
    task = scheduler._task
    assert task is not None

    task.cancel()
    await asyncio.gather(task, return_exceptions=True)
    await asyncio.sleep(0)
    status = scheduler.status(enabled=True)

    assert status["state"] == "stopped"
    assert status["healthy"] is False
    assert status["last_error_at"] is not None
    assert "scheduler_cancelled_unexpectedly" in caplog.messages


def test_readyz_reports_scheduler_disabled(client: TestClient) -> None:
    response = client.get("/readyz")

    assert response.status_code == 200
    assert response.json()["scheduler"] == {
        "enabled": False,
        "running": False,
        "healthy": True,
        "state": "disabled",
        "tick_seconds": None,
        "started_at": None,
        "last_tick_at": None,
        "last_success_at": None,
        "last_dispatch_at": None,
        "last_error_at": None,
        "consecutive_errors": 0,
    }


def test_readyz_is_degraded_when_enabled_scheduler_is_not_running(client: TestClient) -> None:
    client.app.state.container.settings.scheduler_enabled = True

    response = client.get("/readyz")

    assert response.status_code == 503
    assert response.json()["status"] == "degraded"
    assert response.json()["scheduler"]["healthy"] is False
    assert response.json()["scheduler"]["state"] == "stopped"