1217 lines
46 KiB
Python
1217 lines
46 KiB
Python
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import logging
|
||
import uuid
|
||
from contextlib import suppress
|
||
from datetime import timedelta
|
||
from typing import Literal
|
||
|
||
from app.accounts.service import AccountService
|
||
from app.core.errors import AppError, ConflictError, NotFoundError, ValidationAppError
|
||
from app.core.time import from_iso, to_iso, utc_now
|
||
from app.integrations.aws_client import LightsailClient, StaticIpSnapshot
|
||
from app.integrations.cloudflare_client import CloudflareClient
|
||
from app.integrations.service import IntegrationService
|
||
from app.rotation.repository import (
|
||
ROTATION_LEASE_TTL_SECONDS,
|
||
FleetRun,
|
||
FleetRunItem,
|
||
RotationLeaseLostError,
|
||
RotationRepository,
|
||
)
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class RotationService:
|
||
def __init__(
|
||
self,
|
||
repository: RotationRepository,
|
||
integration_service: IntegrationService,
|
||
account_service: AccountService | None = None,
|
||
*,
|
||
poll_interval_seconds: float = 3,
|
||
) -> None:
|
||
self.repository = repository
|
||
self.integration_service = integration_service
|
||
self.account_service = account_service
|
||
self.poll_interval_seconds = poll_interval_seconds
|
||
self.worker_id = str(uuid.uuid4())
|
||
self._tasks: dict[str, asyncio.Task[bool]] = {}
|
||
self._lease_lost_runs: set[str] = set()
|
||
|
||
def start_instance(self, instance_id: str) -> FleetRun:
|
||
return self._start("instance", instance_id, "manual")
|
||
|
||
def start_group(
|
||
self,
|
||
group_id: str,
|
||
trigger: Literal["manual", "scheduled"] = "manual",
|
||
) -> FleetRun:
|
||
return self._start("group", group_id, trigger)
|
||
|
||
def _start(self, target_type: str, target_id: str, trigger: str) -> FleetRun:
|
||
if self.account_service is None:
|
||
self.integration_service.ensure_configured()
|
||
else:
|
||
for aws_account_id, cloudflare_account_id in (
|
||
self.repository.target_account_bindings(target_type, target_id)
|
||
):
|
||
self._resolve_credentials(aws_account_id, cloudflare_account_id)
|
||
try:
|
||
run = (
|
||
self.repository.create_for_instance(target_id, trigger) # type: ignore[arg-type]
|
||
if target_type == "instance"
|
||
else self.repository.create_for_group(target_id, trigger) # type: ignore[arg-type]
|
||
)
|
||
except RuntimeError as exc:
|
||
raise self._repository_error(exc) from exc
|
||
self._spawn(run.id)
|
||
return run
|
||
|
||
def recover(self) -> None:
|
||
active = self.repository.get_active()
|
||
if active and active.status in {"queued", "running"}:
|
||
self.repository.add_event(
|
||
active.id,
|
||
item_id=active.current_item_id,
|
||
stage="recovery",
|
||
level="warning",
|
||
message="控制器重启，正在根据 AWS 与 DNS 实际状态续跑",
|
||
)
|
||
self._spawn(active.id)
|
||
|
||
def resume(self, run_id: str) -> FleetRun:
|
||
if self.repository.get(run_id) is None:
|
||
raise NotFoundError("轮换任务不存在", code="ROTATION_NOT_FOUND")
|
||
try:
|
||
run = self.repository.prepare_resume(run_id)
|
||
except RuntimeError as exc:
|
||
if str(exc) == "RUN_NOT_RESUMABLE":
|
||
raise ConflictError(
|
||
"只有待处理或待清理任务可以继续",
|
||
code="ROTATION_NOT_RESUMABLE",
|
||
) from exc
|
||
raise
|
||
self._spawn(run_id)
|
||
return run
|
||
|
||
def cancel(self, run_id: str) -> FleetRun:
|
||
if self.repository.get(run_id) is None:
|
||
raise NotFoundError("轮换任务不存在", code="ROTATION_NOT_FOUND")
|
||
try:
|
||
return self.repository.cancel(run_id)
|
||
except RuntimeError as exc:
|
||
if str(exc) == "RUN_NOT_CANCELABLE":
|
||
raise ConflictError(
|
||
"只有待处理或待清理任务可以放弃",
|
||
code="ROTATION_NOT_CANCELABLE",
|
||
) from exc
|
||
raise
|
||
|
||
def get(self, run_id: str) -> dict[str, object]:
|
||
run = self.repository.get(run_id)
|
||
if run is None:
|
||
raise NotFoundError("轮换任务不存在", code="ROTATION_NOT_FOUND")
|
||
return {
|
||
**run.to_dict(),
|
||
"items": [item.to_dict() for item in self.repository.list_items(run_id)],
|
||
"events": self.repository.list_events(run_id),
|
||
}
|
||
|
||
def list(
|
||
self,
|
||
page: int,
|
||
per_page: int,
|
||
*,
|
||
instance_id: str | None = None,
|
||
group_id: str | None = None,
|
||
status: str | None = None,
|
||
) -> tuple[list[dict[str, object]], int]:
|
||
runs, total = self.repository.list_runs(
|
||
page,
|
||
per_page,
|
||
instance_id=instance_id,
|
||
group_id=group_id,
|
||
status=status,
|
||
)
|
||
return [run.to_dict() for run in runs], total
|
||
|
||
def history_summary(self) -> dict[str, int | str | None]:
|
||
return self.repository.history_summary()
|
||
|
||
async def shutdown(self) -> None:
|
||
tasks = list(self._tasks.values())
|
||
for task in tasks:
|
||
task.cancel()
|
||
if tasks:
|
||
await asyncio.gather(*tasks, return_exceptions=True)
|
||
|
||
def _spawn(self, run_id: str) -> None:
|
||
existing = self._tasks.get(run_id)
|
||
if existing and not existing.done():
|
||
return
|
||
task = asyncio.create_task(self._execute(run_id), name=f"fleet-rotation-{run_id}")
|
||
self._tasks[run_id] = task
|
||
task.add_done_callback(lambda completed: self._rotation_done(run_id, completed))
|
||
|
||
def _rotation_done(self, run_id: str, task: asyncio.Task[bool]) -> None:
|
||
self._tasks.pop(run_id, None)
|
||
if task.cancelled():
|
||
return
|
||
try:
|
||
retry = task.result()
|
||
except Exception:
|
||
logger.exception("Rotation executor task failed", extra={"run_id": run_id})
|
||
return
|
||
if not retry:
|
||
return
|
||
active = self.repository.get_active()
|
||
if active and active.id == run_id and active.status in {"queued", "running"}:
|
||
self._spawn(run_id)
|
||
|
||
async def _execute(self, run_id: str) -> bool:
|
||
if not await self._acquire_lease(run_id):
|
||
return False
|
||
self._lease_lost_runs.discard(run_id)
|
||
heartbeat = asyncio.create_task(
|
||
self._lease_heartbeat(run_id),
|
||
name=f"fleet-rotation-heartbeat-{run_id}",
|
||
)
|
||
try:
|
||
self.repository.transition_run(
|
||
run_id,
|
||
status="running",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
while True:
|
||
run = self.repository.get(run_id)
|
||
item = self.repository.current_item(run_id)
|
||
if run is None or item is None or run.status not in {"queued", "running"}:
|
||
return False
|
||
if item.status == "queued":
|
||
item = self.repository.transition_item(
|
||
run_id,
|
||
item.id,
|
||
status="running",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self._require_lease(run_id)
|
||
use_default, secrets, token = self._resolve_credentials(
|
||
item.aws_account_id,
|
||
item.cloudflare_account_id,
|
||
)
|
||
aws = LightsailClient.from_credentials(
|
||
region=item.aws_region,
|
||
instance_name=item.lightsail_instance_name,
|
||
use_default=use_default,
|
||
secrets=secrets,
|
||
)
|
||
cloudflare = CloudflareClient(
|
||
token=token,
|
||
zone_name=item.cloudflare_zone_name,
|
||
zone_id=item.cloudflare_zone_id,
|
||
record_name=item.cloudflare_record_name,
|
||
)
|
||
await self._dispatch_stage(run, item, aws, cloudflare)
|
||
except asyncio.CancelledError:
|
||
run = self.repository.get(run_id)
|
||
if run and run.status == "running":
|
||
with suppress(RotationLeaseLostError):
|
||
self.repository.add_event(
|
||
run_id,
|
||
item_id=run.current_item_id,
|
||
stage="shutdown",
|
||
level="warning",
|
||
message="控制器正在关闭，任务将在下次启动后续跑",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
raise
|
||
except RotationLeaseLostError:
|
||
logger.warning("Rotation lease lost; executor stopped", extra={"run_id": run_id})
|
||
return True
|
||
except AppError as exc:
|
||
return self._handle_failure(run_id, exc.code, exc.detail)
|
||
except Exception:
|
||
logger.exception("Unexpected rotation error", extra={"run_id": run_id})
|
||
return self._handle_failure(run_id, "UNEXPECTED_ERROR", "轮换遇到未预期错误")
|
||
finally:
|
||
heartbeat.cancel()
|
||
await asyncio.gather(heartbeat, return_exceptions=True)
|
||
self._lease_lost_runs.discard(run_id)
|
||
|
||
def _resolve_credentials(
|
||
self,
|
||
aws_account_id: str | None,
|
||
cloudflare_account_id: str | None,
|
||
) -> tuple[bool, dict[str, str | None], str]:
|
||
legacy_settings = None
|
||
legacy_secrets: dict[str, str | None] = {}
|
||
if not aws_account_id or not cloudflare_account_id:
|
||
legacy_settings, legacy_secrets = self.integration_service.credentials()
|
||
|
||
if aws_account_id and self.account_service is not None:
|
||
use_default, aws_secrets = self.account_service.resolve_aws(aws_account_id)
|
||
else:
|
||
if legacy_settings is None: # pragma: no cover - guarded by condition above
|
||
legacy_settings, legacy_secrets = self.integration_service.credentials()
|
||
use_default = legacy_settings.use_default_aws_credentials
|
||
aws_secrets = legacy_secrets
|
||
if not use_default and not (
|
||
aws_secrets.get("aws_access_key_id")
|
||
and aws_secrets.get("aws_secret_access_key")
|
||
):
|
||
raise ValidationAppError(
|
||
"当前没有可用的 AWS 凭据",
|
||
code="AWS_CREDENTIALS_MISSING",
|
||
)
|
||
|
||
if cloudflare_account_id and self.account_service is not None:
|
||
token = self.account_service.resolve_cloudflare(cloudflare_account_id)
|
||
else:
|
||
token = legacy_secrets.get("cloudflare_api_token")
|
||
if not token:
|
||
raise ValidationAppError(
|
||
"当前没有可用的 Cloudflare API Token",
|
||
code="CLOUDFLARE_TOKEN_MISSING",
|
||
)
|
||
return use_default, aws_secrets, token
|
||
|
||
async def _dispatch_stage(
|
||
self,
|
||
run: FleetRun,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
cloudflare: CloudflareClient,
|
||
) -> None:
|
||
handlers = {
|
||
"preflight": self._preflight,
|
||
"new_allocating": self._allocate_new,
|
||
"old_detaching": self._detach_old,
|
||
"new_attaching": self._attach_new,
|
||
"health_checking": self._health_check,
|
||
"dns_updating": self._update_dns,
|
||
"dns_verifying": self._verify_dns,
|
||
"release_grace": self._wait_release_grace,
|
||
"old_releasing": self._release_old,
|
||
"rollback_reconciling": self._rollback_reconciling,
|
||
"rollback_new_detaching": self._rollback_detach_new,
|
||
"rollback_old_attaching": self._rollback_attach_old,
|
||
"rollback_health_checking": self._rollback_health_check,
|
||
"rollback_new_releasing": self._rollback_release_new,
|
||
}
|
||
handler = handlers.get(item.stage)
|
||
if handler is None:
|
||
raise ValidationAppError(
|
||
f"无法识别任务阶段：{item.stage}",
|
||
code="UNKNOWN_ROTATION_STAGE",
|
||
)
|
||
await handler(run, item, aws, cloudflare)
|
||
|
||
async def _preflight(
|
||
self,
|
||
run: FleetRun,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
cloudflare: CloudflareClient,
|
||
) -> None:
|
||
instance, attached, record = await asyncio.gather(
|
||
asyncio.to_thread(aws.get_instance),
|
||
asyncio.to_thread(aws.get_attached_static_ips, item.lightsail_instance_name),
|
||
asyncio.to_thread(cloudflare.get_a_record),
|
||
)
|
||
if instance.state != "running" or not instance.public_ip:
|
||
raise ValidationAppError(
|
||
"Lightsail 实例必须保持运行并拥有公网 IPv4",
|
||
code="INSTANCE_NOT_RUNNING",
|
||
)
|
||
if len(attached) > 1:
|
||
raise ValidationAppError(
|
||
"检测到同一实例关联了多个 Static IP，请先在 AWS 控制台核对",
|
||
code="MULTIPLE_ATTACHED_STATIC_IPS",
|
||
)
|
||
if record is None:
|
||
raise ValidationAppError(
|
||
"Cloudflare A 记录不存在，请先执行 DNS 同步",
|
||
code="DNS_RECORD_MISSING",
|
||
)
|
||
if bool(record.get("proxied")):
|
||
raise ValidationAppError(
|
||
"Cloudflare 记录必须切换为 DNS-only",
|
||
code="DNS_RECORD_PROXIED",
|
||
)
|
||
dns_ip = str(record.get("content") or "")
|
||
if dns_ip != instance.public_ip:
|
||
raise ValidationAppError(
|
||
"Cloudflare A 记录与实例当前公网 IP 不一致，请先同步 DNS",
|
||
code="DNS_OUT_OF_SYNC",
|
||
)
|
||
old = attached[0] if attached else None
|
||
static_ip_conflict = old is None and instance.is_static_ip
|
||
if old is not None:
|
||
static_ip_conflict = (
|
||
old.ip_address != instance.public_ip
|
||
or old.attached_to != item.lightsail_instance_name
|
||
or not instance.is_static_ip
|
||
)
|
||
if static_ip_conflict:
|
||
raise ValidationAppError(
|
||
"AWS 实例与当前 Static IP 状态不一致，请先人工核对",
|
||
code="STATIC_IP_STATE_CONFLICT",
|
||
)
|
||
new_name = self.repository.static_ip_name(item.lightsail_instance_name, item.id)
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
message="预检通过，准备分配新的 Static IP",
|
||
details={
|
||
"old_ip": instance.public_ip,
|
||
"old_static_ip_name": old.name if old else None,
|
||
"initial_dynamic_ip": old is None,
|
||
},
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self.repository.transition_item(
|
||
run.id,
|
||
item.id,
|
||
stage="new_allocating",
|
||
old_static_ip_name=old.name if old else None,
|
||
old_ip=instance.public_ip,
|
||
dns_ip_before=dns_ip,
|
||
new_static_ip_name=new_name,
|
||
expected_owner=self.worker_id,
|
||
)
|
||
|
||
async def _allocate_new(
|
||
self,
|
||
run: FleetRun,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
_cloudflare: CloudflareClient,
|
||
) -> None:
|
||
if not item.new_static_ip_name:
|
||
raise ValidationAppError("任务缺少新 Static IP 资源名", code="STATIC_IP_NAME_MISSING")
|
||
new_static = await asyncio.to_thread(aws.get_static_ip, item.new_static_ip_name)
|
||
if new_static is None:
|
||
self._require_lease(run.id)
|
||
await asyncio.to_thread(aws.allocate_static_ip, item.new_static_ip_name)
|
||
new_static = await asyncio.to_thread(aws.get_static_ip, item.new_static_ip_name)
|
||
self._validate_owned_new_static(item, new_static)
|
||
if not new_static or not new_static.ip_address:
|
||
raise ValidationAppError(
|
||
"AWS 尚未返回新 Static IP 地址",
|
||
code="STATIC_IP_ALLOCATION_INCOMPLETE",
|
||
)
|
||
next_stage = (
|
||
"health_checking"
|
||
if new_static.is_attached and new_static.attached_to == item.lightsail_instance_name
|
||
else "old_detaching"
|
||
)
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
message="新的 Static IP 已分配",
|
||
details={"name": new_static.name, "ip": new_static.ip_address},
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self.repository.transition_item(
|
||
run.id,
|
||
item.id,
|
||
stage=next_stage,
|
||
new_ip=new_static.ip_address,
|
||
expected_owner=self.worker_id,
|
||
)
|
||
|
||
async def _detach_old(
|
||
self,
|
||
run: FleetRun,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
cloudflare: CloudflareClient,
|
||
) -> None:
|
||
new_static = await asyncio.to_thread(aws.get_static_ip, item.new_static_ip_name)
|
||
self._validate_owned_new_static(item, new_static)
|
||
if new_static and new_static.is_attached:
|
||
self.repository.transition_item(
|
||
run.id,
|
||
item.id,
|
||
stage="health_checking",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
return
|
||
await self._require_dns(cloudflare, item.dns_ip_before)
|
||
if item.old_static_ip_name:
|
||
old_static = await asyncio.to_thread(aws.get_static_ip, item.old_static_ip_name)
|
||
self._validate_old_static(item, old_static)
|
||
if old_static and old_static.is_attached:
|
||
self._require_lease(run.id)
|
||
await asyncio.to_thread(aws.detach_static_ip, item.old_static_ip_name)
|
||
old_static = await asyncio.to_thread(aws.get_static_ip, item.old_static_ip_name)
|
||
if old_static is None or old_static.is_attached:
|
||
raise ValidationAppError(
|
||
"旧 Static IP 解绑后的状态校验失败",
|
||
code="OLD_STATIC_IP_DETACH_NOT_VERIFIED",
|
||
)
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
message="旧 Static IP 已解绑",
|
||
details={"name": item.old_static_ip_name, "ip": item.old_ip},
|
||
expected_owner=self.worker_id,
|
||
)
|
||
else:
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
level="warning",
|
||
message="实例原先使用动态 IP，本次将完成首次 Static IP 纳管",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self.repository.transition_item(
|
||
run.id,
|
||
item.id,
|
||
stage="new_attaching",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
|
||
async def _attach_new(
|
||
self,
|
||
run: FleetRun,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
cloudflare: CloudflareClient,
|
||
) -> None:
|
||
await self._require_dns(cloudflare, item.dns_ip_before)
|
||
new_static = await asyncio.to_thread(aws.get_static_ip, item.new_static_ip_name)
|
||
self._validate_owned_new_static(item, new_static)
|
||
if new_static and not new_static.is_attached:
|
||
self._require_lease(run.id)
|
||
await asyncio.to_thread(
|
||
aws.attach_static_ip,
|
||
item.new_static_ip_name,
|
||
item.lightsail_instance_name,
|
||
)
|
||
await self._require_new_attachment(item, aws)
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
message="新的 Static IP 已附加到实例",
|
||
details={"name": item.new_static_ip_name, "ip": item.new_ip},
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self.repository.transition_item(
|
||
run.id,
|
||
item.id,
|
||
stage="health_checking",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
|
||
async def _health_check(
|
||
self,
|
||
run: FleetRun,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
_cloudflare: CloudflareClient,
|
||
) -> None:
|
||
await self._require_new_attachment(item, aws)
|
||
if item.proxy_health_check:
|
||
await self._wait_for_socks5(
|
||
run.id,
|
||
str(item.new_ip),
|
||
item.socks_port,
|
||
item.health_timeout_seconds,
|
||
)
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
message=(
|
||
"SOCKS5 协议握手通过"
|
||
if item.proxy_health_check
|
||
else "已跳过 SOCKS5 可用性检查"
|
||
),
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self.repository.transition_item(
|
||
run.id,
|
||
item.id,
|
||
stage="dns_updating",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
|
||
async def _update_dns(
|
||
self,
|
||
run: FleetRun,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
cloudflare: CloudflareClient,
|
||
) -> None:
|
||
await self._require_new_attachment(item, aws)
|
||
self._require_lease(run.id)
|
||
record = await asyncio.to_thread(
|
||
cloudflare.upsert_a_record,
|
||
str(item.new_ip),
|
||
expected_current=item.dns_ip_before,
|
||
)
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
message="Cloudflare A 记录已切换为新 Static IP",
|
||
details={"record": record.get("name"), "ip": item.new_ip},
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self.repository.transition_item(
|
||
run.id,
|
||
item.id,
|
||
stage="dns_verifying",
|
||
dns_ip_after=str(record.get("content") or item.new_ip),
|
||
cloudflare_zone_id=cloudflare.zone_id,
|
||
expected_owner=self.worker_id,
|
||
)
|
||
|
||
async def _verify_dns(
|
||
self,
|
||
run: FleetRun,
|
||
item: FleetRunItem,
|
||
_aws: LightsailClient,
|
||
cloudflare: CloudflareClient,
|
||
) -> None:
|
||
record = await asyncio.to_thread(cloudflare.get_a_record)
|
||
if record and record.get("content") == item.new_ip and not record.get("proxied"):
|
||
grace_until = to_iso(
|
||
utc_now() + timedelta(seconds=max(60, item.release_grace_seconds))
|
||
)
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
message="DNS 回读验证通过，进入旧 IP 释放宽限期",
|
||
details={"grace_until": grace_until},
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self.repository.transition_item(
|
||
run.id,
|
||
item.id,
|
||
stage="release_grace",
|
||
dns_ip_after=str(item.new_ip),
|
||
grace_until=grace_until,
|
||
expected_owner=self.worker_id,
|
||
)
|
||
return
|
||
if self.repository.stage_elapsed_seconds(item) > 90:
|
||
raise ValidationAppError(
|
||
"Cloudflare DNS 回读验证超时",
|
||
code="DNS_VERIFICATION_TIMEOUT",
|
||
)
|
||
await self._sleep_with_lease(run.id)
|
||
|
||
async def _wait_release_grace(
|
||
self,
|
||
run: FleetRun,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
cloudflare: CloudflareClient,
|
||
) -> None:
|
||
await self._require_new_attachment(item, aws)
|
||
await self._require_dns(cloudflare, item.new_ip)
|
||
if not item.old_static_ip_name:
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
message="首次 Static IP 纳管完成，没有旧静态资源需要释放",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self.repository.complete_item(run.id, item.id, self.worker_id)
|
||
return
|
||
if item.grace_until and utc_now() < from_iso(item.grace_until):
|
||
await self._sleep_with_lease(run.id)
|
||
return
|
||
self.repository.transition_item(
|
||
run.id,
|
||
item.id,
|
||
stage="old_releasing",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
|
||
async def _release_old(
|
||
self,
|
||
run: FleetRun,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
cloudflare: CloudflareClient,
|
||
) -> None:
|
||
await self._require_new_attachment(item, aws)
|
||
await self._require_dns(cloudflare, item.new_ip)
|
||
if item.proxy_health_check:
|
||
await self._probe_socks5(str(item.new_ip), item.socks_port)
|
||
old_static = await asyncio.to_thread(aws.get_static_ip, item.old_static_ip_name)
|
||
if old_static is not None:
|
||
self._validate_old_static(item, old_static)
|
||
if old_static.is_attached:
|
||
raise ValidationAppError(
|
||
"旧 Static IP 仍处于附加状态，拒绝删除",
|
||
code="OLD_STATIC_IP_STILL_ATTACHED",
|
||
)
|
||
self._require_lease(run.id)
|
||
await asyncio.to_thread(aws.release_static_ip, item.old_static_ip_name)
|
||
old_static = await asyncio.to_thread(aws.get_static_ip, item.old_static_ip_name)
|
||
if old_static is not None:
|
||
raise ValidationAppError(
|
||
"旧 Static IP 删除后的回读校验失败",
|
||
code="OLD_STATIC_IP_RELEASE_NOT_VERIFIED",
|
||
)
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
message="旧 Static IP 已从 AWS 账号删除",
|
||
details={"name": item.old_static_ip_name, "ip": item.old_ip},
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self.repository.complete_item(run.id, item.id, self.worker_id)
|
||
|
||
async def _rollback_reconciling(
|
||
self,
|
||
run: FleetRun,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
cloudflare: CloudflareClient,
|
||
) -> None:
|
||
if not all(
|
||
(
|
||
item.old_static_ip_name,
|
||
item.old_ip,
|
||
item.new_static_ip_name,
|
||
item.new_ip,
|
||
)
|
||
):
|
||
raise ValidationAppError(
|
||
"任务缺少自动回滚所需的 Static IP 快照",
|
||
code="ROLLBACK_SNAPSHOT_INCOMPLETE",
|
||
)
|
||
record, old_static, new_static, instance = await asyncio.gather(
|
||
asyncio.to_thread(cloudflare.get_a_record),
|
||
asyncio.to_thread(aws.get_static_ip, item.old_static_ip_name),
|
||
asyncio.to_thread(aws.get_static_ip, item.new_static_ip_name),
|
||
asyncio.to_thread(aws.get_instance),
|
||
)
|
||
self._validate_old_static(item, old_static)
|
||
self._validate_owned_new_static(item, new_static)
|
||
if record is None or bool(record.get("proxied")):
|
||
raise ValidationAppError(
|
||
"DNS 记录缺失或已启用代理，无法自动回滚",
|
||
code="ROLLBACK_DNS_UNSAFE",
|
||
)
|
||
dns_ip = str(record.get("content") or "")
|
||
if dns_ip == item.new_ip:
|
||
if item.rollback_from_stage != "dns_updating":
|
||
raise ValidationAppError(
|
||
"DNS 已在轮换流程之外切换，无法自动判断安全方向",
|
||
code="ROLLBACK_DNS_UNSAFE",
|
||
)
|
||
await self._require_new_attachment(item, aws)
|
||
if item.proxy_health_check:
|
||
await self._wait_for_socks5(
|
||
run.id,
|
||
str(item.new_ip),
|
||
item.socks_port,
|
||
item.health_timeout_seconds,
|
||
)
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
level="warning",
|
||
message="DNS 已确认指向新地址，取消回滚并继续向前验证",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self.repository.transition_item(
|
||
run.id,
|
||
item.id,
|
||
stage="dns_verifying",
|
||
error_code=None,
|
||
error_message=None,
|
||
expected_owner=self.worker_id,
|
||
)
|
||
return
|
||
if dns_ip != item.old_ip:
|
||
raise ValidationAppError(
|
||
"DNS 已被修改为任务外地址，拒绝自动回滚",
|
||
code="ROLLBACK_DNS_UNSAFE",
|
||
)
|
||
if old_static and old_static.is_attached and new_static and new_static.is_attached:
|
||
raise ValidationAppError(
|
||
"新旧 Static IP 同时显示为已附加，拒绝自动回滚",
|
||
code="ROLLBACK_ATTACHMENT_CONFLICT",
|
||
)
|
||
if old_static and old_static.is_attached:
|
||
if (
|
||
old_static.attached_to != item.lightsail_instance_name
|
||
or instance.public_ip != item.old_ip
|
||
or not instance.is_static_ip
|
||
):
|
||
raise ValidationAppError(
|
||
"旧 Static IP 的附加状态无法确认",
|
||
code="ROLLBACK_OLD_ATTACHMENT_CONFLICT",
|
||
)
|
||
next_stage = "rollback_health_checking"
|
||
elif new_static and new_static.is_attached:
|
||
next_stage = "rollback_new_detaching"
|
||
else:
|
||
next_stage = "rollback_old_attaching"
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
level="warning",
|
||
message="已核对回滚现场，准备恢复旧 Static IP",
|
||
details={"next_stage": next_stage, "dns_ip": dns_ip},
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self.repository.transition_item(
|
||
run.id,
|
||
item.id,
|
||
stage=next_stage,
|
||
expected_owner=self.worker_id,
|
||
)
|
||
|
||
async def _rollback_detach_new(
|
||
self,
|
||
run: FleetRun,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
cloudflare: CloudflareClient,
|
||
) -> None:
|
||
await self._require_dns(cloudflare, item.old_ip)
|
||
old_static, new_static = await asyncio.gather(
|
||
asyncio.to_thread(aws.get_static_ip, item.old_static_ip_name),
|
||
asyncio.to_thread(aws.get_static_ip, item.new_static_ip_name),
|
||
)
|
||
self._validate_old_static(item, old_static)
|
||
self._validate_owned_new_static(item, new_static)
|
||
if old_static and old_static.is_attached and new_static and new_static.is_attached:
|
||
raise ValidationAppError(
|
||
"新旧 Static IP 同时显示为已附加，拒绝继续回滚",
|
||
code="ROLLBACK_ATTACHMENT_CONFLICT",
|
||
)
|
||
if new_static and new_static.is_attached:
|
||
self._require_lease(run.id)
|
||
await asyncio.to_thread(aws.detach_static_ip, item.new_static_ip_name)
|
||
new_static = await asyncio.to_thread(aws.get_static_ip, item.new_static_ip_name)
|
||
self._validate_owned_new_static(item, new_static)
|
||
if new_static and new_static.is_attached:
|
||
raise ValidationAppError(
|
||
"新 Static IP 解绑后的状态校验失败",
|
||
code="ROLLBACK_NEW_DETACH_NOT_VERIFIED",
|
||
)
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
level="warning",
|
||
message="回滚：新 Static IP 已解绑",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self.repository.transition_item(
|
||
run.id,
|
||
item.id,
|
||
stage="rollback_old_attaching",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
|
||
async def _rollback_attach_old(
|
||
self,
|
||
run: FleetRun,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
cloudflare: CloudflareClient,
|
||
) -> None:
|
||
await self._require_dns(cloudflare, item.old_ip)
|
||
old_static, new_static = await asyncio.gather(
|
||
asyncio.to_thread(aws.get_static_ip, item.old_static_ip_name),
|
||
asyncio.to_thread(aws.get_static_ip, item.new_static_ip_name),
|
||
)
|
||
self._validate_old_static(item, old_static)
|
||
self._validate_owned_new_static(item, new_static)
|
||
if new_static and new_static.is_attached:
|
||
raise ValidationAppError(
|
||
"新 Static IP 仍处于附加状态，拒绝覆盖附加旧地址",
|
||
code="ROLLBACK_NEW_IP_STILL_ATTACHED",
|
||
)
|
||
if old_static and not old_static.is_attached:
|
||
self._require_lease(run.id)
|
||
await asyncio.to_thread(
|
||
aws.attach_static_ip,
|
||
item.old_static_ip_name,
|
||
item.lightsail_instance_name,
|
||
)
|
||
await self._require_old_attachment(item, aws)
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
level="warning",
|
||
message="回滚：旧 Static IP 已重新附加",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self.repository.transition_item(
|
||
run.id,
|
||
item.id,
|
||
stage="rollback_health_checking",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
|
||
async def _rollback_health_check(
|
||
self,
|
||
run: FleetRun,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
cloudflare: CloudflareClient,
|
||
) -> None:
|
||
await self._require_old_attachment(item, aws)
|
||
await self._require_dns(cloudflare, item.old_ip)
|
||
if item.proxy_health_check:
|
||
await self._wait_for_socks5(
|
||
run.id,
|
||
str(item.old_ip),
|
||
item.socks_port,
|
||
item.health_timeout_seconds,
|
||
)
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
level="warning",
|
||
message="回滚：旧地址与 SOCKS5 路由已恢复",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self.repository.transition_item(
|
||
run.id,
|
||
item.id,
|
||
stage="rollback_new_releasing",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
|
||
async def _rollback_release_new(
|
||
self,
|
||
run: FleetRun,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
cloudflare: CloudflareClient,
|
||
) -> None:
|
||
await self._require_old_attachment(item, aws)
|
||
await self._require_dns(cloudflare, item.old_ip)
|
||
if item.proxy_health_check:
|
||
await self._probe_socks5(str(item.old_ip), item.socks_port)
|
||
new_static = await asyncio.to_thread(aws.get_static_ip, item.new_static_ip_name)
|
||
self._validate_owned_new_static(item, new_static)
|
||
if new_static is not None:
|
||
if new_static.is_attached:
|
||
raise ValidationAppError(
|
||
"回滚后的新 Static IP 仍处于附加状态，拒绝删除",
|
||
code="ROLLBACK_NEW_IP_STILL_ATTACHED",
|
||
)
|
||
self._require_lease(run.id)
|
||
await asyncio.to_thread(aws.release_static_ip, item.new_static_ip_name)
|
||
if await asyncio.to_thread(aws.get_static_ip, item.new_static_ip_name) is not None:
|
||
raise ValidationAppError(
|
||
"回滚后的新 Static IP 删除校验失败",
|
||
code="ROLLBACK_NEW_RELEASE_NOT_VERIFIED",
|
||
)
|
||
self.repository.add_event(
|
||
run.id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
level="warning",
|
||
message="自动回滚完成，旧 Static IP 已恢复，新地址已删除",
|
||
details={"old_ip": item.old_ip, "released_new_ip": item.new_ip},
|
||
expected_owner=self.worker_id,
|
||
)
|
||
reason_code = item.rollback_reason_code or "ROTATION_ROLLED_BACK"
|
||
reason_message = item.rollback_reason_message or "轮换失败后已自动恢复旧 Static IP"
|
||
self.repository.fail_execution(
|
||
run.id,
|
||
item.id,
|
||
error_code=reason_code,
|
||
error_message=f"{reason_message}；已自动恢复旧 Static IP",
|
||
outcome="failed",
|
||
expected_owner=self.worker_id,
|
||
)
|
||
|
||
async def _require_new_attachment(
|
||
self,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
) -> StaticIpSnapshot:
|
||
new_static, instance = await asyncio.gather(
|
||
asyncio.to_thread(aws.get_static_ip, item.new_static_ip_name),
|
||
asyncio.to_thread(aws.get_instance),
|
||
)
|
||
self._validate_owned_new_static(item, new_static)
|
||
if (
|
||
new_static is None
|
||
or not new_static.is_attached
|
||
or new_static.attached_to != item.lightsail_instance_name
|
||
or new_static.ip_address != item.new_ip
|
||
or instance.public_ip != item.new_ip
|
||
or not instance.is_static_ip
|
||
):
|
||
raise ValidationAppError(
|
||
"新 Static IP 与实例的附加状态不一致",
|
||
code="NEW_STATIC_IP_ATTACHMENT_CONFLICT",
|
||
)
|
||
return new_static
|
||
|
||
async def _require_old_attachment(
|
||
self,
|
||
item: FleetRunItem,
|
||
aws: LightsailClient,
|
||
) -> StaticIpSnapshot:
|
||
old_static, instance = await asyncio.gather(
|
||
asyncio.to_thread(aws.get_static_ip, item.old_static_ip_name),
|
||
asyncio.to_thread(aws.get_instance),
|
||
)
|
||
self._validate_old_static(item, old_static)
|
||
if (
|
||
old_static is None
|
||
or not old_static.is_attached
|
||
or old_static.attached_to != item.lightsail_instance_name
|
||
or instance.public_ip != item.old_ip
|
||
or not instance.is_static_ip
|
||
):
|
||
raise ValidationAppError(
|
||
"旧 Static IP 的恢复附加状态不一致",
|
||
code="ROLLBACK_OLD_ATTACHMENT_CONFLICT",
|
||
)
|
||
return old_static
|
||
|
||
@staticmethod
|
||
def _validate_owned_new_static(
|
||
item: FleetRunItem,
|
||
static_ip: StaticIpSnapshot | None,
|
||
) -> None:
|
||
if static_ip is None:
|
||
return
|
||
if static_ip.name != item.new_static_ip_name:
|
||
raise ValidationAppError(
|
||
"新 Static IP 资源身份与任务快照不一致",
|
||
code="STATIC_IP_OWNERSHIP_CONFLICT",
|
||
)
|
||
if static_ip.region != item.aws_region:
|
||
raise ValidationAppError(
|
||
"新 Static IP 与实例不在同一区域",
|
||
code="STATIC_IP_REGION_MISMATCH",
|
||
)
|
||
if static_ip.is_attached and static_ip.attached_to != item.lightsail_instance_name:
|
||
raise ValidationAppError(
|
||
"新 Static IP 已被附加到其他实例，已停止自动操作",
|
||
code="STATIC_IP_OWNERSHIP_CONFLICT",
|
||
)
|
||
|
||
@staticmethod
|
||
def _validate_old_static(
|
||
item: FleetRunItem,
|
||
static_ip: StaticIpSnapshot | None,
|
||
) -> None:
|
||
if static_ip is None:
|
||
raise ValidationAppError(
|
||
"旧 Static IP 已被外部删除，无法安全继续",
|
||
code="OLD_STATIC_IP_EXTERNALLY_REMOVED",
|
||
)
|
||
if static_ip.name != item.old_static_ip_name or static_ip.ip_address != item.old_ip:
|
||
raise ValidationAppError(
|
||
"旧 Static IP 与任务快照不一致，已停止自动操作",
|
||
code="OLD_STATIC_IP_OWNERSHIP_CONFLICT",
|
||
)
|
||
if static_ip.region != item.aws_region:
|
||
raise ValidationAppError(
|
||
"旧 Static IP 与实例不在同一区域",
|
||
code="OLD_STATIC_IP_REGION_MISMATCH",
|
||
)
|
||
if static_ip.is_attached and static_ip.attached_to != item.lightsail_instance_name:
|
||
raise ValidationAppError(
|
||
"旧 Static IP 已被附加到其他实例，拒绝继续",
|
||
code="OLD_STATIC_IP_OWNERSHIP_CONFLICT",
|
||
)
|
||
|
||
@staticmethod
|
||
async def _require_dns(
|
||
cloudflare: CloudflareClient,
|
||
expected_ip: str | None,
|
||
) -> None:
|
||
record = await asyncio.to_thread(cloudflare.get_a_record)
|
||
if (
|
||
record is None
|
||
or record.get("content") != expected_ip
|
||
or bool(record.get("proxied"))
|
||
):
|
||
raise ValidationAppError(
|
||
"Cloudflare DNS 已被外部修改，已停止自动操作",
|
||
code="DNS_EXTERNALLY_MODIFIED",
|
||
)
|
||
|
||
async def _wait_for_socks5(
|
||
self,
|
||
run_id: str,
|
||
ip: str,
|
||
port: int,
|
||
timeout_seconds: int,
|
||
) -> None:
|
||
deadline = asyncio.get_running_loop().time() + timeout_seconds
|
||
last_error = ""
|
||
while asyncio.get_running_loop().time() < deadline:
|
||
self._require_lease(run_id)
|
||
try:
|
||
await self._probe_socks5(ip, port)
|
||
return
|
||
except (OSError, TimeoutError, asyncio.IncompleteReadError) as exc:
|
||
last_error = str(exc)
|
||
await asyncio.sleep(self.poll_interval_seconds)
|
||
raise ValidationAppError(
|
||
f"新 IP 的 SOCKS5 协议检查超时：{last_error or '无响应'}",
|
||
code="SOCKS5_HEALTH_TIMEOUT",
|
||
)
|
||
|
||
@staticmethod
|
||
async def _probe_socks5(ip: str, port: int) -> None:
|
||
reader: asyncio.StreamReader
|
||
writer: asyncio.StreamWriter
|
||
reader, writer = await asyncio.wait_for(asyncio.open_connection(ip, port), timeout=5)
|
||
try:
|
||
writer.write(b"\x05\x02\x00\x02")
|
||
await writer.drain()
|
||
response = await asyncio.wait_for(reader.readexactly(2), timeout=5)
|
||
if response[0] != 5 or response[1] not in {0, 2}:
|
||
raise OSError("目标端口未返回可识别的 SOCKS5 握手")
|
||
finally:
|
||
writer.close()
|
||
with suppress(Exception):
|
||
await writer.wait_closed()
|
||
|
||
def _handle_failure(self, run_id: str, code: str, message: str) -> bool:
|
||
run = self.repository.get(run_id)
|
||
item = self.repository.current_item(run_id)
|
||
if run is None or item is None or run.status not in {"queued", "running"}:
|
||
return False
|
||
rollback_sources = {
|
||
"old_detaching",
|
||
"new_attaching",
|
||
"health_checking",
|
||
"dns_updating",
|
||
}
|
||
if item.old_static_ip_name and item.stage in rollback_sources:
|
||
try:
|
||
self.repository.add_event(
|
||
run_id,
|
||
item_id=item.id,
|
||
stage=item.stage,
|
||
level="warning",
|
||
message="轮换失败，准备自动恢复旧 Static IP",
|
||
details={"code": code, "message": message},
|
||
expected_owner=self.worker_id,
|
||
)
|
||
self.repository.transition_item(
|
||
run_id,
|
||
item.id,
|
||
stage="rollback_reconciling",
|
||
rollback_from_stage=item.stage,
|
||
rollback_reason_code=code,
|
||
rollback_reason_message=message,
|
||
error_code=code,
|
||
error_message=message,
|
||
expected_owner=self.worker_id,
|
||
)
|
||
return True
|
||
except RotationLeaseLostError:
|
||
logger.warning(
|
||
"Rotation rollback handoff lost its lease",
|
||
extra={"run_id": run_id},
|
||
)
|
||
return True
|
||
if item.stage == "preflight":
|
||
outcome: Literal["failed", "needs_attention", "cleanup_pending"] = "failed"
|
||
elif item.stage in {"old_releasing", "rollback_new_releasing"}:
|
||
outcome = "cleanup_pending"
|
||
else:
|
||
outcome = "needs_attention"
|
||
try:
|
||
self.repository.fail_execution(
|
||
run_id,
|
||
item.id,
|
||
error_code=code,
|
||
error_message=message,
|
||
outcome=outcome,
|
||
expected_owner=self.worker_id,
|
||
)
|
||
except RotationLeaseLostError:
|
||
logger.warning(
|
||
"Rotation failure ignored after lease loss",
|
||
extra={"run_id": run_id},
|
||
)
|
||
return True
|
||
return False
|
||
|
||
async def _sleep_with_lease(self, run_id: str) -> None:
|
||
self._require_lease(run_id)
|
||
await asyncio.sleep(self.poll_interval_seconds)
|
||
|
||
def _require_lease(self, run_id: str) -> None:
|
||
if run_id in self._lease_lost_runs:
|
||
raise RotationLeaseLostError("ROTATION_LEASE_LOST")
|
||
if not self.repository.renew_lease(run_id, self.worker_id):
|
||
raise RotationLeaseLostError("ROTATION_LEASE_LOST")
|
||
|
||
async def _lease_heartbeat(self, run_id: str) -> None:
|
||
interval = max(1.0, ROTATION_LEASE_TTL_SECONDS / 3)
|
||
try:
|
||
while True:
|
||
await asyncio.sleep(interval)
|
||
if not self.repository.renew_lease(run_id, self.worker_id):
|
||
self._lease_lost_runs.add(run_id)
|
||
return
|
||
except asyncio.CancelledError:
|
||
raise
|
||
except Exception:
|
||
logger.exception(
|
||
"Rotation lease heartbeat failed",
|
||
extra={"run_id": run_id},
|
||
)
|
||
self._lease_lost_runs.add(run_id)
|
||
|
||
async def _acquire_lease(self, run_id: str) -> bool:
|
||
deadline = asyncio.get_running_loop().time() + ROTATION_LEASE_TTL_SECONDS + 15
|
||
while asyncio.get_running_loop().time() < deadline:
|
||
if self.repository.acquire_lease(run_id, self.worker_id):
|
||
return True
|
||
active = self.repository.get_active()
|
||
if active is None or active.id != run_id:
|
||
return False
|
||
await asyncio.sleep(min(max(self.poll_interval_seconds, 0.1), 2))
|
||
return False
|
||
|
||
@staticmethod
|
||
def _repository_error(exc: RuntimeError) -> AppError:
|
||
code = str(exc)
|
||
if code == "ACTIVE_ROTATION":
|
||
return ConflictError(
|
||
"已有轮换或 DNS 同步任务进行中",
|
||
code="ROTATION_IN_PROGRESS",
|
||
)
|
||
if code == "INSTANCE_NOT_FOUND":
|
||
return NotFoundError("托管实例不存在或已停用", code="INSTANCE_NOT_FOUND")
|
||
if code == "GROUP_NOT_FOUND":
|
||
return NotFoundError("实例组不存在", code="INSTANCE_GROUP_NOT_FOUND")
|
||
if code == "GROUP_HAS_NO_ENABLED_INSTANCES":
|
||
return ValidationAppError("实例组中没有启用的实例")
|
||
if code == "GROUP_SCHEDULE_DISABLED":
|
||
return ConflictError("实例组计划未启用", code="GROUP_SCHEDULE_DISABLED")
|
||
return ValidationAppError("无法创建轮换任务", code=code)
|