Compare commits

...
Author SHA1 Message Date
Nicolò Boschi e3656dfabd fix(worker): handle NotImplementedError from add_signal_handler on Windows
asyncio.AbstractEventLoop.add_signal_handler is Unix-only and raises
NotImplementedError on the Windows ProactorEventLoop. The worker would
crash silently ~30s into startup while the API process kept serving reads,
masking the failure (pending operations accumulate, consolidation never
runs).

Wrap the SIGINT/SIGTERM registration in a helper that swallows the
exception and reports back. On Windows we log a warning that the in-loop
two-stage shutdown is disabled; default Python SIGINT behavior still
terminates the process on Ctrl+C.

Fixes #1411
2026-05-04 12:29:41 +02:00
2 changed files with 71 additions and 5 deletions
@@ -16,6 +16,7 @@ import signal
import socket
import sys
import warnings
from collections.abc import Callable
from ..config import get_config
from ..engine.task_backend import WorkerTaskBackend
@@ -31,6 +32,26 @@ os.environ["TOKENIZERS_PARALLELISM"] = "false"
logger = logging.getLogger(__name__)
def _install_shutdown_signal_handlers(
loop: asyncio.AbstractEventLoop,
handler: Callable[[], None],
) -> bool:
"""Register SIGINT/SIGTERM handlers on the asyncio loop.
Returns True when handlers were installed via ``loop.add_signal_handler``.
Returns False on platforms (Windows ProactorEventLoop) where asyncio
does not implement signal handlers; the caller falls back to Python's
default SIGINT behavior, which still terminates the process on Ctrl+C
but loses the in-loop two-stage graceful shutdown.
"""
try:
loop.add_signal_handler(signal.SIGINT, handler)
loop.add_signal_handler(signal.SIGTERM, handler)
except NotImplementedError:
return False
return True
def create_worker_app(poller: WorkerPoller, memory):
"""Create a minimal FastAPI app for worker metrics and health."""
from fastapi import FastAPI
@@ -243,6 +264,7 @@ def main():
# Setup signal handlers for graceful shutdown using asyncio
shutdown_requested = asyncio.Event()
force_exit = False
async_handlers_installed = False
loop = asyncio.get_event_loop()
@@ -253,17 +275,26 @@ def main():
print("\nReceived second signal, forcing immediate exit...")
force_exit = True
# Restore default handler so third signal kills process
loop.remove_signal_handler(signal.SIGINT)
loop.remove_signal_handler(signal.SIGTERM)
if async_handlers_installed:
loop.remove_signal_handler(signal.SIGINT)
loop.remove_signal_handler(signal.SIGTERM)
sys.exit(1)
else:
print("\nReceived shutdown signal, initiating graceful shutdown...")
print("(Press Ctrl+C again to force immediate exit)")
shutdown_requested.set()
# Use asyncio's signal handlers which work properly with the event loop
loop.add_signal_handler(signal.SIGINT, signal_handler)
loop.add_signal_handler(signal.SIGTERM, signal_handler)
async_handlers_installed = _install_shutdown_signal_handlers(loop, signal_handler)
if not async_handlers_installed:
# Windows ProactorEventLoop: asyncio.add_signal_handler is Unix-only
# and raises NotImplementedError. Default Python SIGINT handler still
# terminates the worker on Ctrl+C, just without the two-stage path.
print(
f"WARN: asyncio signal handlers unavailable on this platform "
f"({sys.platform}); graceful two-stage shutdown disabled, "
f"default Python SIGINT handler remains active.",
flush=True,
)
# Create uvicorn config and server
uvicorn_config = uvicorn.Config(
@@ -0,0 +1,35 @@
"""Tests for hindsight_api.worker.main entry-point helpers."""
import asyncio
import signal
from unittest.mock import MagicMock
from hindsight_api.worker.main import _install_shutdown_signal_handlers
def test_install_shutdown_signal_handlers_unix_path():
"""On platforms where asyncio supports signal handlers (Unix), both
SIGINT and SIGTERM are registered and the helper reports success."""
loop = MagicMock(spec=asyncio.AbstractEventLoop)
handler = MagicMock()
installed = _install_shutdown_signal_handlers(loop, handler)
assert installed is True
loop.add_signal_handler.assert_any_call(signal.SIGINT, handler)
loop.add_signal_handler.assert_any_call(signal.SIGTERM, handler)
assert loop.add_signal_handler.call_count == 2
def test_install_shutdown_signal_handlers_windows_path():
"""On Windows, asyncio's ProactorEventLoop raises NotImplementedError
from add_signal_handler. The helper must swallow it and report failure
so the worker keeps running with default Python signal behavior
(regression test for issue #1411)."""
loop = MagicMock(spec=asyncio.AbstractEventLoop)
loop.add_signal_handler.side_effect = NotImplementedError
handler = MagicMock()
installed = _install_shutdown_signal_handlers(loop, handler)
assert installed is False