This commit is contained in:
Gilles Soulier
2026-01-14 21:54:55 +01:00
parent c91c0f1fc9
commit d0b73b9319
140 changed files with 5822 additions and 161 deletions

75
pricewatch/app/tasks/scheduler.py Executable file → Normal file
View File

@@ -9,6 +9,8 @@ from datetime import datetime, timedelta, timezone
from typing import Optional
import redis
from redis.exceptions import ConnectionError as RedisConnectionError
from redis.exceptions import RedisError, TimeoutError as RedisTimeoutError
from rq import Queue
from rq_scheduler import Scheduler
@@ -19,6 +21,15 @@ from pricewatch.app.tasks.scrape import scrape_product
logger = get_logger("tasks.scheduler")
class RedisUnavailableError(Exception):
"""Exception levee quand Redis n'est pas disponible."""
def __init__(self, message: str = "Redis non disponible", cause: Optional[Exception] = None):
self.message = message
self.cause = cause
super().__init__(self.message)
@dataclass
class ScheduledJobInfo:
"""Infos de retour pour un job planifie."""
@@ -27,14 +38,72 @@ class ScheduledJobInfo:
next_run: datetime
def check_redis_connection(redis_url: str) -> bool:
"""
Verifie si Redis est accessible.
Returns:
True si Redis repond, False sinon.
"""
try:
conn = redis.from_url(redis_url)
conn.ping()
return True
except (RedisConnectionError, RedisTimeoutError, RedisError) as e:
logger.debug(f"Redis ping echoue: {e}")
return False
class ScrapingScheduler:
"""Scheduler pour les jobs de scraping avec RQ."""
def __init__(self, config: Optional[AppConfig] = None, queue_name: str = "default") -> None:
self.config = config or get_config()
self.redis = redis.from_url(self.config.redis.url)
self.queue = Queue(queue_name, connection=self.redis)
self.scheduler = Scheduler(queue=self.queue, connection=self.redis)
self._queue_name = queue_name
self._redis: Optional[redis.Redis] = None
self._queue: Optional[Queue] = None
self._scheduler: Optional[Scheduler] = None
def _ensure_connected(self) -> None:
"""Etablit la connexion Redis si necessaire, leve RedisUnavailableError si echec."""
if self._redis is not None:
return
try:
self._redis = redis.from_url(self.config.redis.url)
# Ping pour verifier la connexion
self._redis.ping()
self._queue = Queue(self._queue_name, connection=self._redis)
self._scheduler = Scheduler(queue=self._queue, connection=self._redis)
logger.debug(f"Connexion Redis etablie: {self.config.redis.url}")
except (RedisConnectionError, RedisTimeoutError) as e:
self._redis = None
msg = f"Impossible de se connecter a Redis ({self.config.redis.url}): {e}"
logger.error(msg)
raise RedisUnavailableError(msg, cause=e) from e
except RedisError as e:
self._redis = None
msg = f"Erreur Redis: {e}"
logger.error(msg)
raise RedisUnavailableError(msg, cause=e) from e
@property
def redis(self) -> redis.Redis:
"""Acces a la connexion Redis (lazy)."""
self._ensure_connected()
return self._redis # type: ignore
@property
def queue(self) -> Queue:
"""Acces a la queue RQ (lazy)."""
self._ensure_connected()
return self._queue # type: ignore
@property
def scheduler(self) -> Scheduler:
"""Acces au scheduler RQ (lazy)."""
self._ensure_connected()
return self._scheduler # type: ignore
def enqueue_immediate(
self,