gilles/scrap
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

before claude

Browse Source
This commit is contained in:
Gilles Soulier
2026-01-18 06:26:17 +01:00
parent dc19315e5d
commit 740c3d7516
60 changed files with 3815 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
pricewatch/app/api/main.py
View File

@@ -22,6 +22,10 @@ from sqlalchemy.orm import Session
from pricewatch.app.api.schemas import (
BackendLogEntry,
ClassificationOptionsOut,
ClassificationRuleCreate,
ClassificationRuleOut,
ClassificationRuleUpdate,
EnqueueRequest,
EnqueueResponse,
HealthStatus,
@@ -52,7 +56,8 @@ from pricewatch.app.core.config import get_config
from pricewatch.app.core.logging import get_logger
from pricewatch.app.core.schema import ProductSnapshot
from pricewatch.app.db.connection import check_db_connection, get_session
from pricewatch.app.db.models import PriceHistory, Product, ScrapingLog, Webhook
from pricewatch.app.db.models import ClassificationRule, PriceHistory, Product, ScrapingLog, Webhook
from pricewatch.app.db.repository import ProductRepository
from pricewatch.app.scraping.pipeline import ScrapingPipeline
from pricewatch.app.tasks.scrape import scrape_product
from pricewatch.app.tasks.scheduler import RedisUnavailableError, check_redis_connection, ScrapingScheduler
@@ -188,6 +193,7 @@ def create_product(
url=payload.url,
title=payload.title,
category=payload.category,
type=payload.type,
description=payload.description,
currency=payload.currency,
msrp=payload.msrp,
@@ -241,6 +247,129 @@ def update_product(
return _product_to_out(session, product)
@app.get(
"/classification/rules",
response_model=list[ClassificationRuleOut],
dependencies=[Depends(require_token)],
)
def list_classification_rules(
session: Session = Depends(get_db_session),
) -> list[ClassificationRuleOut]:
"""Liste les regles de classification."""
rules = (
session.query(ClassificationRule)
.order_by(ClassificationRule.sort_order, ClassificationRule.id)
.all()
)
return [
ClassificationRuleOut(
id=rule.id,
category=rule.category,
type=rule.type,
keywords=rule.keywords or [],
sort_order=rule.sort_order,
is_active=rule.is_active,
)
for rule in rules
]
@app.post(
"/classification/rules",
response_model=ClassificationRuleOut,
dependencies=[Depends(require_token)],
)
def create_classification_rule(
payload: ClassificationRuleCreate,
session: Session = Depends(get_db_session),
) -> ClassificationRuleOut:
"""Cree une regle de classification."""
rule = ClassificationRule(
category=payload.category,
type=payload.type,
keywords=payload.keywords,
sort_order=payload.sort_order or 0,
is_active=True if payload.is_active is None else payload.is_active,
)
session.add(rule)
session.commit()
session.refresh(rule)
return ClassificationRuleOut(
id=rule.id,
category=rule.category,
type=rule.type,
keywords=rule.keywords or [],
sort_order=rule.sort_order,
is_active=rule.is_active,
)
@app.patch(
"/classification/rules/{rule_id}",
response_model=ClassificationRuleOut,
dependencies=[Depends(require_token)],
)
def update_classification_rule(
rule_id: int,
payload: ClassificationRuleUpdate,
session: Session = Depends(get_db_session),
) -> ClassificationRuleOut:
"""Met a jour une regle de classification."""
rule = session.query(ClassificationRule).filter(ClassificationRule.id == rule_id).one_or_none()
if not rule:
raise HTTPException(status_code=404, detail="Regle non trouvee")
updates = payload.model_dump(exclude_unset=True)
for key, value in updates.items():
setattr(rule, key, value)
session.commit()
session.refresh(rule)
return ClassificationRuleOut(
id=rule.id,
category=rule.category,
type=rule.type,
keywords=rule.keywords or [],
sort_order=rule.sort_order,
is_active=rule.is_active,
)
@app.delete(
"/classification/rules/{rule_id}",
dependencies=[Depends(require_token)],
)
def delete_classification_rule(
rule_id: int,
session: Session = Depends(get_db_session),
) -> dict[str, str]:
"""Supprime une regle de classification."""
rule = session.query(ClassificationRule).filter(ClassificationRule.id == rule_id).one_or_none()
if not rule:
raise HTTPException(status_code=404, detail="Regle non trouvee")
session.delete(rule)
session.commit()
return {"status": "deleted"}
@app.get(
"/classification/options",
response_model=ClassificationOptionsOut,
dependencies=[Depends(require_token)],
)
def get_classification_options(
session: Session = Depends(get_db_session),
) -> ClassificationOptionsOut:
"""Expose la liste des categories et types issus des regles actives."""
rules = (
session.query(ClassificationRule)
.filter(ClassificationRule.is_active == True)
.order_by(ClassificationRule.sort_order, ClassificationRule.id)
.all()
)
categories = sorted({rule.category for rule in rules if rule.category})
types = sorted({rule.type for rule in rules if rule.type})
return ClassificationOptionsOut(categories=categories, types=types)
@app.delete("/products/{product_id}", dependencies=[Depends(require_token)])
def delete_product(
product_id: int,
@@ -703,6 +832,13 @@ def preview_scrape(payload: ScrapePreviewRequest) -> ScrapePreviewResponse:
if snapshot is None:
_add_backend_log("ERROR", f"Preview scraping KO: {payload.url}")
return ScrapePreviewResponse(success=False, snapshot=None, error=result.get("error"))
config = get_config()
if config.enable_db:
try:
with get_session(config) as session:
ProductRepository(session).apply_classification(snapshot)
except Exception as exc:
snapshot.add_note(f"Classification ignoree: {exc}")
return ScrapePreviewResponse(
success=bool(result.get("success")),
snapshot=snapshot.model_dump(mode="json"),
@@ -719,7 +855,9 @@ def commit_scrape(payload: ScrapeCommitRequest) -> ScrapeCommitResponse:
_add_backend_log("ERROR", "Commit scraping KO: snapshot invalide")
raise HTTPException(status_code=400, detail="Snapshot invalide") from exc
product_id = ScrapingPipeline(config=get_config()).process_snapshot(snapshot, save_to_db=True)
product_id = ScrapingPipeline(config=get_config()).process_snapshot(
snapshot, save_to_db=True, apply_classification=False
)
_add_backend_log("INFO", f"Commit scraping OK: product_id={product_id}")
return ScrapeCommitResponse(success=True, product_id=product_id)
@@ -808,12 +946,9 @@ def _product_to_out(session: Session, product: Product) -> ProductOut:
)
images = [image.image_url for image in product.images]
specs = {spec.spec_key: spec.spec_value for spec in product.specs}
discount_amount = None
discount_percent = None
if latest and latest.price is not None and product.msrp:
discount_amount = float(product.msrp) - float(latest.price)
if product.msrp > 0:
discount_percent = (discount_amount / float(product.msrp)) * 100
main_image = images[0] if images else None
gallery_images = images[1:] if len(images) > 1 else []
asin = product.reference if product.source == "amazon" else None
history_rows = (
session.query(PriceHistory)
.filter(PriceHistory.product_id == product.id, PriceHistory.price != None)
@@ -830,12 +965,23 @@ def _product_to_out(session: Session, product: Product) -> ProductOut:
id=product.id,
source=product.source,
reference=product.reference,
asin=asin,
url=product.url,
title=product.title,
category=product.category,
type=product.type,
description=product.description,
currency=product.currency,
msrp=float(product.msrp) if product.msrp is not None else None,
rating_value=float(product.rating_value) if product.rating_value is not None else None,
rating_count=product.rating_count,
amazon_choice=product.amazon_choice,
amazon_choice_label=product.amazon_choice_label,
discount_text=product.discount_text,
stock_text=product.stock_text,
in_stock=product.in_stock,
model_number=product.model_number,
model_name=product.model_name,
first_seen_at=product.first_seen_at,
last_updated_at=product.last_updated_at,
latest_price=float(latest.price) if latest and latest.price is not None else None,
@@ -845,9 +991,11 @@ def _product_to_out(session: Session, product: Product) -> ProductOut:
latest_stock_status=latest.stock_status if latest else None,
latest_fetched_at=latest.fetched_at if latest else None,
images=images,
main_image=main_image,
gallery_images=gallery_images,
specs=specs,
discount_amount=discount_amount,
discount_percent=discount_percent,
discount_amount=None,
discount_percent=None,
history=history_points,
)

45
pricewatch/app/api/schemas.py
View File

@@ -22,12 +22,23 @@ class ProductOut(BaseModel):
id: int
source: str
reference: str
asin: Optional[str] = None
url: str
title: Optional[str] = None
category: Optional[str] = None
type: Optional[str] = None
description: Optional[str] = None
currency: Optional[str] = None
msrp: Optional[float] = None
rating_value: Optional[float] = None
rating_count: Optional[int] = None
amazon_choice: Optional[bool] = None
amazon_choice_label: Optional[str] = None
discount_text: Optional[str] = None
stock_text: Optional[str] = None
in_stock: Optional[bool] = None
model_number: Optional[str] = None
model_name: Optional[str] = None
first_seen_at: datetime
last_updated_at: datetime
latest_price: Optional[float] = None
@@ -35,6 +46,8 @@ class ProductOut(BaseModel):
latest_stock_status: Optional[str] = None
latest_fetched_at: Optional[datetime] = None
images: list[str] = []
main_image: Optional[str] = None
gallery_images: list[str] = []
specs: dict[str, str] = {}
discount_amount: Optional[float] = None
discount_percent: Optional[float] = None
@@ -47,6 +60,7 @@ class ProductCreate(BaseModel):
url: str
title: Optional[str] = None
category: Optional[str] = None
type: Optional[str] = None
description: Optional[str] = None
currency: Optional[str] = None
msrp: Optional[float] = None
@@ -56,6 +70,7 @@ class ProductUpdate(BaseModel):
url: Optional[str] = None
title: Optional[str] = None
category: Optional[str] = None
type: Optional[str] = None
description: Optional[str] = None
currency: Optional[str] = None
msrp: Optional[float] = None
@@ -208,6 +223,36 @@ class VersionResponse(BaseModel):
api_version: str
class ClassificationRuleOut(BaseModel):
id: int
category: Optional[str] = None
type: Optional[str] = None
keywords: list[str] = Field(default_factory=list)
sort_order: int = 0
is_active: bool = True
class ClassificationRuleCreate(BaseModel):
category: Optional[str] = None
type: Optional[str] = None
keywords: list[str] = Field(default_factory=list)
sort_order: Optional[int] = 0
is_active: Optional[bool] = True
class ClassificationRuleUpdate(BaseModel):
category: Optional[str] = None
type: Optional[str] = None
keywords: Optional[list[str]] = None
sort_order: Optional[int] = None
is_active: Optional[bool] = None
class ClassificationOptionsOut(BaseModel):
categories: list[str] = Field(default_factory=list)
types: list[str] = Field(default_factory=list)
class BackendLogEntry(BaseModel):
time: datetime
level: str