feat: improve SPA scraping and increase test coverage
- Add SPA support for Playwright with wait_for_network_idle and extra_wait_ms - Add BaseStore.get_spa_config() and requires_playwright() methods - Implement AliExpress SPA config with JSON price extraction patterns - Fix Amazon price parsing to prioritize whole+fraction combination - Fix AliExpress regex patterns (remove double backslashes) - Add CLI tests: detect, doctor, fetch, parse, run commands - Add API tests: auth, logs, products, scraping_logs, webhooks Tests: 417 passed, 85% coverage Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Gilles Soulier
@@ -29,13 +29,39 @@ logger = get_logger("stores.aliexpress")
|
||||
|
||||
|
||||
class AliexpressStore(BaseStore):
|
||||
"""Store pour AliExpress.com (marketplace chinois)."""
|
||||
"""Store pour AliExpress.com (marketplace chinois).
|
||||
|
||||
AliExpress est une SPA (Single Page Application) qui charge
|
||||
le contenu via JavaScript/AJAX. Nécessite Playwright avec
|
||||
attente du chargement dynamique.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialise le store AliExpress avec ses sélecteurs."""
|
||||
selectors_path = Path(__file__).parent / "selectors.yml"
|
||||
super().__init__(store_id="aliexpress", selectors_path=selectors_path)
|
||||
|
||||
def get_spa_config(self) -> dict:
|
||||
"""
|
||||
Configuration SPA pour AliExpress.
|
||||
|
||||
AliExpress charge les données produit (prix, titre) via AJAX.
|
||||
Il faut attendre que le réseau soit inactif ET ajouter un délai
|
||||
pour laisser le JS terminer le rendu.
|
||||
|
||||
Returns:
|
||||
Configuration Playwright pour SPA
|
||||
"""
|
||||
return {
|
||||
"wait_for_network_idle": True,
|
||||
"wait_for_selector": "h1", # Titre du produit
|
||||
"extra_wait_ms": 2000, # 2s pour le rendu JS
|
||||
}
|
||||
|
||||
def requires_playwright(self) -> bool:
|
||||
"""AliExpress nécessite Playwright pour le rendu SPA."""
|
||||
return True
|
||||
|
||||
def match(self, url: str) -> float:
|
||||
"""
|
||||
Détecte si l'URL est AliExpress.
|
||||
@@ -206,28 +232,71 @@ class AliexpressStore(BaseStore):
|
||||
Extrait le prix.
|
||||
|
||||
AliExpress n'a PAS de sélecteur CSS stable pour le prix.
|
||||
On utilise regex sur le HTML brut.
|
||||
Stratégie multi-niveaux:
|
||||
1. Chercher dans les données JSON embarquées
|
||||
2. Chercher dans les spans avec classes contenant "price"
|
||||
3. Regex sur le HTML brut
|
||||
4. Meta tags og:price
|
||||
"""
|
||||
# Pattern 1: Prix avant € (ex: "136,69 €")
|
||||
match = re.search(r"([0-9][0-9\\s.,\\u00a0\\u202f\\u2009]*)\\s*€", html)
|
||||
# Priorité 1: Extraire depuis JSON embarqué (skuActivityAmount, formattedActivityPrice)
|
||||
json_patterns = [
|
||||
r'"skuActivityAmount"\s*:\s*\{\s*"value"\s*:\s*(\d+(?:\.\d+)?)', # {"value": 123.45}
|
||||
r'"formattedActivityPrice"\s*:\s*"([0-9,.\s]+)\s*€"', # "123,45 €"
|
||||
r'"formattedActivityPrice"\s*:\s*"€\s*([0-9,.\s]+)"', # "€ 123.45"
|
||||
r'"minPrice"\s*:\s*"([0-9,.\s]+)"', # "minPrice": "123.45"
|
||||
r'"price"\s*:\s*"([0-9,.\s]+)"', # "price": "123.45"
|
||||
r'"activityAmount"\s*:\s*\{\s*"value"\s*:\s*(\d+(?:\.\d+)?)', # activityAmount.value
|
||||
]
|
||||
for pattern in json_patterns:
|
||||
match = re.search(pattern, html)
|
||||
if match:
|
||||
price = parse_price_text(match.group(1))
|
||||
if price is not None and price > 0:
|
||||
debug.notes.append(f"Prix extrait depuis JSON: {price}")
|
||||
return price
|
||||
|
||||
# Priorité 2: Chercher dans les spans/divs avec classes contenant "price"
|
||||
price_selectors = [
|
||||
'span[class*="price--current"]',
|
||||
'span[class*="price--sale"]',
|
||||
'div[class*="price--current"]',
|
||||
'span[class*="product-price"]',
|
||||
'span[class*="Price_Price"]',
|
||||
'div[class*="es--wrap"]', # Structure AliExpress spécifique
|
||||
]
|
||||
for selector in price_selectors:
|
||||
elements = soup.select(selector)
|
||||
for elem in elements:
|
||||
text = elem.get_text(strip=True)
|
||||
# Chercher un prix dans le texte
|
||||
price_match = re.search(r'(\d+[,.\s]*\d*)\s*€|€\s*(\d+[,.\s]*\d*)', text)
|
||||
if price_match:
|
||||
price_str = price_match.group(1) or price_match.group(2)
|
||||
price = parse_price_text(price_str)
|
||||
if price is not None and price > 0:
|
||||
debug.notes.append(f"Prix extrait depuis sélecteur {selector}")
|
||||
return price
|
||||
|
||||
# Priorité 3: Prix avant € (ex: "136,69€" ou "136,69 €")
|
||||
match = re.search(r'(\d+[,.\s\u00a0\u202f\u2009]*\d*)\s*€', html)
|
||||
if match:
|
||||
price = parse_price_text(match.group(1))
|
||||
if price is not None:
|
||||
if price is not None and price > 0:
|
||||
return price
|
||||
|
||||
# Pattern 2: € avant prix (ex: "€ 136.69")
|
||||
match = re.search(r"€\\s*([0-9][0-9\\s.,\\u00a0\\u202f\\u2009]*)", html)
|
||||
# Priorité 4: € avant prix (ex: "€136.69" ou "€ 136.69")
|
||||
match = re.search(r'€\s*(\d+[,.\s\u00a0\u202f\u2009]*\d*)', html)
|
||||
if match:
|
||||
price = parse_price_text(match.group(1))
|
||||
if price is not None:
|
||||
if price is not None and price > 0:
|
||||
return price
|
||||
|
||||
# Pattern 3: Chercher dans meta tags (moins fiable)
|
||||
# Priorité 5: Chercher dans meta tags (moins fiable)
|
||||
og_price = soup.find("meta", property="og:price:amount")
|
||||
if og_price:
|
||||
price_str = og_price.get("content", "")
|
||||
price = parse_price_text(price_str)
|
||||
if price is not None:
|
||||
if price is not None and price > 0:
|
||||
return price
|
||||
|
||||
debug.errors.append("Prix non trouvé")
|
||||
@@ -235,7 +304,7 @@ class AliexpressStore(BaseStore):
|
||||
|
||||
def _extract_msrp(self, html: str, debug: DebugInfo) -> Optional[float]:
|
||||
"""Extrait le prix conseille si present."""
|
||||
match = re.search(r"originalPrice\"\\s*:\\s*\"([0-9\\s.,]+)\"", html)
|
||||
match = re.search(r'originalPrice"\s*:\s*"([0-9\s.,]+)"', html)
|
||||
if match:
|
||||
price = parse_price_text(match.group(1))
|
||||
if price is not None:
|
||||
|
||||
Reference in New Issue
Block a user