add debug option
This commit is contained in:
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -27,6 +27,7 @@ class ScrapeConfig(BaseModel):
|
||||
locale: str
|
||||
timezone: str
|
||||
proxy: str | None
|
||||
debug_enabled: bool
|
||||
|
||||
|
||||
class TaxonomyConfig(BaseModel):
|
||||
|
||||
@@ -153,6 +153,8 @@ def main() -> None:
|
||||
max_tests = int(os.getenv("SCRAPE_TEST_MAX", "0"))
|
||||
headful_on_block = os.getenv("SCRAPE_TEST_HEADFUL_ON_BLOCK", "0") == "1"
|
||||
wait_on_block = int(os.getenv("SCRAPE_TEST_WAIT_ON_BLOCK", "60"))
|
||||
# Permet de forcer le debug en tests même si désactivé dans config
|
||||
debug_enabled = os.getenv("SCRAPE_TEST_DEBUG_ENABLED", str(config.scrape.debug_enabled)).lower() in ("1", "true")
|
||||
results = []
|
||||
|
||||
with sync_playwright() as playwright:
|
||||
@@ -186,7 +188,9 @@ def main() -> None:
|
||||
|
||||
logger.info("Scraping {} ({})", test_id, url)
|
||||
page.goto(url, wait_until="domcontentloaded", timeout=config.scrape.timeout_ms)
|
||||
debug_files = save_debug_artifacts(page, test_id, "capture")
|
||||
debug_files = {}
|
||||
if debug_enabled:
|
||||
debug_files = save_debug_artifacts(page, test_id, "capture")
|
||||
data = extract_product_data(page, url)
|
||||
if not data.get("titre"):
|
||||
logger.warning("Titre absent, suspicion de blocage pour {}", test_id)
|
||||
@@ -197,7 +201,8 @@ def main() -> None:
|
||||
manual_context = manual_browser.new_context(**manual_context_kwargs)
|
||||
manual_page = manual_context.new_page()
|
||||
manual_page.goto(url, wait_until="domcontentloaded", timeout=config.scrape.timeout_ms)
|
||||
save_debug_artifacts(manual_page, test_id, "manual")
|
||||
if debug_enabled:
|
||||
save_debug_artifacts(manual_page, test_id, "manual")
|
||||
logger.info("Résoudre le captcha puis appuyer sur Entrée.")
|
||||
try:
|
||||
input()
|
||||
|
||||
@@ -172,8 +172,10 @@ def _process_product(
|
||||
|
||||
page.goto(product.url, wait_until="domcontentloaded", timeout=config.scrape.timeout_ms)
|
||||
|
||||
# Toujours sauvegarder les artifacts de debug
|
||||
debug_files = _save_debug_artifacts(page, product.id, "capture")
|
||||
# Sauvegarder les artifacts de debug si activé dans la configuration
|
||||
debug_files = {}
|
||||
if config.scrape.debug_enabled:
|
||||
debug_files = _save_debug_artifacts(page, product.id, "capture")
|
||||
|
||||
# Extraire les données
|
||||
data = extract_product_data(page, product.url)
|
||||
|
||||
@@ -21,7 +21,8 @@
|
||||
},
|
||||
"locale": "fr-FR",
|
||||
"timezone": "Europe/Paris",
|
||||
"proxy": null
|
||||
"proxy": null,
|
||||
"debug_enabled": false
|
||||
},
|
||||
"stores_enabled": [
|
||||
"amazon_fr"
|
||||
|
||||
@@ -388,6 +388,24 @@ const SettingsPage = () => {
|
||||
<span className="form-hint">Désactiver pour debug manuel</span>
|
||||
</div>
|
||||
|
||||
<div className="form-group">
|
||||
<label>Mode debug</label>
|
||||
<label className="checkbox-label">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={backendConfig.scrape?.debug_enabled ?? false}
|
||||
onChange={(e) =>
|
||||
setBackendConfig({
|
||||
...backendConfig,
|
||||
scrape: { ...backendConfig.scrape, debug_enabled: e.target.checked },
|
||||
})
|
||||
}
|
||||
/>
|
||||
Sauvegarder screenshots et HTML
|
||||
</label>
|
||||
<span className="form-hint">Génère des artifacts de debug dans backend/app/samples/debug/</span>
|
||||
</div>
|
||||
|
||||
<div className="form-group">
|
||||
<label>Timeout (ms)</label>
|
||||
<input
|
||||
|
||||
Reference in New Issue
Block a user