add debug option
This commit is contained in:
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -27,6 +27,7 @@ class ScrapeConfig(BaseModel):
|
|||||||
locale: str
|
locale: str
|
||||||
timezone: str
|
timezone: str
|
||||||
proxy: str | None
|
proxy: str | None
|
||||||
|
debug_enabled: bool
|
||||||
|
|
||||||
|
|
||||||
class TaxonomyConfig(BaseModel):
|
class TaxonomyConfig(BaseModel):
|
||||||
|
|||||||
@@ -153,6 +153,8 @@ def main() -> None:
|
|||||||
max_tests = int(os.getenv("SCRAPE_TEST_MAX", "0"))
|
max_tests = int(os.getenv("SCRAPE_TEST_MAX", "0"))
|
||||||
headful_on_block = os.getenv("SCRAPE_TEST_HEADFUL_ON_BLOCK", "0") == "1"
|
headful_on_block = os.getenv("SCRAPE_TEST_HEADFUL_ON_BLOCK", "0") == "1"
|
||||||
wait_on_block = int(os.getenv("SCRAPE_TEST_WAIT_ON_BLOCK", "60"))
|
wait_on_block = int(os.getenv("SCRAPE_TEST_WAIT_ON_BLOCK", "60"))
|
||||||
|
# Permet de forcer le debug en tests même si désactivé dans config
|
||||||
|
debug_enabled = os.getenv("SCRAPE_TEST_DEBUG_ENABLED", str(config.scrape.debug_enabled)).lower() in ("1", "true")
|
||||||
results = []
|
results = []
|
||||||
|
|
||||||
with sync_playwright() as playwright:
|
with sync_playwright() as playwright:
|
||||||
@@ -186,7 +188,9 @@ def main() -> None:
|
|||||||
|
|
||||||
logger.info("Scraping {} ({})", test_id, url)
|
logger.info("Scraping {} ({})", test_id, url)
|
||||||
page.goto(url, wait_until="domcontentloaded", timeout=config.scrape.timeout_ms)
|
page.goto(url, wait_until="domcontentloaded", timeout=config.scrape.timeout_ms)
|
||||||
debug_files = save_debug_artifacts(page, test_id, "capture")
|
debug_files = {}
|
||||||
|
if debug_enabled:
|
||||||
|
debug_files = save_debug_artifacts(page, test_id, "capture")
|
||||||
data = extract_product_data(page, url)
|
data = extract_product_data(page, url)
|
||||||
if not data.get("titre"):
|
if not data.get("titre"):
|
||||||
logger.warning("Titre absent, suspicion de blocage pour {}", test_id)
|
logger.warning("Titre absent, suspicion de blocage pour {}", test_id)
|
||||||
@@ -197,7 +201,8 @@ def main() -> None:
|
|||||||
manual_context = manual_browser.new_context(**manual_context_kwargs)
|
manual_context = manual_browser.new_context(**manual_context_kwargs)
|
||||||
manual_page = manual_context.new_page()
|
manual_page = manual_context.new_page()
|
||||||
manual_page.goto(url, wait_until="domcontentloaded", timeout=config.scrape.timeout_ms)
|
manual_page.goto(url, wait_until="domcontentloaded", timeout=config.scrape.timeout_ms)
|
||||||
save_debug_artifacts(manual_page, test_id, "manual")
|
if debug_enabled:
|
||||||
|
save_debug_artifacts(manual_page, test_id, "manual")
|
||||||
logger.info("Résoudre le captcha puis appuyer sur Entrée.")
|
logger.info("Résoudre le captcha puis appuyer sur Entrée.")
|
||||||
try:
|
try:
|
||||||
input()
|
input()
|
||||||
|
|||||||
@@ -172,8 +172,10 @@ def _process_product(
|
|||||||
|
|
||||||
page.goto(product.url, wait_until="domcontentloaded", timeout=config.scrape.timeout_ms)
|
page.goto(product.url, wait_until="domcontentloaded", timeout=config.scrape.timeout_ms)
|
||||||
|
|
||||||
# Toujours sauvegarder les artifacts de debug
|
# Sauvegarder les artifacts de debug si activé dans la configuration
|
||||||
debug_files = _save_debug_artifacts(page, product.id, "capture")
|
debug_files = {}
|
||||||
|
if config.scrape.debug_enabled:
|
||||||
|
debug_files = _save_debug_artifacts(page, product.id, "capture")
|
||||||
|
|
||||||
# Extraire les données
|
# Extraire les données
|
||||||
data = extract_product_data(page, product.url)
|
data = extract_product_data(page, product.url)
|
||||||
|
|||||||
@@ -21,7 +21,8 @@
|
|||||||
},
|
},
|
||||||
"locale": "fr-FR",
|
"locale": "fr-FR",
|
||||||
"timezone": "Europe/Paris",
|
"timezone": "Europe/Paris",
|
||||||
"proxy": null
|
"proxy": null,
|
||||||
|
"debug_enabled": false
|
||||||
},
|
},
|
||||||
"stores_enabled": [
|
"stores_enabled": [
|
||||||
"amazon_fr"
|
"amazon_fr"
|
||||||
|
|||||||
@@ -388,6 +388,24 @@ const SettingsPage = () => {
|
|||||||
<span className="form-hint">Désactiver pour debug manuel</span>
|
<span className="form-hint">Désactiver pour debug manuel</span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div className="form-group">
|
||||||
|
<label>Mode debug</label>
|
||||||
|
<label className="checkbox-label">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={backendConfig.scrape?.debug_enabled ?? false}
|
||||||
|
onChange={(e) =>
|
||||||
|
setBackendConfig({
|
||||||
|
...backendConfig,
|
||||||
|
scrape: { ...backendConfig.scrape, debug_enabled: e.target.checked },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
Sauvegarder screenshots et HTML
|
||||||
|
</label>
|
||||||
|
<span className="form-hint">Génère des artifacts de debug dans backend/app/samples/debug/</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div className="form-group">
|
<div className="form-group">
|
||||||
<label>Timeout (ms)</label>
|
<label>Timeout (ms)</label>
|
||||||
<input
|
<input
|
||||||
|
|||||||
Reference in New Issue
Block a user