feat: improve SPA scraping and increase test coverage
- Add SPA support for Playwright with wait_for_network_idle and extra_wait_ms - Add BaseStore.get_spa_config() and requires_playwright() methods - Implement AliExpress SPA config with JSON price extraction patterns - Fix Amazon price parsing to prioritize whole+fraction combination - Fix AliExpress regex patterns (remove double backslashes) - Add CLI tests: detect, doctor, fetch, parse, run commands - Add API tests: auth, logs, products, scraping_logs, webhooks Tests: 417 passed, 85% coverage Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Gilles Soulier
258
tests/cli/test_run_command.py
Normal file
258
tests/cli/test_run_command.py
Normal file
@@ -0,0 +1,258 @@
|
||||
"""Tests pour la commande CLI run."""
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from pricewatch.app.cli.main import app
|
||||
from pricewatch.app.core.schema import ProductSnapshot, DebugInfo, DebugStatus, FetchMethod
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def yaml_config(tmp_path: Path) -> Path:
|
||||
"""Cree un fichier YAML de config temporaire."""
|
||||
yaml_content = """
|
||||
urls:
|
||||
- "https://www.amazon.fr/dp/B08N5WRWNW"
|
||||
options:
|
||||
use_playwright: false
|
||||
force_playwright: false
|
||||
headful: false
|
||||
save_html: false
|
||||
save_screenshot: false
|
||||
timeout_ms: 30000
|
||||
"""
|
||||
file_path = tmp_path / "test_config.yaml"
|
||||
file_path.write_text(yaml_content, encoding="utf-8")
|
||||
return file_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def output_json(tmp_path: Path) -> Path:
|
||||
"""Chemin pour le fichier JSON de sortie."""
|
||||
return tmp_path / "output.json"
|
||||
|
||||
|
||||
class TestRunCommand:
|
||||
"""Tests pour la commande run."""
|
||||
|
||||
@patch("pricewatch.app.cli.main.fetch_http")
|
||||
def test_run_http_success(self, mock_fetch, yaml_config, output_json):
|
||||
"""Run avec HTTP reussi."""
|
||||
# Mock HTTP fetch
|
||||
mock_result = MagicMock()
|
||||
mock_result.success = True
|
||||
mock_result.html = """
|
||||
<html><body>
|
||||
<span id="productTitle">Test Product</span>
|
||||
<span class="a-price-whole">299,99 €</span>
|
||||
</body></html>
|
||||
"""
|
||||
mock_result.error = None
|
||||
mock_fetch.return_value = mock_result
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
["run", "--yaml", str(yaml_config), "--out", str(output_json), "--no-db"],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert output_json.exists()
|
||||
|
||||
@patch("pricewatch.app.cli.main.fetch_http")
|
||||
@patch("pricewatch.app.cli.main.fetch_playwright")
|
||||
def test_run_http_fail_playwright_fallback(
|
||||
self, mock_pw, mock_http, yaml_config, output_json
|
||||
):
|
||||
"""Run avec fallback Playwright quand HTTP echoue."""
|
||||
# Mock HTTP fail
|
||||
mock_http_result = MagicMock()
|
||||
mock_http_result.success = False
|
||||
mock_http_result.error = "403 Forbidden"
|
||||
mock_http.return_value = mock_http_result
|
||||
|
||||
# Mock Playwright success
|
||||
mock_pw_result = MagicMock()
|
||||
mock_pw_result.success = True
|
||||
mock_pw_result.html = """
|
||||
<html><body>
|
||||
<span id="productTitle">Playwright Product</span>
|
||||
<span class="a-price-whole">199,99 €</span>
|
||||
</body></html>
|
||||
"""
|
||||
mock_pw_result.screenshot = None
|
||||
mock_pw.return_value = mock_pw_result
|
||||
|
||||
# Modifier config pour activer playwright
|
||||
yaml_content = """
|
||||
urls:
|
||||
- "https://www.amazon.fr/dp/B08N5WRWNW"
|
||||
options:
|
||||
use_playwright: true
|
||||
force_playwright: false
|
||||
headful: false
|
||||
save_html: false
|
||||
save_screenshot: false
|
||||
timeout_ms: 30000
|
||||
"""
|
||||
yaml_config.write_text(yaml_content, encoding="utf-8")
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
["run", "--yaml", str(yaml_config), "--out", str(output_json), "--no-db"],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
mock_pw.assert_called()
|
||||
|
||||
@patch("pricewatch.app.cli.main.fetch_http")
|
||||
def test_run_http_fail_no_playwright(self, mock_http, yaml_config, output_json):
|
||||
"""Run avec HTTP echoue sans Playwright."""
|
||||
mock_result = MagicMock()
|
||||
mock_result.success = False
|
||||
mock_result.error = "Connection refused"
|
||||
mock_http.return_value = mock_result
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
["run", "--yaml", str(yaml_config), "--out", str(output_json), "--no-db"],
|
||||
)
|
||||
|
||||
# Doit quand meme creer le fichier JSON (avec snapshot failed)
|
||||
assert result.exit_code == 0
|
||||
assert output_json.exists()
|
||||
|
||||
def test_run_invalid_yaml(self, tmp_path, output_json):
|
||||
"""Run avec YAML invalide echoue."""
|
||||
yaml_file = tmp_path / "invalid.yaml"
|
||||
yaml_file.write_text("invalid: [yaml: content", encoding="utf-8")
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
["run", "--yaml", str(yaml_file), "--out", str(output_json)],
|
||||
)
|
||||
|
||||
assert result.exit_code == 1
|
||||
|
||||
def test_run_with_debug(self, yaml_config, output_json):
|
||||
"""Run avec --debug active les logs."""
|
||||
with patch("pricewatch.app.cli.main.fetch_http") as mock_fetch:
|
||||
mock_result = MagicMock()
|
||||
mock_result.success = True
|
||||
mock_result.html = "<html><body>Test</body></html>"
|
||||
mock_fetch.return_value = mock_result
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"run",
|
||||
"--yaml",
|
||||
str(yaml_config),
|
||||
"--out",
|
||||
str(output_json),
|
||||
"--debug",
|
||||
"--no-db",
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
|
||||
@patch("pricewatch.app.cli.main.fetch_playwright")
|
||||
def test_run_force_playwright(self, mock_pw, tmp_path, output_json):
|
||||
"""Run avec force_playwright skip HTTP."""
|
||||
yaml_content = """
|
||||
urls:
|
||||
- "https://www.amazon.fr/dp/B08N5WRWNW"
|
||||
options:
|
||||
use_playwright: true
|
||||
force_playwright: true
|
||||
headful: false
|
||||
save_html: false
|
||||
save_screenshot: false
|
||||
timeout_ms: 30000
|
||||
"""
|
||||
yaml_file = tmp_path / "force_pw.yaml"
|
||||
yaml_file.write_text(yaml_content, encoding="utf-8")
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.success = True
|
||||
mock_result.html = "<html><body>PW content</body></html>"
|
||||
mock_result.screenshot = None
|
||||
mock_pw.return_value = mock_result
|
||||
|
||||
with patch("pricewatch.app.cli.main.fetch_http") as mock_http:
|
||||
result = runner.invoke(
|
||||
app,
|
||||
["run", "--yaml", str(yaml_file), "--out", str(output_json), "--no-db"],
|
||||
)
|
||||
|
||||
# HTTP ne doit pas etre appele
|
||||
mock_http.assert_not_called()
|
||||
mock_pw.assert_called()
|
||||
assert result.exit_code == 0
|
||||
|
||||
@patch("pricewatch.app.cli.main.fetch_http")
|
||||
def test_run_unknown_store(self, mock_fetch, tmp_path, output_json):
|
||||
"""Run avec URL de store inconnu."""
|
||||
yaml_content = """
|
||||
urls:
|
||||
- "https://www.unknown-store.com/product/123"
|
||||
options:
|
||||
use_playwright: false
|
||||
"""
|
||||
yaml_file = tmp_path / "unknown.yaml"
|
||||
yaml_file.write_text(yaml_content, encoding="utf-8")
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
["run", "--yaml", str(yaml_file), "--out", str(output_json), "--no-db"],
|
||||
)
|
||||
|
||||
# Doit continuer sans crash
|
||||
assert result.exit_code == 0
|
||||
# HTTP ne doit pas etre appele (store non trouve)
|
||||
mock_fetch.assert_not_called()
|
||||
|
||||
@patch("pricewatch.app.cli.main.fetch_http")
|
||||
@patch("pricewatch.app.cli.main.fetch_playwright")
|
||||
def test_run_with_save_screenshot(self, mock_pw, mock_http, tmp_path, output_json):
|
||||
"""Run avec save_screenshot."""
|
||||
yaml_content = """
|
||||
urls:
|
||||
- "https://www.amazon.fr/dp/B08N5WRWNW"
|
||||
options:
|
||||
use_playwright: true
|
||||
force_playwright: false
|
||||
save_screenshot: true
|
||||
timeout_ms: 30000
|
||||
"""
|
||||
yaml_file = tmp_path / "screenshot.yaml"
|
||||
yaml_file.write_text(yaml_content, encoding="utf-8")
|
||||
|
||||
# HTTP fail
|
||||
mock_http_result = MagicMock()
|
||||
mock_http_result.success = False
|
||||
mock_http_result.error = "blocked"
|
||||
mock_http.return_value = mock_http_result
|
||||
|
||||
# PW success avec screenshot
|
||||
mock_pw_result = MagicMock()
|
||||
mock_pw_result.success = True
|
||||
mock_pw_result.html = "<html><body>content</body></html>"
|
||||
mock_pw_result.screenshot = b"fake_png_data"
|
||||
mock_pw.return_value = mock_pw_result
|
||||
|
||||
with patch("pricewatch.app.core.io.save_debug_screenshot") as mock_save:
|
||||
result = runner.invoke(
|
||||
app,
|
||||
["run", "--yaml", str(yaml_file), "--out", str(output_json), "--no-db"],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
# Le screenshot doit etre sauvegarde si present
|
||||
mock_save.assert_called()
|
||||
Reference in New Issue
Block a user