382 lines
14 KiB
Python
382 lines
14 KiB
Python
import os
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
from decimal import Decimal
|
|
from psycopg2.extras import RealDictCursor
|
|
|
|
import psycopg2
|
|
import redis
|
|
from flask import Flask, jsonify, render_template_string
|
|
|
|
app = Flask(__name__)
|
|
|
|
|
|
def _env_int(name: str, default: int) -> int:
|
|
try:
|
|
return int(os.getenv(name, "") or default)
|
|
except ValueError:
|
|
return default
|
|
|
|
|
|
def get_db_connection():
|
|
return psycopg2.connect(
|
|
host=os.getenv("PW_DB_HOST", "postgres"),
|
|
port=_env_int("PW_DB_PORT", 5432),
|
|
dbname=os.getenv("PW_DB_NAME", "pricewatch"),
|
|
user=os.getenv("PW_DB_USER", "pricewatch"),
|
|
password=os.getenv("PW_DB_PASSWORD", "pricewatch"),
|
|
)
|
|
|
|
|
|
def fetch_db_metrics() -> Tuple[Dict[str, Any], Optional[str]]:
|
|
data: Dict[str, Any] = {"counts": {}, "latest_products": []}
|
|
try:
|
|
with get_db_connection() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT COUNT(*) FROM products")
|
|
data["counts"]["products"] = cur.fetchone()[0]
|
|
|
|
cur.execute("SELECT COUNT(*) FROM price_history")
|
|
data["counts"]["price_history"] = cur.fetchone()[0]
|
|
|
|
cur.execute(
|
|
"SELECT COUNT(*) FROM scraping_logs"
|
|
)
|
|
data["counts"]["scraping_logs"] = cur.fetchone()[0]
|
|
|
|
cur.execute(
|
|
"""
|
|
SELECT id, source, reference, title, last_updated_at
|
|
FROM products
|
|
ORDER BY last_updated_at DESC
|
|
LIMIT 5
|
|
"""
|
|
)
|
|
rows = cur.fetchall()
|
|
data["latest_products"] = [
|
|
{
|
|
"id": row[0],
|
|
"source": row[1],
|
|
"reference": row[2],
|
|
"title": row[3] or "Sans titre",
|
|
"updated": row[4].strftime("%Y-%m-%d %H:%M:%S")
|
|
if row[4]
|
|
else "n/a",
|
|
}
|
|
for row in rows
|
|
]
|
|
return data, None
|
|
except Exception as exc: # pragma: no cover (simple explorer)
|
|
return data, str(exc)
|
|
|
|
|
|
def _serialize_decimal(value):
|
|
if isinstance(value, Decimal):
|
|
return float(value)
|
|
return value
|
|
|
|
|
|
def fetch_products_list(limit: int = 200) -> Tuple[List[Dict[str, Any]], Optional[str]]:
|
|
rows: List[Dict[str, Any]] = []
|
|
try:
|
|
with get_db_connection() as conn:
|
|
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT
|
|
p.id,
|
|
p.source,
|
|
p.reference,
|
|
p.title,
|
|
p.url,
|
|
p.category,
|
|
p.description,
|
|
p.currency,
|
|
p.msrp,
|
|
p.last_updated_at,
|
|
ph.price,
|
|
ph.stock_status,
|
|
ph.fetch_status,
|
|
ph.fetch_method,
|
|
ph.fetched_at
|
|
FROM products p
|
|
LEFT JOIN LATERAL (
|
|
SELECT price, stock_status, fetch_status, fetch_method, fetched_at
|
|
FROM price_history
|
|
WHERE product_id = p.id
|
|
ORDER BY fetched_at DESC
|
|
LIMIT 1
|
|
) ph ON true
|
|
ORDER BY p.last_updated_at DESC
|
|
LIMIT %s
|
|
""",
|
|
(limit,),
|
|
)
|
|
fetched = cur.fetchall()
|
|
for item in fetched:
|
|
serialized = {key: _serialize_decimal(value) for key, value in item.items()}
|
|
if serialized.get("last_updated_at"):
|
|
serialized["last_updated_at"] = serialized["last_updated_at"].strftime(
|
|
"%Y-%m-%d %H:%M:%S"
|
|
)
|
|
if serialized.get("fetched_at"):
|
|
serialized["fetched_at"] = serialized["fetched_at"].strftime(
|
|
"%Y-%m-%d %H:%M:%S"
|
|
)
|
|
rows.append(serialized)
|
|
return rows, None
|
|
except Exception as exc:
|
|
return rows, str(exc)
|
|
|
|
|
|
def get_redis_client() -> redis.Redis:
|
|
return redis.Redis(
|
|
host=os.getenv("PW_REDIS_HOST", "redis"),
|
|
port=_env_int("PW_REDIS_PORT", 6379),
|
|
db=_env_int("PW_REDIS_DB", 0),
|
|
socket_connect_timeout=2,
|
|
socket_timeout=2,
|
|
)
|
|
|
|
|
|
def check_redis() -> Tuple[str, Optional[str]]:
|
|
client = get_redis_client()
|
|
try:
|
|
client.ping()
|
|
return "OK", None
|
|
except Exception as exc:
|
|
return "KO", str(exc)
|
|
|
|
|
|
TEMPLATE = """
|
|
<!doctype html>
|
|
<html lang="fr">
|
|
<head>
|
|
<meta charset="utf-8" />
|
|
<title>PriceWatch Analytics UI</title>
|
|
<style>
|
|
body { font-family: "JetBrains Mono", system-ui, monospace; background:#1f1f1b; color:#ebe0c8; margin:0; padding:32px; }
|
|
main { max-width: 960px; margin: 0 auto; }
|
|
h1 { margin-bottom: 0; }
|
|
section { margin-top: 24px; background:#282828; border:1px solid rgba(255,255,255,0.08); padding:16px; border-radius:14px; box-shadow:0 14px 30px rgba(0,0,0,0.35); }
|
|
table { width:100%; border-collapse:collapse; margin-top:12px; }
|
|
th, td { text-align:left; padding:6px 8px; border-bottom:1px solid rgba(255,255,255,0.08); }
|
|
.status { display:inline-flex; align-items:center; gap:6px; font-size:14px; padding:4px 10px; border-radius:999px; background:rgba(255,255,255,0.05); }
|
|
.status.ok { background:rgba(184,187,38,0.15); }
|
|
.status.ko { background:rgba(251,73,52,0.2); }
|
|
.muted { color:rgba(255,255,255,0.5); font-size:13px; }
|
|
.browser-panel { margin-top: 16px; display: flex; flex-direction: column; gap: 12px; }
|
|
.browser-controls { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; }
|
|
.browser-controls button { border-radius: 8px; border: 1px solid rgba(255,255,255,0.12); background: rgba(255,255,255,0.04); color: inherit; padding: 6px 12px; cursor: pointer; transition: transform 0.15s ease; }
|
|
.browser-controls button:hover { transform: translateY(-1px); }
|
|
.browser-display { padding: 12px; border-radius: 12px; background: rgba(255,255,255,0.02); border: 1px solid rgba(255,255,255,0.08); min-height: 150px; font-size: 0.85rem; }
|
|
.browser-display dt { font-weight: 700; }
|
|
.browser-display dd { margin: 0 0 8px 0; }
|
|
.browser-indicator { font-size: 0.9rem; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<main>
|
|
<header>
|
|
<h1>PriceWatch Analytics UI</h1>
|
|
<p class="muted">PostgreSQL : {{ db_status }} · Redis : {{ redis_status }}</p>
|
|
</header>
|
|
<section>
|
|
<h2>Vue rapide</h2>
|
|
<div class="status {{ 'ok' if db_error is none else 'ko' }}">
|
|
Base : {{ db_status }}
|
|
</div>
|
|
<div class="status {{ 'ok' if redis_status == 'OK' else 'ko' }}">
|
|
Redis : {{ redis_status }}
|
|
</div>
|
|
{% if db_error or redis_error %}
|
|
<p class="muted">Erreurs : {{ db_error or '' }} {{ redis_error or '' }}</p>
|
|
{% endif %}
|
|
</section>
|
|
<section>
|
|
<h2>Stats métier</h2>
|
|
<table>
|
|
<tr><th>Produits</th><td>{{ metrics.counts.products }}</td></tr>
|
|
<tr><th>Historique prix</th><td>{{ metrics.counts.price_history }}</td></tr>
|
|
<tr><th>Logs de scraping</th><td>{{ metrics.counts.scraping_logs }}</td></tr>
|
|
</table>
|
|
</section>
|
|
<section>
|
|
<h2>Produits récemment mis à jour</h2>
|
|
{% if metrics.latest_products %}
|
|
<table>
|
|
<thead>
|
|
<tr><th>ID</th><th>Store</th><th>Référence</th><th>Révision</th><th>Mis à jour</th></tr>
|
|
</thead>
|
|
<tbody>
|
|
{% for item in metrics.latest_products %}
|
|
<tr>
|
|
<td>{{ item.id }}</td>
|
|
<td>{{ item.source }}</td>
|
|
<td>{{ item.reference }}</td>
|
|
<td>{{ item.title[:40] }}{% if item.title|length > 40 %}…{% endif %}</td>
|
|
<td>{{ item.updated }}</td>
|
|
</tr>
|
|
{% endfor %}
|
|
</tbody>
|
|
</table>
|
|
{% else %}
|
|
<p class="muted">Aucun produit enregistré.</p>
|
|
{% endif %}
|
|
</section>
|
|
<section>
|
|
<h2>Parcourir la base (produits)</h2>
|
|
<div class="browser-panel">
|
|
<div class="browser-controls">
|
|
<button id="load-products">Charger les produits</button>
|
|
<button id="product-prev" disabled>Précédent</button>
|
|
<button id="product-next" disabled>Suivant</button>
|
|
<strong class="browser-indicator" id="product-indicator">0 / 0</strong>
|
|
<span class="muted" id="product-message"></span>
|
|
</div>
|
|
<dl class="browser-display" id="product-details">
|
|
<dt data-field="title">Titre</dt>
|
|
<dd id="product-title">-</dd>
|
|
<dt data-field="store">Store</dt>
|
|
<dd data-field="store">-</dd>
|
|
<dt data-field="reference">Référence</dt>
|
|
<dd data-field="reference">-</dd>
|
|
<dt data-field="price">Dernier prix</dt>
|
|
<dd data-field="price">-</dd>
|
|
<dt data-field="currency">Devise</dt>
|
|
<dd data-field="currency">-</dd>
|
|
<dt data-field="msrp">Prix conseillé</dt>
|
|
<dd data-field="msrp">-</dd>
|
|
<dt data-field="stock_status">Stock</dt>
|
|
<dd data-field="stock_status">-</dd>
|
|
<dt data-field="category">Catégorie</dt>
|
|
<dd data-field="category">-</dd>
|
|
<dt data-field="description">Description</dt>
|
|
<dd data-field="description">-</dd>
|
|
<dt data-field="last_updated_at">Dernière mise à jour</dt>
|
|
<dd data-field="last_updated_at">-</dd>
|
|
<dt data-field="fetched_at">Historique dernier scrap</dt>
|
|
<dd data-field="fetched_at">-</dd>
|
|
</dl>
|
|
</div>
|
|
</section>
|
|
</main>
|
|
<script>
|
|
document.addEventListener("DOMContentLoaded", () => {
|
|
const loadBtn = document.getElementById("load-products");
|
|
const prevBtn = document.getElementById("product-prev");
|
|
const nextBtn = document.getElementById("product-next");
|
|
const indicator = document.getElementById("product-indicator");
|
|
const message = document.getElementById("product-message");
|
|
const titleEl = document.getElementById("product-title");
|
|
const fields = Array.from(document.querySelectorAll("[data-field]")).reduce((acc, el) => {
|
|
acc[el.getAttribute("data-field")] = el;
|
|
return acc;
|
|
}, {});
|
|
let products = [];
|
|
let cursor = 0;
|
|
|
|
const setStatus = (text) => {
|
|
message.textContent = text || "";
|
|
};
|
|
|
|
const renderProduct = () => {
|
|
if (!products.length) {
|
|
indicator.textContent = "0 / 0";
|
|
titleEl.textContent = "-";
|
|
Object.values(fields).forEach((el) => (el.textContent = "-"));
|
|
prevBtn.disabled = true;
|
|
nextBtn.disabled = true;
|
|
return;
|
|
}
|
|
const current = products[cursor];
|
|
indicator.textContent = `${cursor + 1} / ${products.length}`;
|
|
titleEl.textContent = current.title || "Sans titre";
|
|
const mapField = {
|
|
store: current.source,
|
|
reference: current.reference,
|
|
price: current.price !== null && current.price !== undefined ? current.price : "n/a",
|
|
currency: current.currency || "EUR",
|
|
msrp: current.msrp || "-",
|
|
stock_status: current.stock_status || "n/a",
|
|
category: current.category || "n/a",
|
|
description: (current.description || "n/a").slice(0, 200),
|
|
last_updated_at: current.last_updated_at || "n/a",
|
|
fetched_at: current.fetched_at || "n/a",
|
|
};
|
|
Object.entries(mapField).forEach(([key, value]) => {
|
|
if (fields[key]) {
|
|
fields[key].textContent = value;
|
|
}
|
|
});
|
|
prevBtn.disabled = cursor === 0;
|
|
nextBtn.disabled = cursor >= products.length - 1;
|
|
};
|
|
|
|
const fetchProducts = async () => {
|
|
setStatus("Chargement…");
|
|
try {
|
|
const response = await fetch("/products.json");
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP ${response.status}`);
|
|
}
|
|
const data = await response.json();
|
|
if (!Array.isArray(data)) {
|
|
throw new Error("Réponse invalide");
|
|
}
|
|
products = data;
|
|
cursor = 0;
|
|
setStatus(`Chargé ${products.length} produit(s)`);
|
|
renderProduct();
|
|
} catch (err) {
|
|
setStatus(`Erreur: ${err.message}`);
|
|
products = [];
|
|
renderProduct();
|
|
}
|
|
};
|
|
|
|
loadBtn.addEventListener("click", fetchProducts);
|
|
prevBtn.addEventListener("click", () => {
|
|
if (cursor > 0) {
|
|
cursor -= 1;
|
|
renderProduct();
|
|
}
|
|
});
|
|
nextBtn.addEventListener("click", () => {
|
|
if (cursor + 1 < products.length) {
|
|
cursor += 1;
|
|
renderProduct();
|
|
}
|
|
});
|
|
});
|
|
</script>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
@app.route("/")
|
|
def root():
|
|
metrics, db_error = fetch_db_metrics()
|
|
redis_status, redis_error = check_redis()
|
|
return render_template_string(
|
|
TEMPLATE,
|
|
metrics=metrics,
|
|
db_status="connecté" if db_error is None else "erreur",
|
|
db_error=db_error,
|
|
redis_status=redis_status,
|
|
redis_error=redis_error,
|
|
)
|
|
|
|
|
|
@app.route("/products.json")
|
|
def products_json():
|
|
products, error = fetch_products_list()
|
|
if error:
|
|
return jsonify({"error": error}), 500
|
|
return jsonify(products)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
app.run(host="0.0.0.0", port=80)
|