leopold.stroblandClaude Opus 4.8
bdcb17285c
- Poll the public download page for "Aktuelle Version <nr> vom <date>", report changes through the normal change feed and notification mail, and show the current version as a card on the overview. Runs on its own interval (default 30 min); the first run only records a baseline. - Archive epirentServer.zip / epirentClient.zip per version number so we keep a download cache and history: streamed to disk with SHA-256 and size, fetched in a background thread so ticket polling is unaffected, with a free-space check before downloading (~1.1 GB per version). - Retention is configurable via EPIRENT_DOWNLOAD_KEEP, where 0 means unlimited; a successful archive is reported as a change and by mail. - New Downloads page listing the archived versions with checksums and links to fetch the cached files. - Notification lines now omit the arrow when there is no previous value and drop the "#" for entries without a ticket. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
257 lines
8.6 KiB
Python
257 lines
8.6 KiB
Python
"""Download-Archiv: sichert die EPIRent-Pakete je Versionsnummer.
|
||
|
||
Die Dateien sind groß (Server ~923 MB, Client ~218 MB, zusammen ~1,1 GB pro
|
||
Version). Deshalb:
|
||
* Download läuft im Hintergrund-Thread (Ticketabruf blockiert nicht),
|
||
* gestreamt auf Platte (kein Speicherverbrauch),
|
||
* Platzprüfung vorab,
|
||
* Aufbewahrung begrenzt (EPIRENT_DOWNLOAD_KEEP, Standard 3 Stände).
|
||
"""
|
||
import hashlib
|
||
import logging
|
||
import shutil
|
||
import threading
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from urllib.parse import urlparse
|
||
|
||
import requests
|
||
|
||
from . import config, db, notify
|
||
|
||
log = logging.getLogger("episupport.downloads")
|
||
|
||
DOWNLOAD_DIR = config.DATA_DIR / "downloads"
|
||
_lock = threading.Lock()
|
||
_running = False
|
||
|
||
|
||
def _now() -> str:
|
||
return datetime.now(timezone.utc).isoformat()
|
||
|
||
|
||
def filename_for(url: str) -> str:
|
||
name = Path(urlparse(url).path).name
|
||
return name or "download.bin"
|
||
|
||
|
||
def version_dir(version: str) -> Path:
|
||
# Versionsnummern sind rein numerisch – zur Sicherheit trotzdem säubern
|
||
safe = "".join(ch for ch in str(version) if ch.isalnum() or ch in "._-")
|
||
return DOWNLOAD_DIR / (safe or "unbekannt")
|
||
|
||
|
||
def archived(version: str) -> list[dict]:
|
||
with db.get_conn() as conn:
|
||
rows = conn.execute(
|
||
"SELECT * FROM downloads WHERE version=? ORDER BY filename",
|
||
(str(version),)).fetchall()
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
def is_complete(version: str) -> bool:
|
||
"""Sind alle konfigurierten Dateien für diese Version vorhanden?"""
|
||
want = {filename_for(u) for u in config.EPIRENT_DOWNLOAD_URLS}
|
||
have = {r["filename"] for r in archived(version)
|
||
if (version_dir(version) / r["filename"]).exists()}
|
||
return bool(want) and want.issubset(have)
|
||
|
||
|
||
def history() -> list[dict]:
|
||
"""Alle archivierten Versionen mit Dateien und Gesamtgröße."""
|
||
with db.get_conn() as conn:
|
||
rows = conn.execute(
|
||
"SELECT * FROM downloads ORDER BY downloaded_at DESC, filename"
|
||
).fetchall()
|
||
by_version: dict[str, dict] = {}
|
||
for r in rows:
|
||
v = by_version.setdefault(r["version"], {
|
||
"version": r["version"], "files": [], "total": 0,
|
||
"downloaded_at": r["downloaded_at"]})
|
||
v["files"].append(dict(r))
|
||
v["total"] += r["size"] or 0
|
||
return list(by_version.values())
|
||
|
||
|
||
def human_size(n) -> str:
|
||
if not n:
|
||
return "–"
|
||
n = float(n)
|
||
for unit in ("B", "KB", "MB", "GB"):
|
||
if n < 1024 or unit == "GB":
|
||
return f"{n:.0f} {unit}" if unit in ("B", "KB") else f"{n:.1f} {unit}".replace(".", ",")
|
||
n /= 1024
|
||
return f"{n:.1f} GB"
|
||
|
||
|
||
# ------------------------------------------------------------------ Download
|
||
def _remote_info(url: str) -> tuple[int, str | None]:
|
||
"""(Größe, Last-Modified) per HEAD ermitteln."""
|
||
r = requests.head(url, timeout=30, allow_redirects=True,
|
||
headers={"User-Agent": "episupport-monitor/1.0"})
|
||
r.raise_for_status()
|
||
return int(r.headers.get("Content-Length") or 0), r.headers.get("Last-Modified")
|
||
|
||
|
||
def _download(url: str, dest: Path) -> tuple[int, str]:
|
||
"""Streamend herunterladen; gibt (Größe, sha256) zurück."""
|
||
tmp = dest.with_suffix(dest.suffix + ".part")
|
||
h = hashlib.sha256()
|
||
size = 0
|
||
with requests.get(url, stream=True, timeout=(30, 300),
|
||
headers={"User-Agent": "episupport-monitor/1.0"}) as r:
|
||
r.raise_for_status()
|
||
with open(tmp, "wb") as f:
|
||
for chunk in r.iter_content(chunk_size=1024 * 1024):
|
||
if not chunk:
|
||
continue
|
||
f.write(chunk)
|
||
h.update(chunk)
|
||
size += len(chunk)
|
||
tmp.replace(dest)
|
||
return size, h.hexdigest()
|
||
|
||
|
||
def _enough_space(needed: int) -> bool:
|
||
DOWNLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||
free = shutil.disk_usage(DOWNLOAD_DIR).free
|
||
if free < needed * 1.2:
|
||
log.error("Zu wenig Speicherplatz: benötigt ~%s, frei %s.",
|
||
human_size(needed), human_size(free))
|
||
return False
|
||
return True
|
||
|
||
|
||
def fetch_version(version: str) -> list[dict]:
|
||
"""Fehlende Dateien dieser Version holen. Gibt die neu geholten zurück."""
|
||
urls = config.EPIRENT_DOWNLOAD_URLS
|
||
if not urls:
|
||
return []
|
||
target = version_dir(version)
|
||
target.mkdir(parents=True, exist_ok=True)
|
||
|
||
todo = []
|
||
for url in urls:
|
||
name = filename_for(url)
|
||
if (target / name).exists() and any(
|
||
r["filename"] == name for r in archived(version)):
|
||
continue
|
||
todo.append((url, name))
|
||
if not todo:
|
||
return []
|
||
|
||
# Platzbedarf vorab prüfen
|
||
needed = 0
|
||
infos = {}
|
||
for url, name in todo:
|
||
try:
|
||
size, modified = _remote_info(url)
|
||
except requests.RequestException as e:
|
||
log.warning("HEAD fehlgeschlagen für %s: %s", url, e)
|
||
size, modified = 0, None
|
||
infos[name] = (size, modified)
|
||
needed += size
|
||
if needed and not _enough_space(needed):
|
||
return []
|
||
|
||
got = []
|
||
for url, name in todo:
|
||
dest = target / name
|
||
try:
|
||
log.info("Lade %s (%s) für Version %s …", name,
|
||
human_size(infos[name][0]), version)
|
||
size, digest = _download(url, dest)
|
||
with db.get_conn() as conn:
|
||
conn.execute(
|
||
"INSERT OR REPLACE INTO downloads (version, filename, url, "
|
||
"size, sha256, remote_modified, downloaded_at) "
|
||
"VALUES (?,?,?,?,?,?,?)",
|
||
(str(version), name, url, size, digest,
|
||
infos[name][1], _now()))
|
||
got.append({"filename": name, "size": size, "sha256": digest})
|
||
log.info("Fertig: %s (%s)", name, human_size(size))
|
||
except (requests.RequestException, OSError) as e:
|
||
log.error("Download %s fehlgeschlagen: %s", url, e)
|
||
try:
|
||
(target / (name + ".part")).unlink(missing_ok=True)
|
||
except OSError:
|
||
pass
|
||
return got
|
||
|
||
|
||
def cleanup(keep: int | None = None) -> list[str]:
|
||
"""Alte Versionsstände löschen, nur die neuesten `keep` behalten.
|
||
|
||
`keep = 0` (oder negativ) bedeutet: unbegrenzt aufbewahren – es wird
|
||
nichts gelöscht.
|
||
"""
|
||
keep = config.EPIRENT_DOWNLOAD_KEEP if keep is None else keep
|
||
if keep <= 0:
|
||
return []
|
||
versions = [v["version"] for v in history()]
|
||
removed = []
|
||
for v in versions[keep:]:
|
||
d = version_dir(v)
|
||
shutil.rmtree(d, ignore_errors=True)
|
||
with db.get_conn() as conn:
|
||
conn.execute("DELETE FROM downloads WHERE version=?", (v,))
|
||
removed.append(v)
|
||
log.info("Alten Download-Stand %s entfernt.", v)
|
||
return removed
|
||
|
||
|
||
# --------------------------------------------------------------- Hintergrund
|
||
def archive_async(version: str) -> None:
|
||
"""Archivierung im Hintergrund starten (max. eine gleichzeitig)."""
|
||
global _running
|
||
if not config.EPIRENT_DOWNLOAD_ENABLED or not version:
|
||
return
|
||
with _lock:
|
||
if _running:
|
||
log.info("Archivierung läuft bereits – überspringe.")
|
||
return
|
||
_running = True
|
||
|
||
def _work():
|
||
global _running
|
||
try:
|
||
got = fetch_version(version)
|
||
removed = cleanup()
|
||
if got:
|
||
_report(version, got, removed)
|
||
except Exception: # noqa: BLE001
|
||
log.exception("Archivierung fehlgeschlagen")
|
||
finally:
|
||
with _lock:
|
||
_running = False
|
||
|
||
threading.Thread(target=_work, name="epirent-archive", daemon=True).start()
|
||
|
||
|
||
def _report(version: str, got: list[dict], removed: list[str]) -> None:
|
||
"""Erfolgreiche Archivierung als Änderung erfassen und mailen."""
|
||
parts = [f"{g['filename']} ({human_size(g['size'])})" for g in got]
|
||
detail = ", ".join(parts)
|
||
if removed:
|
||
detail += f" · alte Stände entfernt: {', '.join(removed)}"
|
||
change = {
|
||
"ticket_number": None,
|
||
"ticket_title": "EPIRent Download-Archiv",
|
||
"field": "epirent_download",
|
||
"label": "Downloads gesichert",
|
||
"old_value": None,
|
||
"new_value": f"Version {version}",
|
||
"detail": detail,
|
||
}
|
||
with db.get_conn() as conn:
|
||
conn.execute(
|
||
"INSERT INTO changes (ticket_id, ticket_number, ticket_title, field, "
|
||
"label, old_value, new_value, detail, detected_at) "
|
||
"VALUES (?,?,?,?,?,?,?,?,?)",
|
||
(0, None, change["ticket_title"], change["field"], change["label"],
|
||
None, change["new_value"], detail, _now()))
|
||
try:
|
||
notify.send_change_mail([change])
|
||
except Exception: # noqa: BLE001
|
||
log.exception("Mail zur Archivierung fehlgeschlagen")
|