Watch EPIRent version and archive its download packages
- Poll the public download page for "Aktuelle Version <nr> vom <date>", report changes through the normal change feed and notification mail, and show the current version as a card on the overview. Runs on its own interval (default 30 min); the first run only records a baseline. - Archive epirentServer.zip / epirentClient.zip per version number so we keep a download cache and history: streamed to disk with SHA-256 and size, fetched in a background thread so ticket polling is unaffected, with a free-space check before downloading (~1.1 GB per version). - Retention is configurable via EPIRENT_DOWNLOAD_KEEP, where 0 means unlimited; a successful archive is reported as a change and by mail. - New Downloads page listing the archived versions with checksums and links to fetch the cached files. - Notification lines now omit the arrow when there is no previous value and drop the "#" for entries without a ticket. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,256 @@
|
||||
"""Download-Archiv: sichert die EPIRent-Pakete je Versionsnummer.
|
||||
|
||||
Die Dateien sind groß (Server ~923 MB, Client ~218 MB, zusammen ~1,1 GB pro
|
||||
Version). Deshalb:
|
||||
* Download läuft im Hintergrund-Thread (Ticketabruf blockiert nicht),
|
||||
* gestreamt auf Platte (kein Speicherverbrauch),
|
||||
* Platzprüfung vorab,
|
||||
* Aufbewahrung begrenzt (EPIRENT_DOWNLOAD_KEEP, Standard 3 Stände).
|
||||
"""
|
||||
import hashlib
|
||||
import logging
|
||||
import shutil
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
|
||||
from . import config, db, notify
|
||||
|
||||
log = logging.getLogger("episupport.downloads")
|
||||
|
||||
DOWNLOAD_DIR = config.DATA_DIR / "downloads"
|
||||
_lock = threading.Lock()
|
||||
_running = False
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def filename_for(url: str) -> str:
|
||||
name = Path(urlparse(url).path).name
|
||||
return name or "download.bin"
|
||||
|
||||
|
||||
def version_dir(version: str) -> Path:
|
||||
# Versionsnummern sind rein numerisch – zur Sicherheit trotzdem säubern
|
||||
safe = "".join(ch for ch in str(version) if ch.isalnum() or ch in "._-")
|
||||
return DOWNLOAD_DIR / (safe or "unbekannt")
|
||||
|
||||
|
||||
def archived(version: str) -> list[dict]:
|
||||
with db.get_conn() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM downloads WHERE version=? ORDER BY filename",
|
||||
(str(version),)).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def is_complete(version: str) -> bool:
|
||||
"""Sind alle konfigurierten Dateien für diese Version vorhanden?"""
|
||||
want = {filename_for(u) for u in config.EPIRENT_DOWNLOAD_URLS}
|
||||
have = {r["filename"] for r in archived(version)
|
||||
if (version_dir(version) / r["filename"]).exists()}
|
||||
return bool(want) and want.issubset(have)
|
||||
|
||||
|
||||
def history() -> list[dict]:
|
||||
"""Alle archivierten Versionen mit Dateien und Gesamtgröße."""
|
||||
with db.get_conn() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM downloads ORDER BY downloaded_at DESC, filename"
|
||||
).fetchall()
|
||||
by_version: dict[str, dict] = {}
|
||||
for r in rows:
|
||||
v = by_version.setdefault(r["version"], {
|
||||
"version": r["version"], "files": [], "total": 0,
|
||||
"downloaded_at": r["downloaded_at"]})
|
||||
v["files"].append(dict(r))
|
||||
v["total"] += r["size"] or 0
|
||||
return list(by_version.values())
|
||||
|
||||
|
||||
def human_size(n) -> str:
|
||||
if not n:
|
||||
return "–"
|
||||
n = float(n)
|
||||
for unit in ("B", "KB", "MB", "GB"):
|
||||
if n < 1024 or unit == "GB":
|
||||
return f"{n:.0f} {unit}" if unit in ("B", "KB") else f"{n:.1f} {unit}".replace(".", ",")
|
||||
n /= 1024
|
||||
return f"{n:.1f} GB"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ Download
|
||||
def _remote_info(url: str) -> tuple[int, str | None]:
|
||||
"""(Größe, Last-Modified) per HEAD ermitteln."""
|
||||
r = requests.head(url, timeout=30, allow_redirects=True,
|
||||
headers={"User-Agent": "episupport-monitor/1.0"})
|
||||
r.raise_for_status()
|
||||
return int(r.headers.get("Content-Length") or 0), r.headers.get("Last-Modified")
|
||||
|
||||
|
||||
def _download(url: str, dest: Path) -> tuple[int, str]:
|
||||
"""Streamend herunterladen; gibt (Größe, sha256) zurück."""
|
||||
tmp = dest.with_suffix(dest.suffix + ".part")
|
||||
h = hashlib.sha256()
|
||||
size = 0
|
||||
with requests.get(url, stream=True, timeout=(30, 300),
|
||||
headers={"User-Agent": "episupport-monitor/1.0"}) as r:
|
||||
r.raise_for_status()
|
||||
with open(tmp, "wb") as f:
|
||||
for chunk in r.iter_content(chunk_size=1024 * 1024):
|
||||
if not chunk:
|
||||
continue
|
||||
f.write(chunk)
|
||||
h.update(chunk)
|
||||
size += len(chunk)
|
||||
tmp.replace(dest)
|
||||
return size, h.hexdigest()
|
||||
|
||||
|
||||
def _enough_space(needed: int) -> bool:
|
||||
DOWNLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||||
free = shutil.disk_usage(DOWNLOAD_DIR).free
|
||||
if free < needed * 1.2:
|
||||
log.error("Zu wenig Speicherplatz: benötigt ~%s, frei %s.",
|
||||
human_size(needed), human_size(free))
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def fetch_version(version: str) -> list[dict]:
|
||||
"""Fehlende Dateien dieser Version holen. Gibt die neu geholten zurück."""
|
||||
urls = config.EPIRENT_DOWNLOAD_URLS
|
||||
if not urls:
|
||||
return []
|
||||
target = version_dir(version)
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
todo = []
|
||||
for url in urls:
|
||||
name = filename_for(url)
|
||||
if (target / name).exists() and any(
|
||||
r["filename"] == name for r in archived(version)):
|
||||
continue
|
||||
todo.append((url, name))
|
||||
if not todo:
|
||||
return []
|
||||
|
||||
# Platzbedarf vorab prüfen
|
||||
needed = 0
|
||||
infos = {}
|
||||
for url, name in todo:
|
||||
try:
|
||||
size, modified = _remote_info(url)
|
||||
except requests.RequestException as e:
|
||||
log.warning("HEAD fehlgeschlagen für %s: %s", url, e)
|
||||
size, modified = 0, None
|
||||
infos[name] = (size, modified)
|
||||
needed += size
|
||||
if needed and not _enough_space(needed):
|
||||
return []
|
||||
|
||||
got = []
|
||||
for url, name in todo:
|
||||
dest = target / name
|
||||
try:
|
||||
log.info("Lade %s (%s) für Version %s …", name,
|
||||
human_size(infos[name][0]), version)
|
||||
size, digest = _download(url, dest)
|
||||
with db.get_conn() as conn:
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO downloads (version, filename, url, "
|
||||
"size, sha256, remote_modified, downloaded_at) "
|
||||
"VALUES (?,?,?,?,?,?,?)",
|
||||
(str(version), name, url, size, digest,
|
||||
infos[name][1], _now()))
|
||||
got.append({"filename": name, "size": size, "sha256": digest})
|
||||
log.info("Fertig: %s (%s)", name, human_size(size))
|
||||
except (requests.RequestException, OSError) as e:
|
||||
log.error("Download %s fehlgeschlagen: %s", url, e)
|
||||
try:
|
||||
(target / (name + ".part")).unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
return got
|
||||
|
||||
|
||||
def cleanup(keep: int | None = None) -> list[str]:
|
||||
"""Alte Versionsstände löschen, nur die neuesten `keep` behalten.
|
||||
|
||||
`keep = 0` (oder negativ) bedeutet: unbegrenzt aufbewahren – es wird
|
||||
nichts gelöscht.
|
||||
"""
|
||||
keep = config.EPIRENT_DOWNLOAD_KEEP if keep is None else keep
|
||||
if keep <= 0:
|
||||
return []
|
||||
versions = [v["version"] for v in history()]
|
||||
removed = []
|
||||
for v in versions[keep:]:
|
||||
d = version_dir(v)
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
with db.get_conn() as conn:
|
||||
conn.execute("DELETE FROM downloads WHERE version=?", (v,))
|
||||
removed.append(v)
|
||||
log.info("Alten Download-Stand %s entfernt.", v)
|
||||
return removed
|
||||
|
||||
|
||||
# --------------------------------------------------------------- Hintergrund
|
||||
def archive_async(version: str) -> None:
|
||||
"""Archivierung im Hintergrund starten (max. eine gleichzeitig)."""
|
||||
global _running
|
||||
if not config.EPIRENT_DOWNLOAD_ENABLED or not version:
|
||||
return
|
||||
with _lock:
|
||||
if _running:
|
||||
log.info("Archivierung läuft bereits – überspringe.")
|
||||
return
|
||||
_running = True
|
||||
|
||||
def _work():
|
||||
global _running
|
||||
try:
|
||||
got = fetch_version(version)
|
||||
removed = cleanup()
|
||||
if got:
|
||||
_report(version, got, removed)
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("Archivierung fehlgeschlagen")
|
||||
finally:
|
||||
with _lock:
|
||||
_running = False
|
||||
|
||||
threading.Thread(target=_work, name="epirent-archive", daemon=True).start()
|
||||
|
||||
|
||||
def _report(version: str, got: list[dict], removed: list[str]) -> None:
|
||||
"""Erfolgreiche Archivierung als Änderung erfassen und mailen."""
|
||||
parts = [f"{g['filename']} ({human_size(g['size'])})" for g in got]
|
||||
detail = ", ".join(parts)
|
||||
if removed:
|
||||
detail += f" · alte Stände entfernt: {', '.join(removed)}"
|
||||
change = {
|
||||
"ticket_number": None,
|
||||
"ticket_title": "EPIRent Download-Archiv",
|
||||
"field": "epirent_download",
|
||||
"label": "Downloads gesichert",
|
||||
"old_value": None,
|
||||
"new_value": f"Version {version}",
|
||||
"detail": detail,
|
||||
}
|
||||
with db.get_conn() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO changes (ticket_id, ticket_number, ticket_title, field, "
|
||||
"label, old_value, new_value, detail, detected_at) "
|
||||
"VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
(0, None, change["ticket_title"], change["field"], change["label"],
|
||||
None, change["new_value"], detail, _now()))
|
||||
try:
|
||||
notify.send_change_mail([change])
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("Mail zur Archivierung fehlgeschlagen")
|
||||
Reference in New Issue
Block a user