explore/extract_rii.py

#!/usr/bin/env python3
"""
Gerichtsentscheidungen von rechtsprechung-im-internet.de holen.

  python3 extract_rii.py --limit 5 --court bgh,bverfg
  python3 extract_rii.py --court bgh --since 2020
  python3 extract_rii.py                 # voller Bestand (lang)

Wiederaufsetzbar: vorhandene doknr in ge_md/ werden übersprungen.
Roh-ZIP/XML landen in ge_xml/ (gitignoriert).
"""

from __future__ import annotations

import argparse
import io
import re
import sys
import time
import zipfile
import xml.etree.ElementTree as ET
from pathlib import Path

import requests

from rii_to_markdown import (
    COURT_MAP,
    convert_xml_to_markdown_file,
    iso_date,
    parse_rii_xml,
    relative_path,
)

TOC_URL = "https://www.rechtsprechung-im-internet.de/rii-toc.xml"
UA = "explor-code-law/0.1 (https://git.coded.law; RII Open Data)"
SESSION = requests.Session()
SESSION.headers.update({"User-Agent": UA})

# Kurzname → Anzeigewert in TOC <gericht>
COURT_FILTER: dict[str, tuple[str, ...]] = {
    "bverfg": ("BVerfG", "Bundesverfassungsgericht"),
    "bgh": ("BGH", "Bundesgerichtshof"),
    "bverwg": ("BVerwG", "Bundesverwaltungsgericht"),
    "bfh": ("BFH", "Bundesfinanzhof"),
    "bag": ("BAG", "Bundesarbeitsgericht"),
    "bsg": ("BSG", "Bundessozialgericht"),
    "bpatg": ("BPatG", "Bundespatentgericht"),
}


def load_toc(url: str = TOC_URL) -> list[dict]:
    print(f"Lade Inhaltsverzeichnis: {url}")
    resp = SESSION.get(url, timeout=120)
    resp.raise_for_status()
    root = ET.fromstring(resp.content)
    items: list[dict] = []
    for item in root.findall("item"):
        def t(tag: str) -> str:
            el = item.find(tag)
            return (el.text or "").strip() if el is not None else ""

        gericht = t("gericht")
        link = t("link")
        az = t("aktenzeichen")
        datum = iso_date(t("entsch-datum"))
        # doknr manchmal im Link / Titel
        doknr = t("doknr")
        if not doknr and link:
            m = re.search(r"/([A-Z]{4}\d+)", link)
            if m:
                doknr = m.group(1)
        if not link:
            continue
        items.append(
            {
                "gericht": gericht,
                "aktenzeichen": az,
                "entsch-datum": datum,
                "link": link,
                "doknr": doknr,
                "title": t("title") or t("titel"),
            }
        )
    print(f"  {len(items)} Einträge")
    return items


def match_court(gericht: str, wanted: set[str]) -> bool:
    if not wanted:
        return True
    g = gericht or ""
    for key in wanted:
        aliases = COURT_FILTER.get(key, (key,))
        for a in aliases:
            if a.lower() in g.lower() or g.lower() in a.lower():
                return True
            # exakter Typ
            if g.strip().upper() == a.upper():
                return True
    return False


def court_keys_from_arg(raw: str | None) -> set[str]:
    if not raw:
        return set()
    keys = set()
    for part in raw.split(","):
        k = part.strip().lower()
        if not k:
            continue
        if k in COURT_FILTER:
            keys.add(k)
        elif k.upper() in COURT_MAP:
            keys.add(COURT_MAP[k.upper()][1])
        else:
            keys.add(k)
    return keys


def existing_doknrs(ge_md: Path) -> set[str]:
    found: set[str] = set()
    if not ge_md.is_dir():
        return found
    for path in ge_md.rglob("*.md"):
        try:
            head = path.read_text(encoding="utf-8", errors="replace")[:1200]
        except OSError:
            continue
        m = re.search(r'doknr:\s*"([^"]+)"', head)
        if m:
            found.add(m.group(1))
            continue
        m = re.search(r"doknr:\s*(\S+)", head)
        if m:
            found.add(m.group(1).strip('"'))
    return found


def main() -> int:
    ap = argparse.ArgumentParser(description="RII-Entscheidungen extrahieren")
    ap.add_argument("--toc", default=TOC_URL, help="URL des Inhaltsverzeichnisses")
    ap.add_argument(
        "--court",
        default="",
        help="Kommagetrennt: bgh,bverfg,bverwg,bfh,bag,bsg,bpatg",
    )
    ap.add_argument("--since", type=int, default=0, help="Nur ab Entscheidungsjahr")
    ap.add_argument("--limit", type=int, default=0, help="Max. neu geschriebene Dateien")
    ap.add_argument("--delay", type=float, default=0.4, help="Pause zwischen Downloads (s)")
    ap.add_argument("--ge-md", default="ge_md")
    ap.add_argument("--ge-xml", default="ge_xml")
    ap.add_argument("--dry-run", action="store_true")
    args = ap.parse_args()

    ge_md = Path(args.ge_md)
    ge_xml = Path(args.ge_xml)
    wanted = court_keys_from_arg(args.court)

    items = load_toc(args.toc)
    known = existing_doknrs(ge_md)
    print(f"  bereits in ge_md: {len(known)} doknr")

    work: list[dict] = []
    skipped_year = 0
    skipped_court = 0
    skipped_known = 0
    by_court: dict[str, list[dict]] = {k: [] for k in wanted} if wanted else {"_all": []}

    for it in items:
        if not match_court(it["gericht"], wanted):
            skipped_court += 1
            continue
        year_s = (it["entsch-datum"] or "")[:4]
        if args.since and year_s.isdigit() and int(year_s) < args.since:
            skipped_year += 1
            continue
        if it["doknr"] and it["doknr"] in known:
            skipped_known += 1
            continue
        if wanted:
            bucket = next(
                (k for k in wanted if match_court(it["gericht"], {k})),
                None,
            )
            if bucket is None:
                continue
            by_court[bucket].append(it)
        else:
            by_court["_all"].append(it)

    # Bei --limit: Gerichte round-robin mischen, damit z. B. bgh,bverfg beide vorkommen
    if args.limit and wanted:
        per = max(1, (args.limit + len(wanted) - 1) // len(wanted))
        queues = {k: v[: per * 2] for k, v in by_court.items()}
        while any(queues.values()) and len(work) < args.limit * 2:
            for k in list(queues.keys()):
                if queues[k]:
                    work.append(queues[k].pop(0))
                if len(work) >= args.limit * 2:
                    break
    else:
        for v in by_court.values():
            work.extend(v)

    print(
        f"  Filter: Gericht übersprungen={skipped_court}, "
        f"Jahr={skipped_year}, bekannt={skipped_known}, Kandidaten={len(work)}"
    )
    if wanted:
        print(
            "  je Gericht: "
            + ", ".join(f"{k}={len(by_court[k])}" for k in sorted(by_court))
        )

    if args.dry_run:
        for it in work[: args.limit or 20]:
            print(f"  would fetch: {it['gericht']} {it['aktenzeichen']} {it['link']}")
        return 0

    written = 0
    failed = 0
    limit = args.limit or 10**9

    for i, it in enumerate(work):
        if written >= limit:
            break
        link = it["link"]
        try:
            resp = SESSION.get(link, timeout=60)
            resp.raise_for_status()
            blob = resp.content
        except requests.RequestException as e:
            print(f"  Fehler {link}: {e}", file=sys.stderr)
            failed += 1
            time.sleep(args.delay)
            continue

        is_zip = link.endswith(".zip") or blob[:2] == b"PK"
        if is_zip:
            try:
                with zipfile.ZipFile(io.BytesIO(blob)) as zf:
                    names = [n for n in zf.namelist() if n.endswith(".xml")]
                    if not names:
                        failed += 1
                        time.sleep(args.delay)
                        continue
                    xml_bytes = zf.read(names[0])
            except zipfile.BadZipFile:
                failed += 1
                time.sleep(args.delay)
                continue
        else:
            xml_bytes = blob

        parsed = parse_rii_xml(xml_bytes)
        if not parsed:
            failed += 1
            time.sleep(args.delay)
            continue

        doknr = parsed["meta"].get("doknr") or ""
        if doknr and doknr in known:
            skipped_known += 1
            time.sleep(args.delay)
            continue

        # Zielpfad schon da?
        rel = relative_path(parsed["meta"])
        target = ge_md / rel
        if target.is_file():
            known.add(doknr)
            time.sleep(args.delay)
            continue

        out = convert_xml_to_markdown_file(xml_bytes, ge_md)
        if out is None:
            failed += 1
            time.sleep(args.delay)
            continue

        # Roh speichern
        try:
            gertyp = parsed["meta"].get("gericht") or "unbekannt"
            court = COURT_MAP.get(gertyp, ("bund", gertyp.lower()))[1]
            dest_dir = ge_xml / court
            dest_dir.mkdir(parents=True, exist_ok=True)
            raw_name = doknr or out.stem
            if is_zip:
                (dest_dir / f"{raw_name}.zip").write_bytes(blob)
            else:
                (dest_dir / f"{raw_name}.xml").write_bytes(xml_bytes)
        except OSError as e:
            print(f"  Warnung Rohspeicher: {e}", file=sys.stderr)

        if doknr:
            known.add(doknr)
        written += 1
        if written % 50 == 0 or written <= 5:
            print(f"  geschrieben {written}: {out}")

        time.sleep(args.delay)

    print(f"Fertig. neu={written}, Fehler={failed}")
    return 0 if failed == 0 or written > 0 else 1


if __name__ == "__main__":
    raise SystemExit(main())