Initialisiere Explor.Code.Law mit RII-Pipeline und Strato-Deploy.

Entscheidungen als Markdown, Normen-Index und schlankes Git-Sync nach /opt/explore (Gitea Actions, ohne Docker).

Co-authored-by: Cursor <cursoragent@cursor.com>

extract_rii.py

@@ -0,0 +1,308 @@
+#!/usr/bin/env python3
+"""
+Gerichtsentscheidungen von rechtsprechung-im-internet.de holen.
+
+  python3 extract_rii.py --limit 5 --court bgh,bverfg
+  python3 extract_rii.py --court bgh --since 2020
+  python3 extract_rii.py                 # voller Bestand (lang)
+
+Wiederaufsetzbar: vorhandene doknr in ge_md/ werden übersprungen.
+Roh-ZIP/XML landen in ge_xml/ (gitignoriert).
+"""
+
+from __future__ import annotations
+
+import argparse
+import io
+import re
+import sys
+import time
+import zipfile
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+import requests
+
+from rii_to_markdown import (
+    COURT_MAP,
+    convert_xml_to_markdown_file,
+    iso_date,
+    parse_rii_xml,
+    relative_path,
+)
+
+TOC_URL = "https://www.rechtsprechung-im-internet.de/rii-toc.xml"
+UA = "explor-code-law/0.1 (https://git.coded.law; RII Open Data)"
+SESSION = requests.Session()
+SESSION.headers.update({"User-Agent": UA})
+
+# Kurzname → Anzeigewert in TOC <gericht>
+COURT_FILTER: dict[str, tuple[str, ...]] = {
+    "bverfg": ("BVerfG", "Bundesverfassungsgericht"),
+    "bgh": ("BGH", "Bundesgerichtshof"),
+    "bverwg": ("BVerwG", "Bundesverwaltungsgericht"),
+    "bfh": ("BFH", "Bundesfinanzhof"),
+    "bag": ("BAG", "Bundesarbeitsgericht"),
+    "bsg": ("BSG", "Bundessozialgericht"),
+    "bpatg": ("BPatG", "Bundespatentgericht"),
+}
+
+
+def load_toc(url: str = TOC_URL) -> list[dict]:
+    print(f"Lade Inhaltsverzeichnis: {url}")
+    resp = SESSION.get(url, timeout=120)
+    resp.raise_for_status()
+    root = ET.fromstring(resp.content)
+    items: list[dict] = []
+    for item in root.findall("item"):
+        def t(tag: str) -> str:
+            el = item.find(tag)
+            return (el.text or "").strip() if el is not None else ""
+
+        gericht = t("gericht")
+        link = t("link")
+        az = t("aktenzeichen")
+        datum = iso_date(t("entsch-datum"))
+        # doknr manchmal im Link / Titel
+        doknr = t("doknr")
+        if not doknr and link:
+            m = re.search(r"/([A-Z]{4}\d+)", link)
+            if m:
+                doknr = m.group(1)
+        if not link:
+            continue
+        items.append(
+            {
+                "gericht": gericht,
+                "aktenzeichen": az,
+                "entsch-datum": datum,
+                "link": link,
+                "doknr": doknr,
+                "title": t("title") or t("titel"),
+            }
+        )
+    print(f"  {len(items)} Einträge")
+    return items
+
+
+def match_court(gericht: str, wanted: set[str]) -> bool:
+    if not wanted:
+        return True
+    g = gericht or ""
+    for key in wanted:
+        aliases = COURT_FILTER.get(key, (key,))
+        for a in aliases:
+            if a.lower() in g.lower() or g.lower() in a.lower():
+                return True
+            # exakter Typ
+            if g.strip().upper() == a.upper():
+                return True
+    return False
+
+
+def court_keys_from_arg(raw: str | None) -> set[str]:
+    if not raw:
+        return set()
+    keys = set()
+    for part in raw.split(","):
+        k = part.strip().lower()
+        if not k:
+            continue
+        if k in COURT_FILTER:
+            keys.add(k)
+        elif k.upper() in COURT_MAP:
+            keys.add(COURT_MAP[k.upper()][1])
+        else:
+            keys.add(k)
+    return keys
+
+
+def existing_doknrs(ge_md: Path) -> set[str]:
+    found: set[str] = set()
+    if not ge_md.is_dir():
+        return found
+    for path in ge_md.rglob("*.md"):
+        try:
+            head = path.read_text(encoding="utf-8", errors="replace")[:1200]
+        except OSError:
+            continue
+        m = re.search(r'doknr:\s*"([^"]+)"', head)
+        if m:
+            found.add(m.group(1))
+            continue
+        m = re.search(r"doknr:\s*(\S+)", head)
+        if m:
+            found.add(m.group(1).strip('"'))
+    return found
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="RII-Entscheidungen extrahieren")
+    ap.add_argument("--toc", default=TOC_URL, help="URL des Inhaltsverzeichnisses")
+    ap.add_argument(
+        "--court",
+        default="",
+        help="Kommagetrennt: bgh,bverfg,bverwg,bfh,bag,bsg,bpatg",
+    )
+    ap.add_argument("--since", type=int, default=0, help="Nur ab Entscheidungsjahr")
+    ap.add_argument("--limit", type=int, default=0, help="Max. neu geschriebene Dateien")
+    ap.add_argument("--delay", type=float, default=0.4, help="Pause zwischen Downloads (s)")
+    ap.add_argument("--ge-md", default="ge_md")
+    ap.add_argument("--ge-xml", default="ge_xml")
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+
+    ge_md = Path(args.ge_md)
+    ge_xml = Path(args.ge_xml)
+    wanted = court_keys_from_arg(args.court)
+
+    items = load_toc(args.toc)
+    known = existing_doknrs(ge_md)
+    print(f"  bereits in ge_md: {len(known)} doknr")
+
+    work: list[dict] = []
+    skipped_year = 0
+    skipped_court = 0
+    skipped_known = 0
+    by_court: dict[str, list[dict]] = {k: [] for k in wanted} if wanted else {"_all": []}
+
+    for it in items:
+        if not match_court(it["gericht"], wanted):
+            skipped_court += 1
+            continue
+        year_s = (it["entsch-datum"] or "")[:4]
+        if args.since and year_s.isdigit() and int(year_s) < args.since:
+            skipped_year += 1
+            continue
+        if it["doknr"] and it["doknr"] in known:
+            skipped_known += 1
+            continue
+        if wanted:
+            bucket = next(
+                (k for k in wanted if match_court(it["gericht"], {k})),
+                None,
+            )
+            if bucket is None:
+                continue
+            by_court[bucket].append(it)
+        else:
+            by_court["_all"].append(it)
+
+    # Bei --limit: Gerichte round-robin mischen, damit z. B. bgh,bverfg beide vorkommen
+    if args.limit and wanted:
+        per = max(1, (args.limit + len(wanted) - 1) // len(wanted))
+        queues = {k: v[: per * 2] for k, v in by_court.items()}
+        while any(queues.values()) and len(work) < args.limit * 2:
+            for k in list(queues.keys()):
+                if queues[k]:
+                    work.append(queues[k].pop(0))
+                if len(work) >= args.limit * 2:
+                    break
+    else:
+        for v in by_court.values():
+            work.extend(v)
+
+    print(
+        f"  Filter: Gericht übersprungen={skipped_court}, "
+        f"Jahr={skipped_year}, bekannt={skipped_known}, Kandidaten={len(work)}"
+    )
+    if wanted:
+        print(
+            "  je Gericht: "
+            + ", ".join(f"{k}={len(by_court[k])}" for k in sorted(by_court))
+        )
+
+    if args.dry_run:
+        for it in work[: args.limit or 20]:
+            print(f"  would fetch: {it['gericht']} {it['aktenzeichen']} {it['link']}")
+        return 0
+
+    written = 0
+    failed = 0
+    limit = args.limit or 10**9
+
+    for i, it in enumerate(work):
+        if written >= limit:
+            break
+        link = it["link"]
+        try:
+            resp = SESSION.get(link, timeout=60)
+            resp.raise_for_status()
+            blob = resp.content
+        except requests.RequestException as e:
+            print(f"  Fehler {link}: {e}", file=sys.stderr)
+            failed += 1
+            time.sleep(args.delay)
+            continue
+
+        is_zip = link.endswith(".zip") or blob[:2] == b"PK"
+        if is_zip:
+            try:
+                with zipfile.ZipFile(io.BytesIO(blob)) as zf:
+                    names = [n for n in zf.namelist() if n.endswith(".xml")]
+                    if not names:
+                        failed += 1
+                        time.sleep(args.delay)
+                        continue
+                    xml_bytes = zf.read(names[0])
+            except zipfile.BadZipFile:
+                failed += 1
+                time.sleep(args.delay)
+                continue
+        else:
+            xml_bytes = blob
+
+        parsed = parse_rii_xml(xml_bytes)
+        if not parsed:
+            failed += 1
+            time.sleep(args.delay)
+            continue
+
+        doknr = parsed["meta"].get("doknr") or ""
+        if doknr and doknr in known:
+            skipped_known += 1
+            time.sleep(args.delay)
+            continue
+
+        # Zielpfad schon da?
+        rel = relative_path(parsed["meta"])
+        target = ge_md / rel
+        if target.is_file():
+            known.add(doknr)
+            time.sleep(args.delay)
+            continue
+
+        out = convert_xml_to_markdown_file(xml_bytes, ge_md)
+        if out is None:
+            failed += 1
+            time.sleep(args.delay)
+            continue
+
+        # Roh speichern
+        try:
+            gertyp = parsed["meta"].get("gericht") or "unbekannt"
+            court = COURT_MAP.get(gertyp, ("bund", gertyp.lower()))[1]
+            dest_dir = ge_xml / court
+            dest_dir.mkdir(parents=True, exist_ok=True)
+            raw_name = doknr or out.stem
+            if is_zip:
+                (dest_dir / f"{raw_name}.zip").write_bytes(blob)
+            else:
+                (dest_dir / f"{raw_name}.xml").write_bytes(xml_bytes)
+        except OSError as e:
+            print(f"  Warnung Rohspeicher: {e}", file=sys.stderr)
+
+        if doknr:
+            known.add(doknr)
+        written += 1
+        if written % 50 == 0 or written <= 5:
+            print(f"  geschrieben {written}: {out}")
+
+        time.sleep(args.delay)
+
+    print(f"Fertig. neu={written}, Fehler={failed}")
+    return 0 if failed == 0 or written > 0 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())