explore/rii_to_markdown.py

#!/usr/bin/env python3
"""
RII-XML (rechtsprechung-im-internet.de, rii-dok.dtd) → Markdown mit YAML-Kopf.

Eine Entscheidung wird eine Datei unter ge_md/<gericht>/<jahr>/<az-slug>.md.
"""

from __future__ import annotations

import re
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Any

# Gerichtstyp → Ebene und Ordnername
COURT_MAP: dict[str, tuple[str, str]] = {
    "BVerfG": ("verfg", "bverfg"),
    "BGH": ("bund", "bgh"),
    "BVerwG": ("bund", "bverwg"),
    "BFH": ("bund", "bfh"),
    "BAG": ("bund", "bag"),
    "BSG": ("bund", "bsg"),
    "BPatG": ("bund", "bpatg"),
}

BUND_COURTS = {"BGH", "BVerwG", "BFH", "BAG", "BSG", "BPatG"}


def local_tag(tag: str | None) -> str:
    if not tag:
        return ""
    if "}" in tag:
        return tag.rsplit("}", 1)[-1]
    return tag


def elem_text(elem: ET.Element | None) -> str:
    if elem is None:
        return ""
    return "".join(elem.itertext()).strip()


def first_child(parent: ET.Element, name: str) -> ET.Element | None:
    for child in parent:
        if local_tag(child.tag) == name:
            return child
    return None


def all_children(parent: ET.Element, name: str) -> list[ET.Element]:
    return [c for c in parent if local_tag(c.tag) == name]


def find_deep(parent: ET.Element, name: str) -> ET.Element | None:
    for elem in parent.iter():
        if local_tag(elem.tag) == name:
            return elem
    return None


def xml_fragment_to_markdown(elem: ET.Element | None) -> str:
    """HTML/XML-Fragment (P, BR, DL, …) zu lesbarem Markdown."""
    if elem is None:
        return ""

    result: list[str] = []

    def process_list(dl_elem: ET.Element) -> None:
        for child in dl_elem:
            tag = local_tag(child.tag)
            if tag == "DT":
                if result and result[-1] not in ("\n", "\n\n"):
                    result.append("\n")
                process_element(child)
            elif tag == "DD":
                process_element(child)
                result.append("\n")
            else:
                process_element(child)

    def process_element(e: ET.Element) -> None:
        if e.text and e.text.strip():
            result.append(e.text.strip())

        for child in e:
            tag = local_tag(child.tag)
            if tag == "P":
                if result and result[-1] != "\n\n":
                    result.append("\n\n")
                process_element(child)
                result.append("\n\n")
            elif tag == "BR":
                result.append("\n")
                if child.text and child.text.strip():
                    result.append(child.text.strip())
            elif tag == "DL":
                process_list(child)
            elif tag in ("DT", "DD"):
                if tag == "DT" and result and result[-1] not in ("\n", "\n\n"):
                    result.append("\n")
                process_element(child)
                if tag == "DD":
                    result.append("\n")
            elif tag == "LA":
                process_element(child)
            elif tag in ("table", "tgroup", "tbody", "thead", "row", "entry"):
                if tag in ("table", "tgroup"):
                    result.append("\n[Tabelle]\n")
            else:
                process_element(child)

            if child.tail and child.tail.strip():
                result.append(child.tail.strip())

    process_element(elem)
    text = "".join(result)
    text = (
        text.replace("&quot;", '"')
        .replace("&amp;", "&")
        .replace("&lt;", "<")
        .replace("&gt;", ">")
        .replace("&nbsp;", " ")
        .replace("&#167;", "§")
    )
    text = re.sub(r"\n{3,}", "\n\n", text)
    return text.strip()


def iso_date(raw: str) -> str:
    s = (raw or "").strip()
    m = re.match(r"^(\d{4})-(\d{2})-(\d{2})", s)
    if m:
        return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
    m = re.match(r"^(\d{2})\.(\d{2})\.(\d{4})", s)
    if m:
        return f"{m.group(3)}-{m.group(2)}-{m.group(1)}"
    m = re.match(r"^(\d{4})(\d{2})(\d{2})$", s)
    if m:
        return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
    return ""


def ebene_for(gertyp: str) -> str:
    g = (gertyp or "").strip()
    if g in COURT_MAP:
        return COURT_MAP[g][0]
    if g.upper().startswith("OLG") or g.upper().startswith("LG"):
        return "olg"
    if g in BUND_COURTS:
        return "bund"
    return "bund"


def court_dir(gertyp: str) -> str:
    g = (gertyp or "").strip()
    if g in COURT_MAP:
        return COURT_MAP[g][1]
    slug = re.sub(r"[^a-z0-9]+", "-", g.lower()).strip("-")
    return slug or "unbekannt"


def az_slug(aktenzeichen: str, doknr: str = "") -> str:
    s = (
        aktenzeichen.lower()
        .replace("ä", "ae")
        .replace("ö", "oe")
        .replace("ü", "ue")
        .replace("ß", "ss")
    )
    s = re.sub(r"[\s/]+", "-", s)
    s = re.sub(r"[^a-z0-9-]", "", s)
    s = re.sub(r"-+", "-", s).strip("-")
    if len(s) <= 80:
        return s or (doknr.lower() if doknr else "ohne-az")
    suf = "-" + re.sub(r"[^a-z0-9]", "", doknr.lower()) if doknr else ""
    return s[: max(1, 80 - len(suf))].rstrip("-") + suf


def parse_norm_string(roh: str) -> dict[str, str] | None:
    """
    Einzelnen Norm-String zu {gesetz, paragraph, roh} zerlegen.

    Beispiele:
      § 242 BGB
      §§ 133, 134 InsO
      Art. 2 Abs. 1 GG
      § 31 Abs. 1 BVerfGG
    """
    roh = (roh or "").strip()
    if not roh:
        return None

    # Gesetz am Ende: Abkürzung (Buchstaben/Ziffern, mind. 2 Zeichen)
    m = re.search(
        r"^(?:§{1,2}|Art\.?)\s*(.+?)\s+([A-Za-zÄÖÜäöüß][A-Za-zÄÖÜäöüß0-9.\-]{1,})\s*$",
        roh,
    )
    if not m:
        # nur Abkürzung ohne Paragraph
        m2 = re.match(r"^([A-Za-zÄÖÜäöüß][A-Za-zÄÖÜäöüß0-9.\-]{1,})$", roh)
        if m2:
            return {"gesetz": m2.group(1).lower(), "paragraph": "", "roh": roh}
        return {"gesetz": "", "paragraph": "", "roh": roh}

    middle, gesetz = m.group(1).strip(), m.group(2).strip()
    # erste Paragraphen-/Artikelnummer
    num = re.search(r"(\d+[a-z]?)", middle, re.I)
    paragraph = num.group(1) if num else ""
    return {
        "gesetz": gesetz.lower().replace("ä", "ae").replace("ö", "oe").replace("ü", "ue"),
        "paragraph": paragraph,
        "roh": roh,
    }


def collect_normen(root: ET.Element) -> list[dict[str, str]]:
    raw_parts: list[str] = []
    for norm_el in root.iter():
        if local_tag(norm_el.tag) != "norm":
            continue
        text = elem_text(norm_el)
        if not text:
            continue
        # Zeilen, Semikolon, und Komma vor neuem §/Art. trennen
        chunks = re.split(r"[\n;]+|,\s*(?=§|Art\.?\s)", text)
        for part in chunks:
            part = part.strip().rstrip(",").strip()
            if not part:
                continue
            if re.search(r"(§{1,2}|Art\.?)\s*\d", part, re.I):
                raw_parts.append(part)
            elif re.match(r"^[A-Za-zÄÖÜäöüß]", part):
                raw_parts.append(part)

    seen: set[str] = set()
    out: list[dict[str, str]] = []
    for roh in raw_parts:
        if roh in seen:
            continue
        seen.add(roh)
        parsed = parse_norm_string(roh)
        if parsed:
            out.append(parsed)
    return out


def collect_vorinstanz(root: ET.Element) -> list[str]:
    out: list[str] = []
    for el in root.iter():
        if local_tag(el.tag) != "vorinstanz":
            continue
        t = elem_text(el)
        if t:
            out.append(t)
        else:
            # strukturierte Vorinstanz: Gericht + AZ
            bits = []
            for child in el:
                ct = elem_text(child)
                if ct:
                    bits.append(ct)
            if bits:
                out.append(" — ".join(bits))
    return out


def yaml_scalar(value: str) -> str:
    """YAML-sicherer Skalar (JSON-String-Stil)."""
    import json

    return json.dumps(value, ensure_ascii=False)


def build_frontmatter(meta: dict[str, Any]) -> str:
    lines = ["---"]
    for key in (
        "ebene",
        "gericht",
        "spruchkoerper",
        "datum",
        "aktenzeichen",
        "ecli",
        "doknr",
        "doktyp",
        "quelle",
    ):
        val = meta.get(key, "")
        if val is None or val == "":
            continue
        lines.append(f"{key}: {yaml_scalar(str(val))}")

    normen = meta.get("normen") or []
    if normen:
        lines.append("normen:")
        for n in normen:
            gesetz = yaml_scalar(n.get("gesetz", ""))
            paragraph = yaml_scalar(n.get("paragraph", ""))
            roh = yaml_scalar(n.get("roh", ""))
            lines.append(
                f"  - {{ gesetz: {gesetz}, paragraph: {paragraph}, roh: {roh} }}"
            )
    else:
        lines.append("normen: []")

    vor = meta.get("vorinstanz") or []
    if vor:
        lines.append("vorinstanz:")
        for v in vor:
            lines.append(f"  - {yaml_scalar(v)}")
    else:
        lines.append("vorinstanz: []")

    lines.append("---")
    return "\n".join(lines)


def section_markdown(root: ET.Element, tag_names: tuple[str, ...]) -> str:
    for name in tag_names:
        el = find_deep(root, name)
        if el is not None:
            # Wenn das Element Kinder mit Markup hat, fragmentarisch konvertieren
            if list(el):
                md = xml_fragment_to_markdown(el)
            else:
                md = elem_text(el)
            if md:
                return md
    return ""


def parse_rii_xml(xml_content: str | bytes) -> dict[str, Any] | None:
    """Parst ein RII-Dokument. Rückgabe: Meta + Body-Abschnitte oder None."""
    if isinstance(xml_content, bytes):
        xml_content = xml_content.decode("utf-8", errors="replace")

    # Externe DTD nicht laden; DOCTYPE entfernen
    xml_content = re.sub(r"<!DOCTYPE[^>]*>", "", xml_content, count=1, flags=re.I)
    cleaned = re.sub(r"&(?!(amp|lt|gt|quot|apos|#\d+|#x[0-9a-fA-F]+);)", "&amp;", xml_content)
    try:
        root = ET.fromstring(cleaned)
    except ET.ParseError:
        return None

    doknr = elem_text(find_deep(root, "doknr")) or root.get("doknr", "")
    gertyp = elem_text(find_deep(root, "gertyp")) or "unbekannt"
    az = elem_text(find_deep(root, "aktenzeichen"))
    if not az and not doknr:
        return None

    datum = iso_date(elem_text(find_deep(root, "entsch-datum")))
    ecli = elem_text(find_deep(root, "ecli"))
    doktyp = elem_text(find_deep(root, "doktyp")) or "Entscheidung"
    spruch = elem_text(find_deep(root, "spruchkoerper"))

    meta = {
        "ebene": ebene_for(gertyp),
        "gericht": gertyp,
        "spruchkoerper": spruch,
        "datum": datum,
        "aktenzeichen": az,
        "ecli": ecli,
        "doknr": doknr,
        "doktyp": doktyp,
        "quelle": "rii",
        "normen": collect_normen(root),
        "vorinstanz": collect_vorinstanz(root),
    }

    body = {
        "leitsatz": section_markdown(root, ("leitsatz",)),
        "tenor": section_markdown(root, ("tenor",)),
        "tatbestand": section_markdown(root, ("tatbestand",)),
        "gruende": section_markdown(
            root, ("entscheidungsgruende", "gruende")
        ),
        "abwmeinung": section_markdown(root, ("abwmeinung",)),
        "titelzeile": section_markdown(root, ("titelzeile",)),
    }
    return {"meta": meta, "body": body}


def to_markdown(parsed: dict[str, Any]) -> str:
    meta = parsed["meta"]
    body = parsed["body"]
    parts = [build_frontmatter(meta), ""]

    title_bits = [meta.get("gericht") or "", meta.get("doktyp") or ""]
    if meta.get("aktenzeichen"):
        title_bits.append(f"— {meta['aktenzeichen']}")
    if meta.get("datum"):
        ddmmyyyy = "-".join(reversed(meta["datum"].split("-"))) if meta["datum"] else ""
        # datum is YYYY-MM-DD → DD.MM.YYYY
        if meta["datum"] and len(meta["datum"]) == 10:
            y, m, d = meta["datum"].split("-")
            ddmmyyyy = f"{d}.{m}.{y}"
        title_bits.append(f"vom {ddmmyyyy}")
    parts.append("# " + " ".join(b for b in title_bits if b).strip())
    parts.append("")

    if body.get("titelzeile"):
        parts.extend(["## Titelzeile", "", body["titelzeile"], ""])
    if body.get("leitsatz"):
        parts.extend(["## Leitsatz", "", body["leitsatz"], ""])
    if body.get("tenor"):
        parts.extend(["## Tenor", "", body["tenor"], ""])
    if body.get("tatbestand"):
        parts.extend(["## Tatbestand", "", body["tatbestand"], ""])
    if body.get("gruende"):
        parts.extend(["## Gründe", "", body["gruende"], ""])
    if body.get("abwmeinung"):
        parts.extend(["## Abweichende Meinung", "", body["abwmeinung"], ""])

    text = "\n".join(parts).rstrip() + "\n"
    text = re.sub(r"\n{3,}", "\n\n", text)
    return text


def relative_path(meta: dict[str, Any]) -> Path:
    """Relativer Pfad unter ge_md/: <court>/<year>/<az-slug>.md"""
    court = court_dir(meta.get("gericht", ""))
    year = (meta.get("datum") or "")[:4] or "ohne-jahr"
    slug = az_slug(meta.get("aktenzeichen", ""), meta.get("doknr", ""))
    return Path(court) / year / f"{slug}.md"


def convert_xml_to_markdown_file(
    xml_content: str | bytes,
    output_base: Path | str = "ge_md",
) -> Path | None:
    """Schreibt Markdown-Datei; Rückgabe: Pfad oder None."""
    parsed = parse_rii_xml(xml_content)
    if not parsed:
        return None
    rel = relative_path(parsed["meta"])
    out = Path(output_base) / rel
    out.parent.mkdir(parents=True, exist_ok=True)
    out.write_text(to_markdown(parsed), encoding="utf-8")
    return out


def doknr_already_exists(doknr: str, output_base: Path | str = "ge_md") -> Path | None:
    """Sucht eine vorhandene Markdown-Datei mit dieser doknr im YAML-Kopf."""
    if not doknr:
        return None
    base = Path(output_base)
    if not base.is_dir():
        return None
    needle = f'doknr: "{doknr}"'
    needle2 = f"doknr: {doknr}"
    for path in base.rglob("*.md"):
        try:
            head = path.read_text(encoding="utf-8", errors="replace")[:800]
        except OSError:
            continue
        if needle in head or f"doknr: {yaml_scalar(doknr)}" in head or needle2 in head:
            return path
    return None