Initial commit
This commit is contained in:
577
ecodms_extract.py
Normal file
577
ecodms_extract.py
Normal file
@@ -0,0 +1,577 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
ecodms_extract.py - Stufe 1 der ecoDMS -> Paperless-ngx Migration.
|
||||
|
||||
Liest eine Exporttranche (export.data + export.xml + archive/) und erzeugt
|
||||
ein normalisiertes JSON-Manifest plus einen Pruefbericht. Beruehrt Paperless
|
||||
nicht - die Stufe ist offline testbar und beliebig oft wiederholbar.
|
||||
|
||||
Aufruf:
|
||||
./ecodms_extract.py /pfad/zu/offline_export/archive -o tranche_01.json
|
||||
|
||||
Die hier kodierten Regeln stammen aus der Analyse dreier Testexporte:
|
||||
|
||||
1. Gruppierung nach docid, nicht nach docs_id. Eine docid kann mehrere
|
||||
Klassifizierungsinstanzen haben (Mehrfachablage in mehreren Ordnern).
|
||||
2. Beim Zusammenfuehren gewinnt bei Skalaren der juengste ctimestamp,
|
||||
Ordner werden als Menge vereinigt.
|
||||
3. Immer realname (Original) verwenden, nie pdfrealname. Sonst gehen
|
||||
eingebettete ZUGFeRD-XML und E-Mail-Anhaenge verloren.
|
||||
4. Dokumente ohne Zeilen in ecosimsversions fallen auf ecosimsarchive
|
||||
zurueck und werden als einzige Version behandelt.
|
||||
5. docrollen_hist.docid enthaelt docs_id, nicht docid. Join ueber docs.id.
|
||||
6. Rechte sind historisiert. Nur die hoechste in docrollen_hist
|
||||
vorhandene Revision je docs_id gilt - nicht die aus klassifizierung.
|
||||
7. Revisionen werden als Zahlentupel sortiert ('1.10' > '1.9').
|
||||
8. base64 an unregelmaessigen Stellen: ecosimsversions.savedate und
|
||||
.filename sowie ecosimsarchive.version sind kodiert, realname und
|
||||
pdfrealname nicht.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import binascii
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
SCHEMA_VERSION = 2
|
||||
|
||||
# Dateitypen, die Paperless-ngx konsumieren kann. Massgeblich ist die
|
||||
# laufende Instanz:
|
||||
# docker compose exec webserver python manage.py shell -c \
|
||||
# "from documents.parsers import get_supported_file_extensions as f; print(sorted(f()))"
|
||||
#
|
||||
# Dokumente mit anderen Endungen werden nicht ins Manifest aufgenommen,
|
||||
# sondern in eine Ausschlussliste geschrieben. Sonst scheitern sie erst
|
||||
# beim Upload mit HTTP 400 und muellen das Journal zu.
|
||||
SUPPORTED_EXTENSIONS = {
|
||||
".pdf", ".txt", ".text", ".csv", ".rtf", ".xml", ".eml", ".mail",
|
||||
".mht", ".mhtml", ".nws", ".brf", ".srt", ".rdf", ".wsdl", ".xsl", ".xpdl",
|
||||
".doc", ".docx", ".dot", ".odt",
|
||||
".xls", ".xlsx", ".xlb", ".xlc", ".xlm", ".xlt", ".xlw", ".xla", ".ods",
|
||||
".ppt", ".pptx", ".pps", ".ppsx", ".pot", ".ppa", ".pwz", ".wiz", ".odp",
|
||||
".odg",
|
||||
".png", ".jpg", ".jpe", ".jpeg", ".jfif", ".gif", ".bmp", ".webp",
|
||||
".tif", ".tiff", ".heic", ".art",
|
||||
".bat", ".c", ".h", ".ksh", ".pl",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- Hilfsmittel
|
||||
|
||||
def b64(value):
|
||||
"""Dekodiert base64, gibt den Originalwert zurueck wenn das misslingt."""
|
||||
if not value:
|
||||
return value
|
||||
try:
|
||||
return base64.b64decode(value, validate=True).decode("utf-8")
|
||||
except (binascii.Error, UnicodeDecodeError, ValueError):
|
||||
return value
|
||||
|
||||
|
||||
def revtuple(revision):
|
||||
"""'1.10' -> (1, 10). Lexikografische Sortierung waere hier falsch."""
|
||||
try:
|
||||
return tuple(int(p) for p in str(revision).split("."))
|
||||
except (TypeError, ValueError):
|
||||
return (-1,)
|
||||
|
||||
|
||||
def clean(value):
|
||||
"""ecoDMS schreibt teils den String 'null' statt eines Leerwerts."""
|
||||
if value is None:
|
||||
return ""
|
||||
value = str(value).strip()
|
||||
return "" if value.lower() == "null" else value
|
||||
|
||||
|
||||
class Report:
|
||||
"""Sammelt Warnungen nach Kategorie, damit der Bericht lesbar bleibt."""
|
||||
|
||||
def __init__(self):
|
||||
self.items = defaultdict(list)
|
||||
|
||||
def warn(self, category, message):
|
||||
self.items[category].append(message)
|
||||
|
||||
def __len__(self):
|
||||
return sum(len(v) for v in self.items.values())
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- Stammdaten
|
||||
|
||||
class Lookups:
|
||||
def __init__(self, con):
|
||||
self.docart = {
|
||||
r["daid"]: (r["name"], r["trashed"] == "true")
|
||||
for r in con.execute("SELECT daid, name, trashed FROM documentenart")
|
||||
}
|
||||
self.status = {
|
||||
r["sid"]: (r["name"], r["trashed"] == "true")
|
||||
for r in con.execute("SELECT sid, name, trashed FROM status")
|
||||
}
|
||||
self.folder = {
|
||||
r["oid"]: dict(name=r["name"], parent=r["parentid"], deleted=r["deleted"])
|
||||
for r in con.execute(
|
||||
"SELECT oid, name, parentid, deleted FROM systemordner"
|
||||
)
|
||||
}
|
||||
# Nur nicht geloeschte Attribute werden zu Custom Fields.
|
||||
self.dyn = {
|
||||
r["spaltenname"]: dict(name=r["name"], typ=r["datatyp"])
|
||||
for r in con.execute(
|
||||
"SELECT spaltenname, name, datatyp FROM dynattribute "
|
||||
"WHERE deleted <> 'true'"
|
||||
)
|
||||
}
|
||||
|
||||
def folder_parts(self, oid):
|
||||
"""
|
||||
'6.8.1.2' -> ['Allgemeines', 'Steuer', 'ESt', 'Anlage N']
|
||||
|
||||
Bewusst eine Liste statt eines verketteten Pfads: Ordnernamen in
|
||||
ecoDMS koennen selbst Schraegstriche enthalten ("Telefon / Internet"),
|
||||
ein spaeteres Zerlegen am Schraegstrich wuerde solche Namen
|
||||
zerschneiden.
|
||||
"""
|
||||
parts, seen = [], set()
|
||||
while oid and oid not in ("-1", "") and oid not in seen:
|
||||
seen.add(oid)
|
||||
node = self.folder.get(oid)
|
||||
if node is None:
|
||||
parts.append(f"?{oid}")
|
||||
break
|
||||
parts.append(node["name"].strip())
|
||||
oid = node["parent"]
|
||||
return list(reversed(parts))
|
||||
|
||||
def folder_path(self, oid):
|
||||
"""Nur fuer Anzeige und Bericht."""
|
||||
return " / ".join(self.folder_parts(oid))
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- Rechte
|
||||
|
||||
def effective_rights(con, docs_ids, report):
|
||||
"""
|
||||
Wirksame Rechte je docs_id: nur Zeilen der hoechsten dort vorhandenen
|
||||
Revision. Aeltere Revisionen enthalten z.B. noch ecoSIMSUSER aus dem
|
||||
Posteingang, das darf nicht mitwandern.
|
||||
"""
|
||||
result = {}
|
||||
for docs_id in docs_ids:
|
||||
rows = list(
|
||||
con.execute(
|
||||
"SELECT role, doc_right, revision FROM docrollen_hist WHERE docid = ?",
|
||||
(docs_id,),
|
||||
)
|
||||
)
|
||||
if not rows:
|
||||
report.warn("rechte_fehlen", f"docs_id {docs_id} ohne Rechteeintrag")
|
||||
result[docs_id] = []
|
||||
continue
|
||||
top = max(revtuple(r["revision"]) for r in rows)
|
||||
result[docs_id] = [
|
||||
{"role": r["role"], "right": r["doc_right"], "revision": r["revision"]}
|
||||
for r in rows
|
||||
if revtuple(r["revision"]) == top
|
||||
]
|
||||
return result
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ Versionen
|
||||
|
||||
def unsupported_ext(filename):
|
||||
"""Gibt die Endung zurueck, wenn Paperless sie nicht konsumieren kann."""
|
||||
ext = ("." + filename.rsplit(".", 1)[-1].lower()) if "." in filename else ""
|
||||
return None if ext in SUPPORTED_EXTENSIONS else (ext or "(ohne Endung)")
|
||||
|
||||
|
||||
def versions_for(con, docid, archive_dir, report):
|
||||
rows = list(
|
||||
con.execute(
|
||||
"SELECT version, savedate, filename, realname, pdfrealname, size, "
|
||||
" comment, fixiert, checksum "
|
||||
"FROM ecosimsversions WHERE docid = ? ORDER BY version",
|
||||
(docid,),
|
||||
)
|
||||
)
|
||||
|
||||
if rows:
|
||||
out = []
|
||||
for r in rows:
|
||||
fname = r["realname"]
|
||||
if not fname:
|
||||
report.warn(
|
||||
"datei_fehlt",
|
||||
f"docid {docid} Version {r['version']}: kein realname",
|
||||
)
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"version": r["version"],
|
||||
"file": fname,
|
||||
"original_name": b64(r["filename"]) or r["realname"],
|
||||
"saved_at": b64(r["savedate"]),
|
||||
"comment": clean(r["comment"]),
|
||||
"fixed": r["fixiert"] == "true",
|
||||
"checksum": r["checksum"],
|
||||
"source": "ecosimsversions",
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
# Regel 4: Rueckfall auf das Archivdokument.
|
||||
arc = con.execute(
|
||||
"SELECT filename, realname, size, inserttime, version, fixiert, checksum "
|
||||
"FROM ecosimsarchive WHERE id = ?",
|
||||
(docid,),
|
||||
).fetchone()
|
||||
if arc is None or not arc["realname"]:
|
||||
report.warn("ohne_datei", f"docid {docid}: weder Version noch Archivdatei")
|
||||
return []
|
||||
report.warn("nur_archiv", f"docid {docid}: keine Versionszeilen, nutze Archivdatei")
|
||||
return [
|
||||
{
|
||||
"version": 1,
|
||||
"file": arc["realname"],
|
||||
"original_name": arc["filename"],
|
||||
"saved_at": arc["inserttime"],
|
||||
"comment": "",
|
||||
"fixed": arc["fixiert"] == "true",
|
||||
"checksum": arc["checksum"],
|
||||
"source": "ecosimsarchive",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
# --------------------------------------------------------- Klassifizierungen
|
||||
|
||||
def merge_classifications(rows, lookups, report, docid):
|
||||
"""
|
||||
Regel 2: Ordner vereinigen, bei Skalaren gewinnt der juengste ctimestamp.
|
||||
'rows' sind alle klassifizierung-Zeilen einer docid.
|
||||
"""
|
||||
rows = sorted(rows, key=lambda r: r["ctimestamp"] or "")
|
||||
newest = rows[-1]
|
||||
|
||||
folders, folder_parts, conflicts = [], [], {}
|
||||
for r in rows:
|
||||
parts = lookups.folder_parts(r["folder"] or r["mainfolder"])
|
||||
if parts and parts not in folder_parts:
|
||||
folder_parts.append(parts)
|
||||
folders.append(" / ".join(parts))
|
||||
|
||||
if len(rows) > 1:
|
||||
report.warn(
|
||||
"mehrfachklassifizierung",
|
||||
f"docid {docid}: {len(rows)} Klassifizierungen -> {', '.join(folders)}",
|
||||
)
|
||||
for field in ("bemerkung", "docart", "status", "cdate", "defdate"):
|
||||
values = {clean(r[field]) for r in rows}
|
||||
if len(values) > 1:
|
||||
conflicts[field] = sorted(values)
|
||||
|
||||
docart_name, docart_trashed = lookups.docart.get(
|
||||
newest["docart"], (f"?{newest['docart']}", False)
|
||||
)
|
||||
status_name, status_trashed = lookups.status.get(
|
||||
newest["status"], (f"?{newest['status']}", False)
|
||||
)
|
||||
if docart_trashed:
|
||||
report.warn("geloeschte_stammdaten", f"docid {docid}: Dokumentart '{docart_name}' ist geloescht")
|
||||
if status_trashed:
|
||||
report.warn("geloeschte_stammdaten", f"docid {docid}: Status '{status_name}' ist geloescht")
|
||||
|
||||
custom = {}
|
||||
for column, meta in lookups.dyn.items():
|
||||
value = clean(newest[column] if column in newest.keys() else "")
|
||||
if value:
|
||||
custom[meta["name"]] = {"value": value, "type": meta["typ"]}
|
||||
|
||||
# Wiedervorlage steckt in klassifizierung.defdate, nicht in dynattribute -
|
||||
# sie ist in ecoDMS eine eingebaute Funktion, kein dynamisches Attribut.
|
||||
# Passt inhaltlich zum Statuswert "Wiedervorlage" (sid 2).
|
||||
wv = clean(newest["defdate"] if "defdate" in newest.keys() else "")
|
||||
if wv:
|
||||
custom["Wiedervorlage"] = {"value": wv, "type": "Date"}
|
||||
|
||||
return {
|
||||
"title": clean(newest["bemerkung"]) or f"ecoDMS {docid}",
|
||||
"created": clean(newest["cdate"]),
|
||||
"changed_at": clean(newest["ctimestamp"]),
|
||||
"changed_by": clean(newest["changeid"]),
|
||||
"document_type": docart_name,
|
||||
"status": status_name,
|
||||
"folders": folders, # nur Anzeige
|
||||
"folder_parts": folder_parts, # massgeblich fuer die Tags
|
||||
"custom_fields": custom,
|
||||
"revision": clean(newest["revision"]),
|
||||
"conflicts": conflicts,
|
||||
}
|
||||
|
||||
|
||||
# ------------------------------------------------------------- XML-Historie
|
||||
|
||||
def classification_history(xml_path, report):
|
||||
"""Die Klassifizierungshistorie steht nur in der XML, nicht in der SQLite."""
|
||||
history = defaultdict(list)
|
||||
if not xml_path.exists():
|
||||
report.warn("xml_fehlt", f"{xml_path.name} nicht gefunden")
|
||||
return history, set()
|
||||
|
||||
root = ET.parse(xml_path).getroot()
|
||||
referenced = set()
|
||||
for doc in root.findall("document"):
|
||||
docid = int(doc.get("docid"))
|
||||
for el in doc.iter():
|
||||
if el.get("filePath"):
|
||||
referenced.add(el.get("filePath"))
|
||||
for info in doc.findall(".//classifyInfo"):
|
||||
for ver in info.findall("Version"):
|
||||
entry = {
|
||||
child.tag: clean(child.text) for child in ver if clean(child.text)
|
||||
}
|
||||
entry["cla_docs_id"] = info.get("cla_docs_id")
|
||||
history[docid].append(entry)
|
||||
for docid in history:
|
||||
history[docid].sort(key=lambda e: revtuple(e.get("revision", "0")))
|
||||
return history, referenced
|
||||
|
||||
|
||||
def notes_by_docid(con, report):
|
||||
"""
|
||||
econotice enthaelt die Notizfunktion, nicht die Wiedervorlage.
|
||||
|
||||
Zwei Fallstricke:
|
||||
* nid enthaelt die docs_id, NICHT die docid - dasselbe Muster wie bei
|
||||
docrollen_hist. Inhaltlich verifiziert an drei Faellen mit
|
||||
auseinanderlaufenden IDs.
|
||||
* text ist base64-kodiertes Qt-HTML mit CSS-Block im Kopf.
|
||||
|
||||
username ist im gesamten Bestand leer, eine Zuordnung zu Personen
|
||||
entfaellt also.
|
||||
"""
|
||||
notes = defaultdict(list)
|
||||
docs_id_to_docid = {
|
||||
r["id"]: r["docid"] for r in con.execute("SELECT id, docid FROM docs")
|
||||
}
|
||||
|
||||
for row in con.execute("SELECT * FROM econotice ORDER BY id"):
|
||||
try:
|
||||
docs_id = int(row["nid"])
|
||||
except (TypeError, ValueError):
|
||||
report.warn("notiz_ohne_bezug", f"econotice {row['id']}: nid={row['nid']!r}")
|
||||
continue
|
||||
docid = docs_id_to_docid.get(docs_id)
|
||||
if docid is None:
|
||||
report.warn(
|
||||
"notiz_ohne_bezug",
|
||||
f"econotice {row['id']}: docs_id {docs_id} nicht im Export",
|
||||
)
|
||||
continue
|
||||
|
||||
raw = row["text"] or ""
|
||||
try:
|
||||
text = base64.b64decode(raw).decode("utf-8", "replace")
|
||||
except (binascii.Error, ValueError):
|
||||
text = str(raw)
|
||||
text = re.sub(r"<[^>]+>", " ", text) # HTML-Tags
|
||||
text = html.unescape(re.sub(r"\s+", " ", text)).strip()
|
||||
text = re.sub(r"^p,\s*li\s*\{[^}]*\}\s*", "", text) # CSS-Rest
|
||||
|
||||
notes[docid].append(
|
||||
{"created": clean(row["tdate"]), "text": text,
|
||||
"user": clean(row["username"])}
|
||||
)
|
||||
return notes
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ Hauptlauf
|
||||
|
||||
def extract(archive_dir: Path, tranche: str):
|
||||
report = Report()
|
||||
db_path = archive_dir / "export.data"
|
||||
xml_path = archive_dir / "export.xml"
|
||||
if not db_path.exists():
|
||||
sys.exit(f"export.data nicht gefunden unter {archive_dir}")
|
||||
|
||||
con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
|
||||
con.row_factory = sqlite3.Row
|
||||
|
||||
lookups = Lookups(con)
|
||||
history, referenced = classification_history(xml_path, report)
|
||||
notes = notes_by_docid(con, report)
|
||||
|
||||
# klassifizierung nach docid gruppieren (Regel 1)
|
||||
by_docid = defaultdict(list)
|
||||
for row in con.execute("SELECT * FROM klassifizierung"):
|
||||
by_docid[row["docid"]].append(row)
|
||||
|
||||
# docid -> docs_id (fuer den Rechte-Join, Regel 5)
|
||||
docs_ids = defaultdict(list)
|
||||
for row in con.execute("SELECT id, docid, trashed FROM docs"):
|
||||
if row["trashed"] == "true":
|
||||
report.warn("papierkorb", f"docs_id {row['id']} (docid {row['docid']}) ist im Papierkorb")
|
||||
continue
|
||||
docs_ids[row["docid"]].append(row["id"])
|
||||
|
||||
on_disk = {p.name for p in archive_dir.iterdir() if p.is_file()}
|
||||
documents, used_files, skipped = [], set(), []
|
||||
|
||||
for docid in sorted(by_docid):
|
||||
meta = merge_classifications(by_docid[docid], lookups, report, docid)
|
||||
versions = versions_for(con, docid, archive_dir, report)
|
||||
|
||||
for v in versions:
|
||||
used_files.add(v["file"])
|
||||
if on_disk and v["file"] not in on_disk:
|
||||
report.warn("datei_fehlt", f"docid {docid} v{v['version']}: {v['file']}")
|
||||
|
||||
# Nicht konsumierbare Dateitypen aussortieren. Massgeblich ist die
|
||||
# letzte Version - sie bestimmt, was Paperless zu sehen bekaeme.
|
||||
bad = {v["file"]: unsupported_ext(v["file"]) for v in versions}
|
||||
bad = {f: e for f, e in bad.items() if e}
|
||||
if bad:
|
||||
exts = sorted(set(bad.values()))
|
||||
report.warn("nicht_unterstuetzt",
|
||||
f"docid {docid}: {', '.join(exts)} ({meta['title'][:50]})")
|
||||
skipped.append({
|
||||
"ecodms_docid": docid,
|
||||
"title": meta["title"],
|
||||
"extensions": exts,
|
||||
"files": sorted(bad),
|
||||
"folders": meta["folders"],
|
||||
"created": meta["created"],
|
||||
})
|
||||
continue
|
||||
|
||||
rights = effective_rights(con, docs_ids.get(docid, []), report)
|
||||
roles = {}
|
||||
for entries in rights.values(): # Vereinigung ueber alle Instanzen
|
||||
for e in entries:
|
||||
prev = roles.get(e["role"])
|
||||
# W schlaegt R
|
||||
if prev is None or (prev == "R" and e["right"] == "W"):
|
||||
roles[e["role"]] = e["right"]
|
||||
|
||||
if not roles:
|
||||
report.warn("ohne_rechte", f"docid {docid}: keine wirksamen Rechte")
|
||||
|
||||
documents.append(
|
||||
{
|
||||
"ecodms_docid": docid,
|
||||
"tranche": tranche,
|
||||
**meta,
|
||||
"versions": versions,
|
||||
"permissions": [{"role": r, "right": w} for r, w in sorted(roles.items())],
|
||||
"classification_history": history.get(docid, []),
|
||||
"notes": notes.get(docid, []),
|
||||
"docs_ids": docs_ids.get(docid, []),
|
||||
}
|
||||
)
|
||||
|
||||
# Dateien, die auf der Platte liegen aber nicht verwendet werden.
|
||||
ignorable = {"export.xml", "export.data"}
|
||||
orphans = sorted(on_disk - used_files - ignorable) if on_disk else []
|
||||
for name in orphans:
|
||||
# Die gerenderten Kopf-PDFs sind erwartete Waisen (Regel 3).
|
||||
if "_revision_" not in name:
|
||||
continue
|
||||
report.warn("verwaiste_datei", name)
|
||||
|
||||
missing_refs = sorted(referenced - on_disk) if on_disk else []
|
||||
for name in missing_refs:
|
||||
report.warn("xml_referenz_ohne_datei", name)
|
||||
|
||||
manifest = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"tranche": tranche,
|
||||
"source": str(archive_dir),
|
||||
"counts": {
|
||||
"documents": len(documents),
|
||||
"skipped": len(skipped),
|
||||
"versions": sum(len(d["versions"]) for d in documents),
|
||||
"multi_classified": sum(1 for d in documents if len(d["docs_ids"]) > 1),
|
||||
"roles": len({p["role"] for d in documents for p in d["permissions"]}),
|
||||
"notes": sum(len(d["notes"]) for d in documents),
|
||||
},
|
||||
"roles_seen": sorted({p["role"] for d in documents for p in d["permissions"]}),
|
||||
"rights_seen": sorted({p["right"] for d in documents for p in d["permissions"]}),
|
||||
"documents": documents,
|
||||
"skipped": skipped,
|
||||
}
|
||||
return manifest, report
|
||||
|
||||
|
||||
def print_report(manifest, report):
|
||||
c = manifest["counts"]
|
||||
print(f"Tranche {manifest['tranche']}")
|
||||
print(f" Dokumente {c['documents']}")
|
||||
if c.get("skipped"):
|
||||
exts = {}
|
||||
for sk in manifest["skipped"]:
|
||||
for e in sk["extensions"]:
|
||||
exts[e] = exts.get(e, 0) + 1
|
||||
detail = ", ".join(f"{e} {n}" for e, n in sorted(exts.items()))
|
||||
print(f" Uebersprungen {c['skipped']} ({detail})")
|
||||
print(f" Versionen {c['versions']}")
|
||||
print(f" Mehrfachklassifiziert {c['multi_classified']}")
|
||||
print(f" Notizen {c['notes']}")
|
||||
print(f" Rollen {', '.join(manifest['roles_seen']) or '-'}")
|
||||
print(f" Rechtearten {', '.join(manifest['rights_seen']) or '-'}")
|
||||
if len(report):
|
||||
print(f"\n Hinweise ({len(report)}):")
|
||||
for category in sorted(report.items):
|
||||
entries = report.items[category]
|
||||
print(f" [{category}] {len(entries)}")
|
||||
for line in entries[:5]:
|
||||
print(f" {line}")
|
||||
if len(entries) > 5:
|
||||
print(f" ... und {len(entries) - 5} weitere")
|
||||
else:
|
||||
print("\n Keine Hinweise.")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("archive_dir", type=Path, help="Verzeichnis mit export.data")
|
||||
ap.add_argument("-o", "--output", type=Path, help="Ziel fuer das JSON-Manifest")
|
||||
ap.add_argument("-t", "--tranche", default=None, help="Name der Tranche")
|
||||
ap.add_argument("--skipped-csv", type=Path,
|
||||
help="Ausschlussliste als CSV fuer die Nacharbeit")
|
||||
args = ap.parse_args()
|
||||
|
||||
tranche = args.tranche or args.archive_dir.resolve().parent.name
|
||||
manifest, report = extract(args.archive_dir, tranche)
|
||||
print_report(manifest, report)
|
||||
|
||||
if args.skipped_csv and manifest["skipped"]:
|
||||
import csv
|
||||
with args.skipped_csv.open("w", newline="", encoding="utf-8") as fh:
|
||||
w = csv.writer(fh)
|
||||
w.writerow(["ecodms_docid", "endungen", "titel", "erstellt",
|
||||
"ordner", "dateien"])
|
||||
for sk in manifest["skipped"]:
|
||||
w.writerow([sk["ecodms_docid"], " ".join(sk["extensions"]),
|
||||
sk["title"], sk["created"],
|
||||
" | ".join(sk["folders"]), " ".join(sk["files"])])
|
||||
print(f" Ausschlussliste: {args.skipped_csv}")
|
||||
|
||||
if args.output:
|
||||
args.output.write_text(
|
||||
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
print(f"\n Manifest geschrieben: {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user