perf: Mehrblattimport großer Excel-Dateien optimieren

netbox_documentation/__init__.py

@@ -5,7 +5,7 @@ class DocumentationConfig(PluginConfig):
     name = "netbox_documentation"
     verbose_name = "NetBox Dokumentation"
     description = "Wiki und Office-Dokumentation direkt in NetBox"
-    version = "0.7.4"
+    version = "0.7.6"
     author = "LKE"
     base_url = "documentation"
     min_version = "4.0.0"

netbox_documentation/importers.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass, field
 from io import BytesIO
 from pathlib import Path
+from html import escape
 import re
 
 
@@ -12,6 +13,7 @@ class ImportFailure(ValueError):
 class ImportResult:
     markdown: str
     warnings: list[str] = field(default_factory=list)
+    body_format: str = "markdown"
 
 
 @dataclass
@@ -27,6 +29,7 @@ class ExcelSheetResult:
     title: str
     markdown: str
     images: list[ImportedImage] = field(default_factory=list)
+    body_format: str = "markdown"
 
 
 def import_document(upload, flatten_excel=False) -> ImportResult:
@@ -80,7 +83,7 @@ def _escape_markdown_text(value):
 
 def _xlsx(content, flatten=False):
     from openpyxl import load_workbook
-    workbook = load_workbook(BytesIO(content), read_only=not flatten, data_only=True)
+    workbook = load_workbook(BytesIO(content), read_only=False, data_only=True)
     sections = []
     for sheet in workbook.worksheets:
         rows = [["" if cell is None else str(cell) for cell in row] for row in sheet.iter_rows(values_only=True)]
@@ -88,14 +91,17 @@ def _xlsx(content, flatten=False):
             rows.pop()
         if not rows:
             continue
-        rendered = _render_flattened_worksheet(sheet) if flatten else _render_excel_rows(rows)
-        sections.append(f"## {sheet.title}\n\n{rendered}")
+        rendered = _render_flattened_worksheet(sheet) if flatten else _render_excel_worksheet_html(sheet, workbook)
+        sections.append(f"## {sheet.title}\n\n{rendered}" if flatten else f"<h2>{escape(sheet.title)}</h2>\n{rendered}")
     if not sections:
         raise ImportFailure("Die Arbeitsmappe enthält keine Daten.")
-    return ImportResult("\n\n".join(sections))
+    return ImportResult("\n\n".join(sections), body_format="markdown" if flatten else "html")
 
 
-def import_excel_sheets(upload, flatten=False) -> list[ExcelSheetResult]:
+def import_excel_sheets(
+    upload, flatten=False, *, metadata_only=False, preview_max_rows=None,
+    include_image_data=True,
+) -> list[ExcelSheetResult]:
     """Convert each non-empty worksheet into an individual document payload."""
     from openpyxl import load_workbook
     from openpyxl.utils import get_column_letter
@@ -103,14 +109,10 @@ def import_excel_sheets(upload, flatten=False) -> list[ExcelSheetResult]:
     suffix = Path(upload.name).suffix.lower()
     if suffix not in {".xlsx", ".xlsm"}:
         raise ImportFailure("Der Mehrblattimport unterstützt XLSX- und XLSM-Dateien.")
-    content = upload.read()
     upload.seek(0)
-    workbook = load_workbook(BytesIO(content), read_only=False, data_only=True)
+    workbook = load_workbook(upload, read_only=False, data_only=True)
     results = []
     for sheet in workbook.worksheets:
-        rows = [["" if cell is None else str(cell) for cell in row] for row in sheet.iter_rows(values_only=True)]
-        while rows and not any(value for value in rows[-1]):
-            rows.pop()
         images = []
         for number, image in enumerate(getattr(sheet, "_images", ()), 1):
             image_format = (getattr(image, "format", None) or "png").lower()
@@ -121,32 +123,168 @@ def import_excel_sheets(upload, flatten=False) -> list[ExcelSheetResult]:
             anchor = getattr(image, "anchor", None)
             marker = getattr(anchor, "_from", None)
             cell = f"{get_column_letter(marker.col + 1)}{marker.row + 1}" if marker else ""
-            try:
-                image_content = image._data()
-            except (AttributeError, OSError, ValueError):
-                continue
+            image_content = b""
+            if include_image_data and not metadata_only:
+                try:
+                    image_content = image._data()
+                except (AttributeError, OSError, ValueError):
+                    continue
             images.append(ImportedImage(
                 name=f"{slugify_filename(sheet.title)}-{number}.{image_format if image_format != 'jpeg' else 'jpg'}",
                 content=image_content, content_type=f"image/{image_format}", cell=cell,
             ))
-        if not rows and not images:
+        has_content = _worksheet_has_content(sheet)
+        if not has_content and not images:
             continue
-        if rows:
-            markdown = _render_flattened_worksheet(sheet) if flatten else _render_excel_rows(rows)
+        if metadata_only:
+            markdown = ""
+        elif has_content:
+            markdown = (
+                _render_flattened_worksheet(sheet, max_rows=preview_max_rows) if flatten else
+                _render_excel_worksheet_html(sheet, workbook, max_rows=preview_max_rows)
+            )
         else:
             markdown = ""
-        results.append(ExcelSheetResult(title=sheet.title, markdown=markdown, images=images))
+        results.append(ExcelSheetResult(
+            title=sheet.title, markdown=markdown, images=images,
+            body_format="markdown" if flatten else "html",
+        ))
+    workbook.close()
+    upload.seek(0)
     if not results:
         raise ImportFailure("Die Arbeitsmappe enthält keine Daten oder unterstützten Bilder.")
     return results
 
 
+def _worksheet_has_content(sheet):
+    """Check for a value without materializing the complete worksheet in memory."""
+    return any(
+        cell.value not in (None, "")
+        for row in sheet.iter_rows()
+        for cell in row
+    )
+
+
 def slugify_filename(value):
     value = re.sub(r"[^A-Za-z0-9._-]+", "-", value).strip("-.")
     return value[:80] or "arbeitsblatt"
 
 
-def _render_flattened_worksheet(sheet):
+def _excel_color(color, workbook):
+    """Resolve RGB, indexed and theme-based openpyxl colors to a CSS hex value."""
+    if not color or not getattr(color, "type", None):
+        return None
+    value = None
+    if color.type == "rgb" and color.rgb:
+        value = str(color.rgb)[-6:]
+    elif color.type == "indexed" and color.indexed is not None:
+        from openpyxl.styles.colors import COLOR_INDEX
+        index = int(color.indexed)
+        if 0 <= index < len(COLOR_INDEX):
+            value = COLOR_INDEX[index][-6:]
+    elif color.type == "theme" and color.theme is not None and workbook.loaded_theme:
+        from xml.etree import ElementTree
+        try:
+            root = ElementTree.fromstring(workbook.loaded_theme)
+            scheme = root.find(".//{http://schemas.openxmlformats.org/drawingml/2006/main}clrScheme")
+            entries = list(scheme) if scheme is not None else []
+            entry = entries[int(color.theme)]
+            color_node = next(iter(entry))
+            value = color_node.attrib.get("val") or color_node.attrib.get("lastClr")
+        except (ElementTree.ParseError, IndexError, StopIteration, TypeError, ValueError):
+            value = None
+    if not value or not re.fullmatch(r"[0-9A-Fa-f]{6}", value):
+        return None
+    rgb = [int(value[index:index + 2], 16) for index in (0, 2, 4)]
+    tint = float(getattr(color, "tint", 0) or 0)
+    if tint:
+        rgb = [round(component * (1 + tint) if tint < 0 else component + (255 - component) * tint) for component in rgb]
+    return "#" + "".join(f"{max(0, min(255, component)):02x}" for component in rgb)
+
+
+def _excel_cell_style(cell, workbook):
+    declarations = []
+    if cell.fill and cell.fill.fill_type == "solid":
+        fill = _excel_color(cell.fill.fgColor, workbook)
+        if fill:
+            declarations.append(f"background-color: {fill}")
+    font_color = _excel_color(cell.font.color, workbook)
+    if font_color:
+        declarations.append(f"color: {font_color}")
+    if cell.font.bold:
+        declarations.append("font-weight: bold")
+    if cell.font.italic:
+        declarations.append("font-style: italic")
+    if cell.font.sz:
+        declarations.append(f"font-size: {float(cell.font.sz):g}pt")
+    if cell.alignment.horizontal in {"left", "center", "right", "justify"}:
+        declarations.append(f"text-align: {cell.alignment.horizontal}")
+    if cell.alignment.vertical in {"top", "center", "bottom"}:
+        declarations.append(f"vertical-align: {'middle' if cell.alignment.vertical == 'center' else cell.alignment.vertical}")
+    if cell.alignment.wrap_text:
+        declarations.append("white-space: normal")
+    return "; ".join(declarations)
+
+
+def _render_excel_worksheet_html(sheet, workbook, max_rows=None):
+    """Render worksheet values and the most relevant visual Excel cell formatting."""
+    from openpyxl.utils import get_column_letter
+
+    last_row = last_column = 0
+    for row in sheet.iter_rows():
+        for cell in row:
+            if cell.value not in (None, ""):
+                last_row = max(last_row, cell.row)
+                last_column = max(last_column, cell.column)
+    if not last_row:
+        return "<p><em>Keine Inhalte</em></p>"
+    max_row = min(last_row, max_rows) if max_rows else last_row
+    max_column = last_column
+    merged_starts, merged_children = {}, set()
+    for merged in sheet.merged_cells.ranges:
+        if merged.min_row > max_row or merged.min_col > max_column:
+            continue
+        merged_starts[(merged.min_row, merged.min_col)] = (
+            min(merged.max_row, max_row) - merged.min_row + 1,
+            min(merged.max_col, max_column) - merged.min_col + 1,
+        )
+        for row in range(merged.min_row, min(merged.max_row, max_row) + 1):
+            for column in range(merged.min_col, min(merged.max_col, max_column) + 1):
+                if (row, column) != (merged.min_row, merged.min_col):
+                    merged_children.add((row, column))
+    columns = []
+    for column in range(1, max_column + 1):
+        width = sheet.column_dimensions[get_column_letter(column)].width
+        columns.append(f'<col style="width: {max(3, min(float(width or 13), 80)):.2f}ch">')
+    output = ['<table class="doc-table-bordered"><colgroup>', *columns, "</colgroup><tbody>"]
+    for row in range(1, max_row + 1):
+        row_style = ""
+        if sheet.row_dimensions[row].height:
+            row_style = f' style="height: {float(sheet.row_dimensions[row].height):g}pt"'
+        output.append(f"<tr{row_style}>")
+        for column in range(1, max_column + 1):
+            if (row, column) in merged_children:
+                continue
+            cell = sheet.cell(row=row, column=column)
+            attributes = []
+            rowspan, colspan = merged_starts.get((row, column), (1, 1))
+            if rowspan > 1:
+                attributes.append(f'rowspan="{rowspan}"')
+            if colspan > 1:
+                attributes.append(f'colspan="{colspan}"')
+            style = _excel_cell_style(cell, workbook)
+            if style:
+                attributes.append(f'style="{style}"')
+            value = "" if cell.value is None else escape(str(cell.value)).replace("\n", "<br>")
+            output.append(f"<td{' ' if attributes else ''}{' '.join(attributes)}>{value}</td>")
+        output.append("</tr>")
+    output.append("</tbody></table>")
+    if max_rows and last_row > max_rows:
+        output.append(f"<p><em>Vorschau auf {max_rows} von {last_row} Zeilen begrenzt.</em></p>")
+    return "".join(output)
+
+
+def _render_flattened_worksheet(sheet, max_rows=None):
     """Flatten ordinary cells while preserving explicitly defined Excel tables."""
     from openpyxl.utils.cell import range_boundaries
 
@@ -165,7 +303,8 @@ def _render_flattened_worksheet(sheet):
 
     blocks = []
     rendered_tables = set()
-    for row_number in range(1, sheet.max_row + 1):
+    rendered_max_row = min(sheet.max_row, max_rows) if max_rows else sheet.max_row
+    for row_number in range(1, rendered_max_row + 1):
         ordinary_values = []
         tables_starting_here = []
         for column_number in range(1, sheet.max_column + 1):
@@ -183,11 +322,14 @@ def _render_flattened_worksheet(sheet):
         for min_col, min_row, max_col, max_row in tables_starting_here:
             rows = []
             for table_row in sheet.iter_rows(
-                min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col, values_only=True
+                min_row=min_row, max_row=min(max_row, rendered_max_row),
+                min_col=min_col, max_col=max_col, values_only=True
             ):
                 rows.append(["" if value is None else str(value) for value in table_row])
             if rows:
                 blocks.append(_render_excel_rows(rows, flatten=False))
+    if max_rows and sheet.max_row > max_rows:
+        blocks.append(f"_Vorschau auf {max_rows} von {sheet.max_row} Zeilen begrenzt._")
     return "\n\n".join(blocks) or "_Keine Inhalte_"
 
 

netbox_documentation/views.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 from datetime import timedelta
+from html import escape
 import mimetypes
 import tinymce
 from django.conf import settings
@@ -168,7 +169,9 @@ class DocumentImportView(PermissionRequiredMixin, View):
                 while Document.objects.filter(slug=slug).exists():
                     slug = f"{base_slug}-{counter}"
                     counter += 1
-                document = Document.objects.create(title=title, slug=slug, body=result.markdown, body_format="markdown")
+                document = Document.objects.create(
+                    title=title, slug=slug, body=result.markdown, body_format=result.body_format,
+                )
             keep = settings.PLUGINS_CONFIG.get("netbox_documentation", {}).get("keep_imported_file", True)
             if keep:
                 upload.seek(0)
@@ -181,7 +184,10 @@ class DocumentImportView(PermissionRequiredMixin, View):
 
     def _import_excel_sheets(self, request, form, upload):
         try:
-            sheets = import_excel_sheets(upload, flatten=form.cleaned_data.get("flatten_excel_tables", False))
+            sheets = import_excel_sheets(
+                upload, flatten=form.cleaned_data.get("flatten_excel_tables", False),
+                metadata_only=True, include_image_data=False,
+            )
         except (ImportFailure, Exception) as exc:
             form.add_error("file", f"Excel-Mehrblattimport fehlgeschlagen: {exc}")
             return render(request, self.template_name, {"form": form})
@@ -207,10 +213,14 @@ class ExcelImportPreviewView(PermissionRequiredMixin, View):
     def get_preview(self, request, token):
         return get_object_or_404(ExcelImportPreview, pk=token, user=request.user)
 
-    def read_sheets(self, preview):
+    def read_sheets(self, preview, *, for_preview=False):
         preview.file.open("rb")
         try:
-            return import_excel_sheets(preview.file, flatten=preview.flatten)
+            return import_excel_sheets(
+                preview.file, flatten=preview.flatten,
+                preview_max_rows=100 if for_preview else None,
+                include_image_data=not for_preview,
+            )
         finally:
             preview.file.close()
 
@@ -219,11 +229,18 @@ class ExcelImportPreviewView(PermissionRequiredMixin, View):
         for form, sheet in zip(formset.forms, sheets):
             sample = sheet.markdown
             if sheet.images:
-                sample += f"\n\n## Bilder\n\n_{len(sheet.images)} eingebettete Bilder werden beim Import angefügt._"
+                image_notice = f"{len(sheet.images)} eingebettete Bilder werden beim Import angefügt."
+                if sheet.body_format == "html":
+                    sample += f"\n<h2>Bilder</h2><p><em>{image_notice}</em></p>"
+                else:
+                    sample += f"\n\n## Bilder\n\n_{image_notice}_"
             truncated = len(sample) > 100000
             if truncated:
-                sample = sample[:100000] + "\n\n_… Vorschau gekürzt …_"
-            preview_document = Document(body=sample, body_format="markdown")
+                notice = "… Vorschau gekürzt …"
+                sample = sample[:100000] + (
+                    f"<p><em>{notice}</em></p>" if sheet.body_format == "html" else f"\n\n_{notice}_"
+                )
+            preview_document = Document(body=sample, body_format=sheet.body_format)
             items.append({
                 "form": form, "sheet": sheet,
                 "preview_html": preview_document.rendered_body(), "truncated": truncated,
@@ -239,7 +256,7 @@ class ExcelImportPreviewView(PermissionRequiredMixin, View):
             messages.error(request, "Diese Importvorschau ist abgelaufen. Bitte die Exceldatei erneut hochladen.")
             return redirect("plugins:netbox_documentation:document_import")
         try:
-            sheets = self.read_sheets(preview)
+            sheets = self.read_sheets(preview, for_preview=True)
         except (ImportFailure, Exception) as exc:
             self.delete_preview(preview)
             messages.error(request, f"Die Excel-Vorschau konnte nicht erzeugt werden: {exc}")
@@ -300,7 +317,7 @@ class ExcelImportPreviewView(PermissionRequiredMixin, View):
                 counter += 1
             document = Document.objects.create(
                 title=title[:200], slug=document_slug, body=sheet.markdown,
-                body_format="markdown", category=preview.category,
+                body_format=sheet.body_format, category=preview.category,
                 summary=f"Importiert aus {preview.original_name}",
             )
             if sheet.images:
@@ -313,8 +330,16 @@ class ExcelImportPreviewView(PermissionRequiredMixin, View):
                     attachment.file.save(image.name, ContentFile(image.content), save=False)
                     attachment.save()
                     location = f" ({image.cell})" if image.cell else ""
-                    image_lines.append(f"![{image.name}{location}]({attachment.file.url})")
-                document.body += "\n\n## Bilder\n\n" + "\n\n".join(image_lines)
+                    if document.body_format == "html":
+                        image_lines.append(
+                            f'<p><img src="{escape(attachment.file.url)}" alt="{escape(image.name + location)}"></p>'
+                        )
+                    else:
+                        image_lines.append(f"![{image.name}{location}]({attachment.file.url})")
+                if document.body_format == "html":
+                    document.body += "\n<h2>Bilder</h2>\n" + "\n".join(image_lines)
+                else:
+                    document.body += "\n\n## Bilder\n\n" + "\n\n".join(image_lines)
                 document.save(update_fields=("body", "last_updated"))
                 image_count += len(sheet.images)
             created_documents.append(document)

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "netbox-documentation"
-version = "0.7.4"
+version = "0.7.6"
 description = "Integrated Markdown wiki and office document importer for NetBox"
 readme = "README.md"
 requires-python = ">=3.10"

tests/test_importers.py

@@ -5,6 +5,7 @@ import sys
 
 import pytest
 from openpyxl import Workbook
+from openpyxl.styles import PatternFill, Font
 from openpyxl.worksheet.table import Table
 
 
@@ -39,10 +40,31 @@ def test_xlsx_imports_sheets_as_tables():
     stream = BytesIO()
     workbook.save(stream)
     result = import_document(Upload(stream.getvalue(), "server.xlsx"))
-    assert "## Server" in result.markdown
+    assert "<h2>Server</h2>" in result.markdown
+    assert result.body_format == "html"
     assert "web01" in result.markdown
 
 
+def test_xlsx_preserves_cell_fill_and_font_colors_as_html():
+    workbook = Workbook()
+    sheet = workbook.active
+    sheet["A1"] = "Standort"
+    sheet["A1"].fill = PatternFill(fill_type="solid", fgColor="A5BF60")
+    sheet["A1"].font = Font(color="FFFFFF", bold=True)
+    sheet["A2"] = "Verwaltung"
+    sheet["A2"].fill = PatternFill(fill_type="solid", fgColor="EAF0DC")
+    stream = BytesIO()
+    workbook.save(stream)
+
+    result = import_document(Upload(stream.getvalue(), "farben.xlsx"))
+
+    assert result.body_format == "html"
+    assert "background-color: #a5bf60" in result.markdown
+    assert "color: #ffffff" in result.markdown
+    assert "font-weight: bold" in result.markdown
+    assert "background-color: #eaf0dc" in result.markdown
+
+
 def test_rejects_legacy_excel():
     with pytest.raises(ImportFailure, match="xlsx"):
         import_document(Upload(b"", "legacy.xls"))
@@ -68,6 +90,43 @@ def test_excel_multi_sheet_import_creates_one_result_per_sheet():
     assert "Leitstelle" in results[1].markdown
 
 
+def test_excel_metadata_phase_skips_document_rendering():
+    workbook = Workbook()
+    sheet = workbook.active
+    sheet.title = "Großes Blatt"
+    sheet.append(["Name", "IP"])
+    sheet.append(["web01", "10.0.0.1"])
+    stream = BytesIO()
+    workbook.save(stream)
+
+    results = import_excel_sheets(
+        Upload(stream.getvalue(), "kunde.xlsx"), metadata_only=True,
+        include_image_data=False,
+    )
+
+    assert [result.title for result in results] == ["Großes Blatt"]
+    assert results[0].markdown == ""
+
+
+def test_excel_preview_limits_rows_without_affecting_sheet_detection():
+    workbook = Workbook()
+    sheet = workbook.active
+    sheet.append(["Nummer"])
+    for number in range(1, 151):
+        sheet.append([number])
+    stream = BytesIO()
+    workbook.save(stream)
+
+    results = import_excel_sheets(
+        Upload(stream.getvalue(), "gross.xlsx"), preview_max_rows=100,
+        include_image_data=False,
+    )
+
+    assert "Vorschau auf 100 von 151 Zeilen begrenzt" in results[0].markdown
+    assert ">99<" in results[0].markdown
+    assert ">100<" not in results[0].markdown
+
+
 def test_excel_flatten_turns_cells_into_document_flow():
     workbook = Workbook()
     sheet = workbook.active