perf: Mehrblattimport großer Excel-Dateien optimieren

netbox_documentation/__init__.py

@@ -5,7 +5,7 @@ class DocumentationConfig(PluginConfig):
     name = "netbox_documentation"
     verbose_name = "NetBox Dokumentation"
     description = "Wiki und Office-Dokumentation direkt in NetBox"
-    version = "0.7.5"
+    version = "0.7.6"
     author = "LKE"
     base_url = "documentation"
     min_version = "4.0.0"

netbox_documentation/importers.py

@@ -98,7 +98,10 @@ def _xlsx(content, flatten=False):
     return ImportResult("\n\n".join(sections), body_format="markdown" if flatten else "html")
 
 
-def import_excel_sheets(upload, flatten=False) -> list[ExcelSheetResult]:
+def import_excel_sheets(
+    upload, flatten=False, *, metadata_only=False, preview_max_rows=None,
+    include_image_data=True,
+) -> list[ExcelSheetResult]:
     """Convert each non-empty worksheet into an individual document payload."""
     from openpyxl import load_workbook
     from openpyxl.utils import get_column_letter
@@ -106,14 +109,10 @@ def import_excel_sheets(upload, flatten=False) -> list[ExcelSheetResult]:
     suffix = Path(upload.name).suffix.lower()
     if suffix not in {".xlsx", ".xlsm"}:
         raise ImportFailure("Der Mehrblattimport unterstützt XLSX- und XLSM-Dateien.")
-    content = upload.read()
     upload.seek(0)
-    workbook = load_workbook(BytesIO(content), read_only=False, data_only=True)
+    workbook = load_workbook(upload, read_only=False, data_only=True)
     results = []
     for sheet in workbook.worksheets:
-        rows = [["" if cell is None else str(cell) for cell in row] for row in sheet.iter_rows(values_only=True)]
-        while rows and not any(value for value in rows[-1]):
-            rows.pop()
         images = []
         for number, image in enumerate(getattr(sheet, "_images", ()), 1):
             image_format = (getattr(image, "format", None) or "png").lower()
@@ -124,29 +123,48 @@ def import_excel_sheets(upload, flatten=False) -> list[ExcelSheetResult]:
             anchor = getattr(image, "anchor", None)
             marker = getattr(anchor, "_from", None)
             cell = f"{get_column_letter(marker.col + 1)}{marker.row + 1}" if marker else ""
-            try:
+            image_content = b""
-                image_content = image._data()
+            if include_image_data and not metadata_only:
-            except (AttributeError, OSError, ValueError):
+                try:
-                continue
+                    image_content = image._data()
+                except (AttributeError, OSError, ValueError):
+                    continue
             images.append(ImportedImage(
                 name=f"{slugify_filename(sheet.title)}-{number}.{image_format if image_format != 'jpeg' else 'jpg'}",
                 content=image_content, content_type=f"image/{image_format}", cell=cell,
             ))
-        if not rows and not images:
+        has_content = _worksheet_has_content(sheet)
+        if not has_content and not images:
             continue
-        if rows:
+        if metadata_only:
-            markdown = _render_flattened_worksheet(sheet) if flatten else _render_excel_worksheet_html(sheet, workbook)
+            markdown = ""
+        elif has_content:
+            markdown = (
+                _render_flattened_worksheet(sheet, max_rows=preview_max_rows) if flatten else
+                _render_excel_worksheet_html(sheet, workbook, max_rows=preview_max_rows)
+            )
         else:
             markdown = ""
         results.append(ExcelSheetResult(
             title=sheet.title, markdown=markdown, images=images,
             body_format="markdown" if flatten else "html",
         ))
+    workbook.close()
+    upload.seek(0)
     if not results:
         raise ImportFailure("Die Arbeitsmappe enthält keine Daten oder unterstützten Bilder.")
     return results
 
 
+def _worksheet_has_content(sheet):
+    """Check for a value without materializing the complete worksheet in memory."""
+    return any(
+        cell.value not in (None, "")
+        for row in sheet.iter_rows()
+        for cell in row
+    )
+
+
 def slugify_filename(value):
     value = re.sub(r"[^A-Za-z0-9._-]+", "-", value).strip("-.")
     return value[:80] or "arbeitsblatt"
@@ -208,15 +226,20 @@ def _excel_cell_style(cell, workbook):
     return "; ".join(declarations)
 
 
-def _render_excel_worksheet_html(sheet, workbook):
+def _render_excel_worksheet_html(sheet, workbook, max_rows=None):
     """Render worksheet values and the most relevant visual Excel cell formatting."""
     from openpyxl.utils import get_column_letter
 
-    populated = [cell for row in sheet.iter_rows() for cell in row if cell.value not in (None, "")]
+    last_row = last_column = 0
-    if not populated:
+    for row in sheet.iter_rows():
+        for cell in row:
+            if cell.value not in (None, ""):
+                last_row = max(last_row, cell.row)
+                last_column = max(last_column, cell.column)
+    if not last_row:
         return "<p><em>Keine Inhalte</em></p>"
-    max_row = max(cell.row for cell in populated)
+    max_row = min(last_row, max_rows) if max_rows else last_row
-    max_column = max(cell.column for cell in populated)
+    max_column = last_column
     merged_starts, merged_children = {}, set()
     for merged in sheet.merged_cells.ranges:
         if merged.min_row > max_row or merged.min_col > max_column:
@@ -256,10 +279,12 @@ def _render_excel_worksheet_html(sheet, workbook):
             output.append(f"<td{' ' if attributes else ''}{' '.join(attributes)}>{value}</td>")
         output.append("</tr>")
     output.append("</tbody></table>")
+    if max_rows and last_row > max_rows:
+        output.append(f"<p><em>Vorschau auf {max_rows} von {last_row} Zeilen begrenzt.</em></p>")
     return "".join(output)
 
 
-def _render_flattened_worksheet(sheet):
+def _render_flattened_worksheet(sheet, max_rows=None):
     """Flatten ordinary cells while preserving explicitly defined Excel tables."""
     from openpyxl.utils.cell import range_boundaries
 
@@ -278,7 +303,8 @@ def _render_flattened_worksheet(sheet):
 
     blocks = []
     rendered_tables = set()
-    for row_number in range(1, sheet.max_row + 1):
+    rendered_max_row = min(sheet.max_row, max_rows) if max_rows else sheet.max_row
+    for row_number in range(1, rendered_max_row + 1):
         ordinary_values = []
         tables_starting_here = []
         for column_number in range(1, sheet.max_column + 1):
@@ -296,11 +322,14 @@ def _render_flattened_worksheet(sheet):
         for min_col, min_row, max_col, max_row in tables_starting_here:
             rows = []
             for table_row in sheet.iter_rows(
-                min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col, values_only=True
+                min_row=min_row, max_row=min(max_row, rendered_max_row),
+                min_col=min_col, max_col=max_col, values_only=True
             ):
                 rows.append(["" if value is None else str(value) for value in table_row])
             if rows:
                 blocks.append(_render_excel_rows(rows, flatten=False))
+    if max_rows and sheet.max_row > max_rows:
+        blocks.append(f"_Vorschau auf {max_rows} von {sheet.max_row} Zeilen begrenzt._")
     return "\n\n".join(blocks) or "_Keine Inhalte_"
 
 

netbox_documentation/views.py

@@ -184,7 +184,10 @@ class DocumentImportView(PermissionRequiredMixin, View):
 
     def _import_excel_sheets(self, request, form, upload):
         try:
-            sheets = import_excel_sheets(upload, flatten=form.cleaned_data.get("flatten_excel_tables", False))
+            sheets = import_excel_sheets(
+                upload, flatten=form.cleaned_data.get("flatten_excel_tables", False),
+                metadata_only=True, include_image_data=False,
+            )
         except (ImportFailure, Exception) as exc:
             form.add_error("file", f"Excel-Mehrblattimport fehlgeschlagen: {exc}")
             return render(request, self.template_name, {"form": form})
@@ -210,10 +213,14 @@ class ExcelImportPreviewView(PermissionRequiredMixin, View):
     def get_preview(self, request, token):
         return get_object_or_404(ExcelImportPreview, pk=token, user=request.user)
 
-    def read_sheets(self, preview):
+    def read_sheets(self, preview, *, for_preview=False):
         preview.file.open("rb")
         try:
-            return import_excel_sheets(preview.file, flatten=preview.flatten)
+            return import_excel_sheets(
+                preview.file, flatten=preview.flatten,
+                preview_max_rows=100 if for_preview else None,
+                include_image_data=not for_preview,
+            )
         finally:
             preview.file.close()
 
@@ -249,7 +256,7 @@ class ExcelImportPreviewView(PermissionRequiredMixin, View):
             messages.error(request, "Diese Importvorschau ist abgelaufen. Bitte die Exceldatei erneut hochladen.")
             return redirect("plugins:netbox_documentation:document_import")
         try:
-            sheets = self.read_sheets(preview)
+            sheets = self.read_sheets(preview, for_preview=True)
         except (ImportFailure, Exception) as exc:
             self.delete_preview(preview)
             messages.error(request, f"Die Excel-Vorschau konnte nicht erzeugt werden: {exc}")

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "netbox-documentation"
-version = "0.7.5"
+version = "0.7.6"
 description = "Integrated Markdown wiki and office document importer for NetBox"
 readme = "README.md"
 requires-python = ">=3.10"

tests/test_importers.py

@@ -90,6 +90,43 @@ def test_excel_multi_sheet_import_creates_one_result_per_sheet():
     assert "Leitstelle" in results[1].markdown
 
 
+def test_excel_metadata_phase_skips_document_rendering():
+    workbook = Workbook()
+    sheet = workbook.active
+    sheet.title = "Großes Blatt"
+    sheet.append(["Name", "IP"])
+    sheet.append(["web01", "10.0.0.1"])
+    stream = BytesIO()
+    workbook.save(stream)
+
+    results = import_excel_sheets(
+        Upload(stream.getvalue(), "kunde.xlsx"), metadata_only=True,
+        include_image_data=False,
+    )
+
+    assert [result.title for result in results] == ["Großes Blatt"]
+    assert results[0].markdown == ""
+
+
+def test_excel_preview_limits_rows_without_affecting_sheet_detection():
+    workbook = Workbook()
+    sheet = workbook.active
+    sheet.append(["Nummer"])
+    for number in range(1, 151):
+        sheet.append([number])
+    stream = BytesIO()
+    workbook.save(stream)
+
+    results = import_excel_sheets(
+        Upload(stream.getvalue(), "gross.xlsx"), preview_max_rows=100,
+        include_image_data=False,
+    )
+
+    assert "Vorschau auf 100 von 151 Zeilen begrenzt" in results[0].markdown
+    assert ">99<" in results[0].markdown
+    assert ">100<" not in results[0].markdown
+
+
 def test_excel_flatten_turns_cells_into_document_flow():
     workbook = Workbook()
     sheet = workbook.active