From 3f2e7ca66755bacba7b18b9d0df1382d5f096ada Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 23 Jul 2026 15:48:07 +0200 Subject: [PATCH] =?UTF-8?q?perf:=20Mehrblattimport=20gro=C3=9Fer=20Excel-D?= =?UTF-8?q?ateien=20optimieren?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- netbox_documentation/__init__.py | 2 +- netbox_documentation/importers.py | 71 ++++++++++++++++++++++--------- netbox_documentation/views.py | 15 +++++-- pyproject.toml | 2 +- tests/test_importers.py | 37 ++++++++++++++++ 5 files changed, 100 insertions(+), 27 deletions(-) diff --git a/netbox_documentation/__init__.py b/netbox_documentation/__init__.py index 2fc9414..c20d592 100644 --- a/netbox_documentation/__init__.py +++ b/netbox_documentation/__init__.py @@ -5,7 +5,7 @@ class DocumentationConfig(PluginConfig): name = "netbox_documentation" verbose_name = "NetBox Dokumentation" description = "Wiki und Office-Dokumentation direkt in NetBox" - version = "0.7.5" + version = "0.7.6" author = "LKE" base_url = "documentation" min_version = "4.0.0" diff --git a/netbox_documentation/importers.py b/netbox_documentation/importers.py index b8ed296..0dcc0f7 100644 --- a/netbox_documentation/importers.py +++ b/netbox_documentation/importers.py @@ -98,7 +98,10 @@ def _xlsx(content, flatten=False): return ImportResult("\n\n".join(sections), body_format="markdown" if flatten else "html") -def import_excel_sheets(upload, flatten=False) -> list[ExcelSheetResult]: +def import_excel_sheets( + upload, flatten=False, *, metadata_only=False, preview_max_rows=None, + include_image_data=True, +) -> list[ExcelSheetResult]: """Convert each non-empty worksheet into an individual document payload.""" from openpyxl import load_workbook from openpyxl.utils import get_column_letter @@ -106,14 +109,10 @@ def import_excel_sheets(upload, flatten=False) -> list[ExcelSheetResult]: suffix = Path(upload.name).suffix.lower() if suffix not in {".xlsx", ".xlsm"}: raise ImportFailure("Der Mehrblattimport unterstützt XLSX- und XLSM-Dateien.") - content = upload.read() upload.seek(0) - workbook = load_workbook(BytesIO(content), read_only=False, data_only=True) + workbook = load_workbook(upload, read_only=False, data_only=True) results = [] for sheet in workbook.worksheets: - rows = [["" if cell is None else str(cell) for cell in row] for row in sheet.iter_rows(values_only=True)] - while rows and not any(value for value in rows[-1]): - rows.pop() images = [] for number, image in enumerate(getattr(sheet, "_images", ()), 1): image_format = (getattr(image, "format", None) or "png").lower() @@ -124,29 +123,48 @@ def import_excel_sheets(upload, flatten=False) -> list[ExcelSheetResult]: anchor = getattr(image, "anchor", None) marker = getattr(anchor, "_from", None) cell = f"{get_column_letter(marker.col + 1)}{marker.row + 1}" if marker else "" - try: - image_content = image._data() - except (AttributeError, OSError, ValueError): - continue + image_content = b"" + if include_image_data and not metadata_only: + try: + image_content = image._data() + except (AttributeError, OSError, ValueError): + continue images.append(ImportedImage( name=f"{slugify_filename(sheet.title)}-{number}.{image_format if image_format != 'jpeg' else 'jpg'}", content=image_content, content_type=f"image/{image_format}", cell=cell, )) - if not rows and not images: + has_content = _worksheet_has_content(sheet) + if not has_content and not images: continue - if rows: - markdown = _render_flattened_worksheet(sheet) if flatten else _render_excel_worksheet_html(sheet, workbook) + if metadata_only: + markdown = "" + elif has_content: + markdown = ( + _render_flattened_worksheet(sheet, max_rows=preview_max_rows) if flatten else + _render_excel_worksheet_html(sheet, workbook, max_rows=preview_max_rows) + ) else: markdown = "" results.append(ExcelSheetResult( title=sheet.title, markdown=markdown, images=images, body_format="markdown" if flatten else "html", )) + workbook.close() + upload.seek(0) if not results: raise ImportFailure("Die Arbeitsmappe enthält keine Daten oder unterstützten Bilder.") return results +def _worksheet_has_content(sheet): + """Check for a value without materializing the complete worksheet in memory.""" + return any( + cell.value not in (None, "") + for row in sheet.iter_rows() + for cell in row + ) + + def slugify_filename(value): value = re.sub(r"[^A-Za-z0-9._-]+", "-", value).strip("-.") return value[:80] or "arbeitsblatt" @@ -208,15 +226,20 @@ def _excel_cell_style(cell, workbook): return "; ".join(declarations) -def _render_excel_worksheet_html(sheet, workbook): +def _render_excel_worksheet_html(sheet, workbook, max_rows=None): """Render worksheet values and the most relevant visual Excel cell formatting.""" from openpyxl.utils import get_column_letter - populated = [cell for row in sheet.iter_rows() for cell in row if cell.value not in (None, "")] - if not populated: + last_row = last_column = 0 + for row in sheet.iter_rows(): + for cell in row: + if cell.value not in (None, ""): + last_row = max(last_row, cell.row) + last_column = max(last_column, cell.column) + if not last_row: return "

Keine Inhalte

" - max_row = max(cell.row for cell in populated) - max_column = max(cell.column for cell in populated) + max_row = min(last_row, max_rows) if max_rows else last_row + max_column = last_column merged_starts, merged_children = {}, set() for merged in sheet.merged_cells.ranges: if merged.min_row > max_row or merged.min_col > max_column: @@ -256,10 +279,12 @@ def _render_excel_worksheet_html(sheet, workbook): output.append(f"{value}") output.append("") output.append("") + if max_rows and last_row > max_rows: + output.append(f"

Vorschau auf {max_rows} von {last_row} Zeilen begrenzt.

") return "".join(output) -def _render_flattened_worksheet(sheet): +def _render_flattened_worksheet(sheet, max_rows=None): """Flatten ordinary cells while preserving explicitly defined Excel tables.""" from openpyxl.utils.cell import range_boundaries @@ -278,7 +303,8 @@ def _render_flattened_worksheet(sheet): blocks = [] rendered_tables = set() - for row_number in range(1, sheet.max_row + 1): + rendered_max_row = min(sheet.max_row, max_rows) if max_rows else sheet.max_row + for row_number in range(1, rendered_max_row + 1): ordinary_values = [] tables_starting_here = [] for column_number in range(1, sheet.max_column + 1): @@ -296,11 +322,14 @@ def _render_flattened_worksheet(sheet): for min_col, min_row, max_col, max_row in tables_starting_here: rows = [] for table_row in sheet.iter_rows( - min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col, values_only=True + min_row=min_row, max_row=min(max_row, rendered_max_row), + min_col=min_col, max_col=max_col, values_only=True ): rows.append(["" if value is None else str(value) for value in table_row]) if rows: blocks.append(_render_excel_rows(rows, flatten=False)) + if max_rows and sheet.max_row > max_rows: + blocks.append(f"_Vorschau auf {max_rows} von {sheet.max_row} Zeilen begrenzt._") return "\n\n".join(blocks) or "_Keine Inhalte_" diff --git a/netbox_documentation/views.py b/netbox_documentation/views.py index ff58fea..68bc34e 100644 --- a/netbox_documentation/views.py +++ b/netbox_documentation/views.py @@ -184,7 +184,10 @@ class DocumentImportView(PermissionRequiredMixin, View): def _import_excel_sheets(self, request, form, upload): try: - sheets = import_excel_sheets(upload, flatten=form.cleaned_data.get("flatten_excel_tables", False)) + sheets = import_excel_sheets( + upload, flatten=form.cleaned_data.get("flatten_excel_tables", False), + metadata_only=True, include_image_data=False, + ) except (ImportFailure, Exception) as exc: form.add_error("file", f"Excel-Mehrblattimport fehlgeschlagen: {exc}") return render(request, self.template_name, {"form": form}) @@ -210,10 +213,14 @@ class ExcelImportPreviewView(PermissionRequiredMixin, View): def get_preview(self, request, token): return get_object_or_404(ExcelImportPreview, pk=token, user=request.user) - def read_sheets(self, preview): + def read_sheets(self, preview, *, for_preview=False): preview.file.open("rb") try: - return import_excel_sheets(preview.file, flatten=preview.flatten) + return import_excel_sheets( + preview.file, flatten=preview.flatten, + preview_max_rows=100 if for_preview else None, + include_image_data=not for_preview, + ) finally: preview.file.close() @@ -249,7 +256,7 @@ class ExcelImportPreviewView(PermissionRequiredMixin, View): messages.error(request, "Diese Importvorschau ist abgelaufen. Bitte die Exceldatei erneut hochladen.") return redirect("plugins:netbox_documentation:document_import") try: - sheets = self.read_sheets(preview) + sheets = self.read_sheets(preview, for_preview=True) except (ImportFailure, Exception) as exc: self.delete_preview(preview) messages.error(request, f"Die Excel-Vorschau konnte nicht erzeugt werden: {exc}") diff --git a/pyproject.toml b/pyproject.toml index 114465c..7842bf8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "netbox-documentation" -version = "0.7.5" +version = "0.7.6" description = "Integrated Markdown wiki and office document importer for NetBox" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_importers.py b/tests/test_importers.py index b00a44e..2b80e62 100644 --- a/tests/test_importers.py +++ b/tests/test_importers.py @@ -90,6 +90,43 @@ def test_excel_multi_sheet_import_creates_one_result_per_sheet(): assert "Leitstelle" in results[1].markdown +def test_excel_metadata_phase_skips_document_rendering(): + workbook = Workbook() + sheet = workbook.active + sheet.title = "Großes Blatt" + sheet.append(["Name", "IP"]) + sheet.append(["web01", "10.0.0.1"]) + stream = BytesIO() + workbook.save(stream) + + results = import_excel_sheets( + Upload(stream.getvalue(), "kunde.xlsx"), metadata_only=True, + include_image_data=False, + ) + + assert [result.title for result in results] == ["Großes Blatt"] + assert results[0].markdown == "" + + +def test_excel_preview_limits_rows_without_affecting_sheet_detection(): + workbook = Workbook() + sheet = workbook.active + sheet.append(["Nummer"]) + for number in range(1, 151): + sheet.append([number]) + stream = BytesIO() + workbook.save(stream) + + results = import_excel_sheets( + Upload(stream.getvalue(), "gross.xlsx"), preview_max_rows=100, + include_image_data=False, + ) + + assert "Vorschau auf 100 von 151 Zeilen begrenzt" in results[0].markdown + assert ">99<" in results[0].markdown + assert ">100<" not in results[0].markdown + + def test_excel_flatten_turns_cells_into_document_flow(): workbook = Workbook() sheet = workbook.active