MrBlake/Netbox-Documentation
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: Mehrblattimport großer Excel-Dateien optimieren

Browse Source
This commit is contained in:
MrBlake
2026-07-23 15:48:07 +02:00
parent e4f4cd7c14
commit 3f2e7ca667
5 changed files with 100 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
netbox_documentation/__init__.py
+1 -1
View File
@@ -5,7 +5,7 @@ class DocumentationConfig(PluginConfig):
name = "netbox_documentation"
verbose_name = "NetBox Dokumentation"
description = "Wiki und Office-Dokumentation direkt in NetBox"
version = "0.7.5"
version = "0.7.6"
author = "LKE"
base_url = "documentation"
min_version = "4.0.0"
netbox_documentation/importers.py
+50 -21
View File
@@ -98,7 +98,10 @@ def _xlsx(content, flatten=False):
return ImportResult("\n\n".join(sections), body_format="markdown" if flatten else "html")
def import_excel_sheets(upload, flatten=False) -> list[ExcelSheetResult]:
def import_excel_sheets(
upload, flatten=False, *, metadata_only=False, preview_max_rows=None,
include_image_data=True,
) -> list[ExcelSheetResult]:
"""Convert each non-empty worksheet into an individual document payload."""
from openpyxl import load_workbook
from openpyxl.utils import get_column_letter
@@ -106,14 +109,10 @@ def import_excel_sheets(upload, flatten=False) -> list[ExcelSheetResult]:
suffix = Path(upload.name).suffix.lower()
if suffix not in {".xlsx", ".xlsm"}:
raise ImportFailure("Der Mehrblattimport unterstützt XLSX- und XLSM-Dateien.")
content = upload.read()
upload.seek(0)
workbook = load_workbook(BytesIO(content), read_only=False, data_only=True)
workbook = load_workbook(upload, read_only=False, data_only=True)
results = []
for sheet in workbook.worksheets:
rows = [["" if cell is None else str(cell) for cell in row] for row in sheet.iter_rows(values_only=True)]
while rows and not any(value for value in rows[-1]):
rows.pop()
images = []
for number, image in enumerate(getattr(sheet, "_images", ()), 1):
image_format = (getattr(image, "format", None) or "png").lower()
@@ -124,29 +123,48 @@ def import_excel_sheets(upload, flatten=False) -> list[ExcelSheetResult]:
anchor = getattr(image, "anchor", None)
marker = getattr(anchor, "_from", None)
cell = f"{get_column_letter(marker.col + 1)}{marker.row + 1}" if marker else ""
try:
image_content = image._data()
except (AttributeError, OSError, ValueError):
continue
image_content = b""
if include_image_data and not metadata_only:
try:
image_content = image._data()
except (AttributeError, OSError, ValueError):
continue
images.append(ImportedImage(
name=f"{slugify_filename(sheet.title)}-{number}.{image_format if image_format != 'jpeg' else 'jpg'}",
content=image_content, content_type=f"image/{image_format}", cell=cell,
))
if not rows and not images:
has_content = _worksheet_has_content(sheet)
if not has_content and not images:
continue
if rows:
markdown = _render_flattened_worksheet(sheet) if flatten else _render_excel_worksheet_html(sheet, workbook)
if metadata_only:
markdown = ""
elif has_content:
markdown = (
_render_flattened_worksheet(sheet, max_rows=preview_max_rows) if flatten else
_render_excel_worksheet_html(sheet, workbook, max_rows=preview_max_rows)
)
else:
markdown = ""
results.append(ExcelSheetResult(
title=sheet.title, markdown=markdown, images=images,
body_format="markdown" if flatten else "html",
))
workbook.close()
upload.seek(0)
if not results:
raise ImportFailure("Die Arbeitsmappe enthält keine Daten oder unterstützten Bilder.")
return results
def _worksheet_has_content(sheet):
"""Check for a value without materializing the complete worksheet in memory."""
return any(
cell.value not in (None, "")
for row in sheet.iter_rows()
for cell in row
)
def slugify_filename(value):
value = re.sub(r"[^A-Za-z0-9._-]+", "-", value).strip("-.")
return value[:80] or "arbeitsblatt"
@@ -208,15 +226,20 @@ def _excel_cell_style(cell, workbook):
return "; ".join(declarations)
def _render_excel_worksheet_html(sheet, workbook):
def _render_excel_worksheet_html(sheet, workbook, max_rows=None):
"""Render worksheet values and the most relevant visual Excel cell formatting."""
from openpyxl.utils import get_column_letter
populated = [cell for row in sheet.iter_rows() for cell in row if cell.value not in (None, "")]
if not populated:
last_row = last_column = 0
for row in sheet.iter_rows():
for cell in row:
if cell.value not in (None, ""):
last_row = max(last_row, cell.row)
last_column = max(last_column, cell.column)
if not last_row:
return "<p><em>Keine Inhalte</em></p>"
max_row = max(cell.row for cell in populated)
max_column = max(cell.column for cell in populated)
max_row = min(last_row, max_rows) if max_rows else last_row
max_column = last_column
merged_starts, merged_children = {}, set()
for merged in sheet.merged_cells.ranges:
if merged.min_row > max_row or merged.min_col > max_column:
@@ -256,10 +279,12 @@ def _render_excel_worksheet_html(sheet, workbook):
output.append(f"<td{' ' if attributes else ''}{' '.join(attributes)}>{value}</td>")
output.append("</tr>")
output.append("</tbody></table>")
if max_rows and last_row > max_rows:
output.append(f"<p><em>Vorschau auf {max_rows} von {last_row} Zeilen begrenzt.</em></p>")
return "".join(output)
def _render_flattened_worksheet(sheet):
def _render_flattened_worksheet(sheet, max_rows=None):
"""Flatten ordinary cells while preserving explicitly defined Excel tables."""
from openpyxl.utils.cell import range_boundaries
@@ -278,7 +303,8 @@ def _render_flattened_worksheet(sheet):
blocks = []
rendered_tables = set()
for row_number in range(1, sheet.max_row + 1):
rendered_max_row = min(sheet.max_row, max_rows) if max_rows else sheet.max_row
for row_number in range(1, rendered_max_row + 1):
ordinary_values = []
tables_starting_here = []
for column_number in range(1, sheet.max_column + 1):
@@ -296,11 +322,14 @@ def _render_flattened_worksheet(sheet):
for min_col, min_row, max_col, max_row in tables_starting_here:
rows = []
for table_row in sheet.iter_rows(
min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col, values_only=True
min_row=min_row, max_row=min(max_row, rendered_max_row),
min_col=min_col, max_col=max_col, values_only=True
):
rows.append(["" if value is None else str(value) for value in table_row])
if rows:
blocks.append(_render_excel_rows(rows, flatten=False))
if max_rows and sheet.max_row > max_rows:
blocks.append(f"_Vorschau auf {max_rows} von {sheet.max_row} Zeilen begrenzt._")
return "\n\n".join(blocks) or "_Keine Inhalte_"
netbox_documentation/views.py
+11 -4
View File
@@ -184,7 +184,10 @@ class DocumentImportView(PermissionRequiredMixin, View):
def _import_excel_sheets(self, request, form, upload):
try:
sheets = import_excel_sheets(upload, flatten=form.cleaned_data.get("flatten_excel_tables", False))
sheets = import_excel_sheets(
upload, flatten=form.cleaned_data.get("flatten_excel_tables", False),
metadata_only=True, include_image_data=False,
)
except (ImportFailure, Exception) as exc:
form.add_error("file", f"Excel-Mehrblattimport fehlgeschlagen: {exc}")
return render(request, self.template_name, {"form": form})
@@ -210,10 +213,14 @@ class ExcelImportPreviewView(PermissionRequiredMixin, View):
def get_preview(self, request, token):
return get_object_or_404(ExcelImportPreview, pk=token, user=request.user)
def read_sheets(self, preview):
def read_sheets(self, preview, *, for_preview=False):
preview.file.open("rb")
try:
return import_excel_sheets(preview.file, flatten=preview.flatten)
return import_excel_sheets(
preview.file, flatten=preview.flatten,
preview_max_rows=100 if for_preview else None,
include_image_data=not for_preview,
)
finally:
preview.file.close()
@@ -249,7 +256,7 @@ class ExcelImportPreviewView(PermissionRequiredMixin, View):
messages.error(request, "Diese Importvorschau ist abgelaufen. Bitte die Exceldatei erneut hochladen.")
return redirect("plugins:netbox_documentation:document_import")
try:
sheets = self.read_sheets(preview)
sheets = self.read_sheets(preview, for_preview=True)
except (ImportFailure, Exception) as exc:
self.delete_preview(preview)
messages.error(request, f"Die Excel-Vorschau konnte nicht erzeugt werden: {exc}")