From 26aea40e6ae6d0f36598d91e667aad1a76caf246 Mon Sep 17 00:00:00 2001 From: Louis-Alexander Kerst Date: Mon, 24 Aug 2026 21:37:17 +0200 Subject: [PATCH] feat: install plugins from approved source commits --- README.md | 6 +- host_agent/README.md | 7 +- host_agent/examples/agent.toml | 9 +- host_agent/src/netbox_store_agent/catalog.py | 80 ++++++++++++-- host_agent/src/netbox_store_agent/config.py | 14 ++- host_agent/src/netbox_store_agent/executor.py | 100 ++++++++++++++++-- host_agent/src/netbox_store_agent/journal.py | 16 ++- .../src/netbox_store_agent/managed_files.py | 12 ++- host_agent/tests/support.py | 2 + host_agent/tests/test_catalog.py | 32 ++++++ netbox_plugin/netbox_plugin_store/client.py | 4 + .../netbox_plugin_store/lifecycle.py | 4 + store/public/assets/app.css | 12 +++ store/src/Domain/Approval.php | 28 ++++- store/src/Domain/Catalog.php | 2 + store/src/Http/Application.php | 3 + store/src/Sync/Adapter/ForgejoAdapter.php | 11 ++ store/src/Sync/Adapter/GitHubAdapter.php | 10 ++ store/src/Sync/Adapter/SourceAdapter.php | 3 + store/src/Sync/SyncService.php | 30 +++++- store/templates/admin/dashboard.php | 4 +- store/templates/home.php | 4 +- store/templates/installation.php | 42 ++++++++ store/templates/partials/footer.php | 1 + store/templates/partials/head.php | 1 + store/templates/plugin.php | 8 +- store/tests/run.php | 16 ++- 27 files changed, 414 insertions(+), 47 deletions(-) create mode 100644 store/templates/installation.php diff --git a/README.md b/README.md index 3736078..70f255f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Dieses Repository besteht aus drei getrennten Komponenten: | --- | --- | | `store/` | Eigenständiger PHP-Store, direkt durch Apache ausgeliefert. Er synchronisiert Forgejo-/GitHub-Repositories, rendert README-Dateien und stellt den kuratierten API-Katalog bereit. | | `netbox_plugin/` | NetBox-Oberfläche für Katalog, Installation und Plugin-Lebenszyklus. Unterstützt NetBox 4.6.5 bis 4.6.8. | -| `host_agent/` | Kleine privilegierte Linux-Komponente, die freigegebene Wheel-Dateien prüft und die eigentlichen Änderungen am NetBox-Host ausführt. | +| `host_agent/` | Kleine privilegierte Linux-Komponente, die freigegebene Artefakte prüft, Source-Archive lokal als Wheel baut und die Änderungen am NetBox-Host ausführt. | Der Store verwendet standardmäßig eine lokale JSON-Datei. MariaDB ist optional. Die konfigurierte Standardquelle gilt als vom Betreiber freigegeben; weitere Quellen werden im Admin-Bereich zunächst ausstehend angelegt. Automatisch erkannte Plugins und Releases erscheinen erst nach einer ausdrücklichen Freigabe im öffentlichen Katalog. @@ -213,7 +213,7 @@ MariaDB erhält keinen Host-Port und ist nur im internen Compose-Netz erreichbar Der voreingestellte Forgejo-Import liest Repositories von `https://git.mrblake.cc`. GitHub ist bereits als zweiter Provider implementiert und kann im Admin-Bereich über dieselbe Quellenverwaltung hinzugefügt werden. -Mit einem Token lassen sich auch Metadaten und README privater GitHub-Repositories synchronisieren. Deren private Release-Assets bleiben in API v1 bewusst nicht installierbar, weil der Host-Agent keine Provider-Zugangsdaten erhält; für automatische Installationen muss das Wheel als öffentliches Release-Asset oder auf einem anderen freigegebenen HTTPS-Host veröffentlicht werden. +Mit einem Token lassen sich auch Metadaten und README privater GitHub-Repositories synchronisieren. Private Artefakte bleiben in API v1 bewusst nicht installierbar, weil der Host-Agent keine Provider-Zugangsdaten erhält. Öffentliche Projekte benötigen kein Release: Fehlt ein Wheel-Release, erzeugt der Store einen separat freizugebenden Source-Kandidaten für den exakten aktuellen Commit. Der normale Ablauf ist: @@ -294,7 +294,7 @@ PLUGINS_CONFIG["netbox_plugin_store"].update({ }) ``` -Der Agent installiert ausschließlich freigegebene, unveränderliche Wheel-Dateien. Abhängigkeiten müssen im aktuellen MVP bereits durch den Betreiber bereitgestellt sein. Vollständige Sicherheits- und Recovery-Hinweise stehen in [`host_agent/README.md`](host_agent/README.md). +Der Agent installiert ausschließlich freigegebene, unveränderliche Artefakte. Source-Archive werden commitgebunden geprüft, offline lokal in ein Wheel gebaut und als gehashte Datei unter `/opt/netbox/.netbox-store-wheels` persistiert. Abhängigkeiten und Build-Backends müssen bereits durch den Betreiber bereitgestellt sein. Vollständige Sicherheits- und Recovery-Hinweise stehen in [`host_agent/README.md`](host_agent/README.md). ## API v1 diff --git a/host_agent/README.md b/host_agent/README.md index add454e..d035082 100644 --- a/host_agent/README.md +++ b/host_agent/README.md @@ -6,7 +6,7 @@ decision is revalidated against the configured JSON API immediately before a lif The daemon is **dry-run by default**. It serializes operations, persists idempotency and state in SQLite, accepts one bounded JSON line per Unix-stream connection, verifies Linux peer credentials, -downloads only an approved immutable wheel, and invokes subprocesses only as fixed argument arrays +downloads only an approved immutable wheel or commit-bound source archive, and invokes subprocesses only as fixed argument arrays with `shell=False`. ## Install and one-time operator setup @@ -27,7 +27,7 @@ active; this switch only permits private/reserved DNS results. The agent owns only these two configured files: - `paths.include_path`, which contains only `STORE_PLUGINS = [...]`; -- `paths.requirements_path`, which contains the locked direct wheel references. +- `paths.requirements_path`, which contains locked wheel references. Locally built wheels are cached below `/opt/netbox/.netbox-store-wheels`. In the operator-owned NetBox `configuration.py`, add once, after the normal `PLUGINS` declaration: @@ -112,7 +112,7 @@ value as an opaque Store marker, and the agent compares it exactly with the fres Artifact `sha256`, in contrast, is always a lowercase 64-character SHA-256. The plugin and release must be approved, the release immutable and compatible with the configured -NetBox version. The artifact must be a valid wheel whose filename distribution and version match +NetBox version. A wheel's filename distribution and version must match the catalog, and its byte count, digest, host, scheme and DNS addresses are checked while streaming. Redirects, source distributions, private/reserved DNS targets (unless explicitly enabled for a test environment), and catalog additions outside the v1 schema fail closed. @@ -151,6 +151,7 @@ netbox-store-agent status 20f4274f-d4e5-42bf-9164-967b1a774481 - `install` and `update` re-fetch plugin and release, exactly match the approval token, download and validate the wheel, then use `pip --no-index --no-deps --only-binary=:all: --require-hashes`. +- Source archives must be tied to an exact commit. The agent rejects unsafe archive members and requires configured `unshare`/`setpriv` executables plus a dedicated build UID/GID. It builds without network access and with dropped privileges via `pip wheel --no-index --no-deps --no-build-isolation`, validates the result, and caches only that wheel for future NetBox upgrades. Reviewing source code remains a privileged trust decision because Python build backends execute code. - New installs remain disabled. Updating an enabled plugin runs NetBox `migrate`, `collectstatic`, and restarts every configured service (`netbox` and `netbox-rq` in the example). - `enable` revalidates its installed release, writes the include, migrates, collects static files, diff --git a/host_agent/examples/agent.toml b/host_agent/examples/agent.toml index 6512491..7861420 100644 --- a/host_agent/examples/agent.toml +++ b/host_agent/examples/agent.toml @@ -17,7 +17,7 @@ connection_timeout_seconds = 5 worker_threads = 1 [store] -base_url = "https://store.example.invalid" +base_url = "https://netbox.mrblake.cc" plugin_endpoint_template = "/api/v1/plugins/{plugin_slug}/" release_endpoint_template = "/api/v1/plugins/{plugin_slug}/releases/{version}/" timeout_seconds = 10 @@ -26,7 +26,7 @@ max_artifact_bytes = 268435456 allow_private_addresses = false allow_http_for_testing = false # Add an artifact origin only when releases intentionally use that origin. -allowed_hosts = ["store.example.invalid"] +allowed_hosts = ["netbox.mrblake.cc", "git.mrblake.cc", "github.com", "codeload.github.com"] # bearer_token_file = "/etc/netbox-store-agent/store.token" # ca_file = "/etc/ssl/certs/internal-store-ca.pem" @@ -42,6 +42,11 @@ manage_path = "/opt/netbox/netbox/manage.py" systemctl_path = "/usr/bin/systemctl" services = ["netbox", "netbox-rq"] command_timeout_seconds = 900 +unshare_path = "/usr/bin/unshare" +setpriv_path = "/usr/bin/setpriv" +# Dedicated unprivileged account used only for reviewed source builds. +source_build_uid = 65534 +source_build_gid = 65534 [policy] netbox_version = "4.6.8" diff --git a/host_agent/src/netbox_store_agent/catalog.py b/host_agent/src/netbox_store_agent/catalog.py index 588e866..a08eb2f 100644 --- a/host_agent/src/netbox_store_agent/catalog.py +++ b/host_agent/src/netbox_store_agent/catalog.py @@ -8,6 +8,7 @@ import os import socket import ssl import stat +import tarfile import zipfile from dataclasses import dataclass from pathlib import Path @@ -264,6 +265,8 @@ class ReleasePlan: artifact_sha256: str artifact_size: int approved_payload_sha256: str + artifact_kind: str = "wheel" + artifact_filename: str = "" def requirement(self) -> dict[str, Any]: return { @@ -275,6 +278,14 @@ class ReleasePlan: "size": self.artifact_size, } + def artifact_lock(self) -> dict[str, Any]: + return { + **self.requirement(), + "artifact_kind": self.artifact_kind, + "artifact_filename": self.artifact_filename or self.filename, + "approved_payload_sha256": self.approved_payload_sha256, + } + def _object(value: Any, context: str) -> dict[str, Any]: if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): @@ -321,6 +332,8 @@ class StoreClient: "download_url", "sha256", "artifact_size", + "artifact_kind", + "artifact_filename", "commit_sha", "min_netbox_version", "max_netbox_version", @@ -432,12 +445,26 @@ class StoreClient: raise CatalogError("release.commit_sha is invalid") if data["published_at"] is not None: _bounded_string(data["published_at"], "release.published_at", 100) - parsed_url = urlsplit(download_url) - filename = Path(parsed_url.path).name - if not filename or not filename.endswith(".whl") or len(filename) > 255: - raise CatalogError("approved artifact must be a wheel with a safe filename") + artifact_kind = _bounded_string(data["artifact_kind"], "release.artifact_kind", 32) + if artifact_kind not in {"wheel", "source_archive"}: + raise CatalogError("release.artifact_kind is unsupported") + filename = _bounded_string(data["artifact_filename"], "release.artifact_filename", 255) + if Path(filename).name != filename: + raise CatalogError("release.artifact_filename is unsafe") + if artifact_kind == "wheel" and not filename.endswith(".whl"): + raise CatalogError("wheel artifact filename is invalid") + if artifact_kind == "source_archive": + if ( + not filename.endswith(".tar.gz") + or len(commit) != 40 + or commit not in urlsplit(download_url).path + ): + raise CatalogError("source archive must be bound to an exact commit") # URL host/scheme/DNS are revalidated by the transport at download time. - return ReleasePlan(plugin, version_text, download_url, filename, sha256, size, token) + return ReleasePlan( + plugin, version_text, download_url, filename, sha256, size, token, + artifact_kind, filename, + ) def get_release(self, slug: str, version: str) -> ReleasePlan: plugin = self.get_plugin(slug) @@ -464,8 +491,15 @@ class StoreClient: expected_size=plan.artifact_size, expected_sha256=plan.artifact_sha256, ) + if plan.artifact_kind == "source_archive": + self._validate_source_archive(destination) + return destination + self.validate_wheel(plan, destination) + return destination + + def validate_wheel(self, plan: ReleasePlan, destination: Path) -> None: try: - distribution, wheel_version, _build, _tags = parse_wheel_filename(plan.filename) + distribution, wheel_version, _build, _tags = parse_wheel_filename(destination.name) except (InvalidVersion, ValueError) as exc: raise CatalogError("artifact filename is not a valid wheel filename") from exc if canonicalize_name(distribution) != canonicalize_name(plan.plugin.package_name): @@ -494,4 +528,36 @@ class StoreClient: raise CatalogError("wheel archive failed its integrity check") except (OSError, zipfile.BadZipFile) as exc: raise CatalogError("artifact is not a valid wheel archive") from exc - return destination + + def _validate_source_archive(self, archive_path: Path) -> None: + try: + with tarfile.open(archive_path, "r:gz") as archive: + members = archive.getmembers() + if not members or len(members) > 10_000: + raise CatalogError("source archive has an invalid member count") + expanded = 0 + roots: set[str] = set() + has_build_file = False + for member in members: + parts = Path(member.name).parts + if ( + not parts + or member.name.startswith(("/", "\\")) + or ".." in parts + or "\x00" in member.name + ): + raise CatalogError("source archive contains an unsafe member") + if member.issym() or member.islnk() or member.isdev() or member.isfifo(): + raise CatalogError("source archive contains a forbidden member type") + roots.add(parts[0]) + expanded += member.size + if expanded > min( + self.config.store.max_artifact_bytes * 20, 2 * 1024**3 + ): + raise CatalogError("source archive expands beyond the safety limit") + if len(parts) == 2 and parts[1] in {"pyproject.toml", "setup.py"}: + has_build_file = True + if len(roots) != 1 or not has_build_file: + raise CatalogError("source archive must have one root and a Python build file") + except (OSError, tarfile.TarError) as exc: + raise CatalogError("artifact is not a valid source archive") from exc diff --git a/host_agent/src/netbox_store_agent/config.py b/host_agent/src/netbox_store_agent/config.py index 3478b89..36d4691 100644 --- a/host_agent/src/netbox_store_agent/config.py +++ b/host_agent/src/netbox_store_agent/config.py @@ -71,6 +71,10 @@ class CommandSettings: systemctl_path: Path services: tuple[str, ...] command_timeout_seconds: int + unshare_path: Path | None = None + setpriv_path: Path | None = None + source_build_uid: int = 65534 + source_build_gid: int = 65534 @dataclass(frozen=True) @@ -240,7 +244,11 @@ def load_config(path: str | Path, *, allow_insecure_owner: bool = False) -> Conf commands_data = _table( root, "commands", - {"python_path", "manage_path", "systemctl_path", "services", "command_timeout_seconds"}, + { + "python_path", "manage_path", "systemctl_path", "services", + "command_timeout_seconds", "unshare_path", "setpriv_path", + "source_build_uid", "source_build_gid", + }, ) policy_data = _table( root, @@ -342,6 +350,10 @@ def load_config(path: str | Path, *, allow_insecure_owner: bool = False) -> Conf systemctl_path=require_absolute_path(_string(commands_data, "systemctl_path"), "systemctl_path"), services=services, command_timeout_seconds=_int(commands_data, "command_timeout_seconds", 900, 1, 7200), + unshare_path=_optional_path(commands_data, "unshare_path"), + setpriv_path=_optional_path(commands_data, "setpriv_path"), + source_build_uid=_int(commands_data, "source_build_uid", 65534, 1, 2**31 - 1), + source_build_gid=_int(commands_data, "source_build_gid", 65534, 1, 2**31 - 1), ) versions: dict[str, str] = {} diff --git a/host_agent/src/netbox_store_agent/executor.py b/host_agent/src/netbox_store_agent/executor.py index d21083a..e3bb54d 100644 --- a/host_agent/src/netbox_store_agent/executor.py +++ b/host_agent/src/netbox_store_agent/executor.py @@ -1,6 +1,10 @@ from __future__ import annotations +import hashlib +import os import secrets +import shutil +import tarfile import tempfile from pathlib import Path from typing import Callable @@ -56,7 +60,11 @@ class OperationProcessor: @staticmethod def _same_release(existing: ManagedPlugin, plan: ReleasePlan) -> None: - if existing.version != plan.version or existing.requirements != (plan.requirement(),): + expected = plan.artifact_lock() if existing.artifact_lock else plan.requirement() + current_lock = existing.artifact_lock or ( + existing.requirements[0] if existing.requirements else {} + ) + if existing.version != plan.version or current_lock != expected: raise PolicyError( "managed artifact lock differs from the current immutable Store release" ) @@ -105,6 +113,71 @@ class OperationProcessor: ], ) + def _build_source_wheel( + self, operation_id: str, directory: Path, plan: ReleasePlan, archive: Path + ) -> Path: + if ( + os.name != "posix" + or self.config.commands.unshare_path is None + or self.config.commands.setpriv_path is None + ): + raise PolicyError("source builds require configured Linux unshare and setpriv executables") + source_dir = directory / "source" + wheel_dir = directory / "wheel" + source_dir.mkdir(mode=0o700) + wheel_dir.mkdir(mode=0o700) + with tarfile.open(archive, "r:gz") as bundle: + bundle.extractall(source_dir, filter="data") + roots = list(source_dir.iterdir()) + if len(roots) != 1 or not roots[0].is_dir(): + raise PolicyError("source archive extraction produced an invalid root") + uid = self.config.commands.source_build_uid + gid = self.config.commands.source_build_gid + for path in [directory, source_dir, wheel_dir, *source_dir.rglob("*")]: + os.chown(path, uid, gid, follow_symlinks=False) + if path.is_dir(): + os.chmod(path, 0o700) + self._run( + operation_id, + "source_build", + [ + str(self.config.commands.unshare_path), "--net", "--", + str(self.config.commands.setpriv_path), f"--reuid={uid}", f"--regid={gid}", + "--clear-groups", "--no-new-privs", "--", + str(self.config.commands.python_path), "-m", "pip", "wheel", + "--no-input", "--disable-pip-version-check", "--no-index", "--no-deps", + "--no-build-isolation", "--wheel-dir", str(wheel_dir), str(roots[0]), + ], + ) + wheels = list(wheel_dir.glob("*.whl")) + if len(wheels) != 1: + raise PolicyError("source build must produce exactly one wheel") + self.store.validate_wheel(plan, wheels[0]) + return wheels[0] + + def _cache_wheel(self, plan: ReleasePlan, wheel: Path) -> tuple[Path, dict[str, object]]: + digest = hashlib.sha256(wheel.read_bytes()).hexdigest() + cache = ( + self.config.paths.allowed_root + / ".netbox-store-wheels" + / plan.plugin.slug + / plan.approved_payload_sha256 + ) + cache.mkdir(parents=True, exist_ok=True, mode=0o700) + destination = cache / wheel.name + if destination.exists() and destination.is_symlink(): + raise PolicyError("wheel cache destination may not be a symlink") + temporary = cache / (wheel.name + ".tmp") + shutil.copyfile(wheel, temporary) + os.chmod(temporary, 0o600) + os.replace(temporary, destination) + requirement = { + "package_name": plan.plugin.package_name, "version": plan.version, + "download_url": destination.as_uri(), "filename": destination.name, + "sha256": digest, "size": destination.stat().st_size, + } + return destination, requirement + def _pip_uninstall(self, operation_id: str, package_name: str) -> None: self._run( operation_id, @@ -184,23 +257,26 @@ class OperationProcessor: if existing is not None: self._same_identity(existing, plan.plugin) enabled = existing.enabled if existing else False - managed = ManagedPlugin( - slug=request.plugin_slug, - package_name=plan.plugin.package_name, - import_name=plan.plugin.import_name, - version=plan.version, - enabled=enabled, - requirements=(plan.requirement(),), - ) temp_root = self.config.paths.temp_dir if temp_root.is_symlink(): raise PolicyError("temporary directory may not be a symlink") temp_root.mkdir(parents=True, exist_ok=True, mode=0o700) with tempfile.TemporaryDirectory(prefix="operation-", dir=temp_root) as temporary: - self._event(operation_id, "artifact", "Downloading and verifying approved wheel") - wheel = self.store.download_release(plan, Path(temporary)) + self._event(operation_id, "artifact", "Downloading and verifying approved artifact") + artifact = self.store.download_release(plan, Path(temporary)) if self.config.agent.dry_run: return self._dry_result(request, version=plan.version, artifact_verified=True), False + wheel = ( + self._build_source_wheel(operation_id, Path(temporary), plan, artifact) + if plan.artifact_kind == "source_archive" + else artifact + ) + wheel, requirement = self._cache_wheel(plan, wheel) + managed = ManagedPlugin( + slug=request.plugin_slug, package_name=plan.plugin.package_name, + import_name=plan.plugin.import_name, version=plan.version, enabled=enabled, + requirements=(requirement,), artifact_lock=plan.artifact_lock(), + ) self._mark_mutated(operation_id) host_mutated = True self._pip_install(operation_id, Path(temporary), managed, wheel) @@ -239,6 +315,7 @@ class OperationProcessor: existing.version, True, existing.requirements, + existing.artifact_lock, ) if self.config.agent.dry_run: return self._dry_result(request, version=existing.version, enabled=True), False @@ -260,6 +337,7 @@ class OperationProcessor: existing.version, False, existing.requirements, + existing.artifact_lock, ) if self.config.agent.dry_run: return self._dry_result(request, version=existing.version, enabled=False), False diff --git a/host_agent/src/netbox_store_agent/journal.py b/host_agent/src/netbox_store_agent/journal.py index f80b8fb..4a82731 100644 --- a/host_agent/src/netbox_store_agent/journal.py +++ b/host_agent/src/netbox_store_agent/journal.py @@ -31,6 +31,7 @@ class ManagedPlugin: version: str enabled: bool requirements: tuple[dict[str, Any], ...] + artifact_lock: dict[str, Any] | None = None class Journal: @@ -111,10 +112,17 @@ class Journal: version TEXT NOT NULL, enabled INTEGER NOT NULL CHECK(enabled IN (0, 1)), requirements_json TEXT NOT NULL, + artifact_lock_json TEXT NOT NULL DEFAULT '{}', updated_at TEXT NOT NULL ); """ ) + columns = {row[1] for row in connection.execute("PRAGMA table_info(managed_plugins)")} + if "artifact_lock_json" not in columns: + connection.execute( + "ALTER TABLE managed_plugins ADD COLUMN artifact_lock_json " + "TEXT NOT NULL DEFAULT '{}'" + ) @staticmethod def _operation_dict(row: sqlite3.Row, events: list[dict[str, Any]] | None = None) -> dict[str, Any]: @@ -306,6 +314,7 @@ class Journal: version=row["version"], enabled=bool(row["enabled"]), requirements=tuple(requirements), + artifact_lock=json.loads(row["artifact_lock_json"]), ) def get_managed_plugin(self, slug: str) -> ManagedPlugin | None: @@ -322,18 +331,20 @@ class Journal: def upsert_managed_plugin(self, plugin: ManagedPlugin) -> None: requirements_json = canonical_json(list(plugin.requirements)) + artifact_lock_json = canonical_json(plugin.artifact_lock) with self._connect() as connection: connection.execute( """ INSERT INTO managed_plugins( - slug, package_name, import_name, version, enabled, requirements_json, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?) + slug, package_name, import_name, version, enabled, requirements_json, artifact_lock_json, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(slug) DO UPDATE SET package_name=excluded.package_name, import_name=excluded.import_name, version=excluded.version, enabled=excluded.enabled, requirements_json=excluded.requirements_json, + artifact_lock_json=excluded.artifact_lock_json, updated_at=excluded.updated_at """, ( @@ -343,6 +354,7 @@ class Journal: plugin.version, int(plugin.enabled), requirements_json, + artifact_lock_json, utc_now(), ), ) diff --git a/host_agent/src/netbox_store_agent/managed_files.py b/host_agent/src/netbox_store_agent/managed_files.py index efe3b7c..d657612 100644 --- a/host_agent/src/netbox_store_agent/managed_files.py +++ b/host_agent/src/netbox_store_agent/managed_files.py @@ -144,6 +144,7 @@ class ManagedFiles: def render_requirements(self, plugins: Iterable[ManagedPlugin]) -> bytes: requirements: dict[str, dict[str, object]] = {} schemes = {"https", "http"} if self.config.store.allow_http_for_testing else {"https"} + schemes.add("file") for plugin in plugins: for raw in plugin.requirements: if not isinstance(raw, dict): @@ -166,10 +167,17 @@ class ManagedFiles: if not isinstance(url, str) or any(char in url for char in "\r\n\t "): raise ValidationError("managed requirement URL is invalid") parsed = urlsplit(url) + remote_invalid = parsed.scheme != "file" and ( + not parsed.hostname or parsed.hostname.lower() not in self.config.store.allowed_hosts + ) + local_path = Path(parsed.path.lstrip("/") if os.name == "nt" else parsed.path) + local_invalid = parsed.scheme == "file" and not is_relative_to( + local_path.resolve(strict=False), self.config.paths.allowed_root + ) if ( parsed.scheme not in schemes - or not parsed.hostname - or parsed.hostname.lower() not in self.config.store.allowed_hosts + or remote_invalid + or local_invalid or parsed.username or parsed.password or parsed.fragment diff --git a/host_agent/tests/support.py b/host_agent/tests/support.py index 80723b3..cfa21a7 100644 --- a/host_agent/tests/support.py +++ b/host_agent/tests/support.py @@ -88,6 +88,8 @@ def release_json(version: str = "1.2.3", **overrides: Any) -> dict[str, Any]: "download_url": f"http://artifacts.test/demo_plugin-{version}-py3-none-any.whl", "sha256": "a" * 64, "artifact_size": 100, + "artifact_kind": "wheel", + "artifact_filename": f"demo_plugin-{version}-py3-none-any.whl", "commit_sha": "", "min_netbox_version": "4.6.5", "max_netbox_version": "4.6.8", diff --git a/host_agent/tests/test_catalog.py b/host_agent/tests/test_catalog.py index 2ef603a..134f251 100644 --- a/host_agent/tests/test_catalog.py +++ b/host_agent/tests/test_catalog.py @@ -1,6 +1,8 @@ from __future__ import annotations import hashlib +import io +import tarfile import tempfile import unittest from pathlib import Path @@ -48,6 +50,36 @@ class CatalogTests(unittest.TestCase): plan = StoreClient(self.config, transport).get_release("demo-plugin", "1.2.3") self.assertEqual(plan.version, "1.2.3") + def test_commit_bound_source_archive_is_verified(self) -> None: + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz") as archive: + content = b"[build-system]\nrequires = []\n" + info = tarfile.TarInfo("demo/pyproject.toml") + info.size = len(content) + archive.addfile(info, io.BytesIO(content)) + artifact = buffer.getvalue() + commit = "d" * 40 + release = release_json( + download_url=f"http://artifacts.test/demo/archive/{commit}.tar.gz", + artifact_kind="source_archive", + artifact_filename="demo-plugin-1.2.3-source.tar.gz", + commit_sha=commit, + sha256=hashlib.sha256(artifact).hexdigest(), + artifact_size=len(artifact), + ) + transport = FakeTransport( + { + "http://store.test/api/v1/plugins/demo-plugin": plugin_json(), + "http://store.test/api/v1/plugins/demo-plugin/releases/1.2.3": release, + }, + artifact, + ) + client = StoreClient(self.config, transport) + plan = client.get_release("demo-plugin", "1.2.3") + directory = self.root / "source-operation" + directory.mkdir() + self.assertEqual(client.download_release(plan, directory).read_bytes(), artifact) + def test_unapproved_or_mutable_release_rejected(self) -> None: for change in ({"approved": False}, {"immutable": False}, {"status": "pending"}): with self.subTest(change=change): diff --git a/netbox_plugin/netbox_plugin_store/client.py b/netbox_plugin/netbox_plugin_store/client.py index 2ccf497..f06d9ed 100644 --- a/netbox_plugin/netbox_plugin_store/client.py +++ b/netbox_plugin/netbox_plugin_store/client.py @@ -77,6 +77,8 @@ class Release: approved: bool immutable: bool approved_payload_sha256: str + artifact_kind: str = "wheel" + artifact_filename: str = "" @classmethod def from_mapping(cls, value: dict[str, Any]) -> "Release": @@ -104,6 +106,8 @@ class Release: approved=value.get("approved") is True or value.get("status") == "approved", immutable=value.get("immutable") is True, approved_payload_sha256=approved_payload_sha256, + artifact_kind=str(value.get("artifact_kind") or "wheel").strip(), + artifact_filename=str(value.get("artifact_filename") or "").strip(), ) def supports(self, netbox_version: str, plugin: "CatalogPlugin") -> bool: diff --git a/netbox_plugin/netbox_plugin_store/lifecycle.py b/netbox_plugin/netbox_plugin_store/lifecycle.py index 9cb69dd..69cf7cc 100644 --- a/netbox_plugin/netbox_plugin_store/lifecycle.py +++ b/netbox_plugin/netbox_plugin_store/lifecycle.py @@ -235,6 +235,8 @@ class LifecycleService: raise LifecycleError("Approved release has no artifact URL.") if not release.sha256: raise LifecycleError("Approved release has no artifact SHA-256. Commit hashes are not accepted.") + if release.artifact_kind not in {"wheel", "source_archive"}: + raise LifecycleError("Approved release has an unsupported artifact type.") @staticmethod def _assert_state(action: str, installed: str, enabled: bool, runtime_active: bool) -> None: @@ -287,6 +289,8 @@ class LifecycleService: return suffix if re.fullmatch(r"\.[A-Za-z0-9.]{1,16}", suffix) else ".artifact" def _download(self, release: Release, directory: Path) -> Path: + if release.artifact_kind == "source_archive": + raise LifecycleError("Source archives require execution_mode='agent' for a verified local wheel build.") artifact = directory / f"artifact{self._artifact_suffix(release)}" self.client.download_artifact( release.download_url, diff --git a/store/public/assets/app.css b/store/public/assets/app.css index 32812f3..9b173b2 100644 --- a/store/public/assets/app.css +++ b/store/public/assets/app.css @@ -232,6 +232,18 @@ input:focus, select:focus, textarea:focus { border-color: var(--blue); box-shado .sync-errors ul { margin: 7px 0 0; padding-left: 18px; } .sync-errors li { display: list-item; padding: 3px 0; border: 0; overflow-wrap: anywhere; } +.install-hero { padding: 64px 0 48px; color: #fff; background: #102a43; } +.install-hero h1 { margin: 6px 0 12px; font-size: clamp(2rem, 5vw, 3.5rem); letter-spacing: -.04em; } +.install-hero p:last-child { color: #c6d8e5; } +.prose-shell { max-width: 880px; } +.install-guide { padding-block: 42px 80px; } +.install-guide h2 { margin: 38px 0 10px; } +.install-guide p { color: var(--ink-soft); line-height: 1.7; } +.install-guide pre { padding: 19px; overflow-x: auto; border: 1px solid #214761; border-radius: 11px; color: #eaf4fb; background: #102a43; line-height: 1.55; } +.install-guide code { font-family: ui-monospace, SFMono-Regular, Consolas, monospace; } +.notice { padding: 16px 18px; border-left: 4px solid var(--blue); border-radius: 8px; background: #edf5ff; line-height: 1.55; } +.notice.warning { margin-top: 28px; border-left-color: var(--amber); background: var(--amber-bg); } + @media (max-width: 980px) { .hero-grid { grid-template-columns: 1fr; gap: 35px; } .hero-stat { width: min(350px, 100%); } diff --git a/store/src/Domain/Approval.php b/store/src/Domain/Approval.php index 086733e..a88e4e2 100644 --- a/store/src/Domain/Approval.php +++ b/store/src/Domain/Approval.php @@ -17,6 +17,7 @@ final class Approval 'commitSha' => strtolower((string) ($release['commitSha'] ?? '')), 'downloadUrl' => (string) ($release['downloadUrl'] ?? ''), 'artifactKind' => (string) ($release['artifactKind'] ?? ''), + 'artifactFilename' => (string) ($release['artifactFilename'] ?? ''), 'importName' => (string) ($plugin['importName'] ?? ''), 'maxNetboxVersion' => (string) (($release['maxNetboxVersion'] ?? '') ?: ($plugin['maxNetboxVersion'] ?? '')), 'minNetboxVersion' => (string) (($release['minNetboxVersion'] ?? '') ?: ($plugin['minNetboxVersion'] ?? '')), @@ -95,7 +96,10 @@ final class Approval if (!is_array($url) || ($url['scheme'] ?? '') !== 'https' || empty($url['host']) || isset($url['user']) || isset($url['pass'])) { $errors[] = 'Download-URL muss credential-freies HTTPS sein.'; } else { - $errors = array_merge($errors, self::wheelErrors($plugin, $release, (string) ($url['path'] ?? ''))); + $kind = (string) ($release['artifactKind'] ?? ''); + $errors = array_merge($errors, $kind === 'source_archive' + ? self::sourceErrors($release, (string) ($url['path'] ?? '')) + : self::wheelErrors($plugin, $release, (string) ($url['path'] ?? ''))); } $commit = (string) ($release['commitSha'] ?? ''); if ($commit !== '' && preg_match('/^[a-f0-9]{40}$/', $commit) !== 1) { @@ -109,8 +113,8 @@ final class Approval if (!self::immutable($release)) { $errors[] = 'Artefakt-SHA oder Größe fehlt.'; } - if (($release['artifactKind'] ?? '') !== 'wheel') { - $errors[] = 'Im API-v1-Katalog sind ausschließlich Wheel-Artefakte freigabefähig.'; + if (!in_array(($release['artifactKind'] ?? ''), ['wheel', 'source_archive'], true)) { + $errors[] = 'Nur Wheel- oder commitgebundene Source-Artefakte sind freigabefähig.'; } if (!empty($release['draft']) || !empty($release['withdrawn'])) { $errors[] = 'Drafts oder zurückgezogene Releases sind nicht freigabefähig.'; @@ -259,4 +263,22 @@ final class Approval } return $errors; } + + /** @param array $release @return list */ + private static function sourceErrors(array $release, string $path): array + { + $commit = (string) ($release['commitSha'] ?? ''); + $filename = (string) ($release['artifactFilename'] ?? ''); + $errors = []; + if (preg_match('/^[a-f0-9]{40}$/', $commit) !== 1) { + $errors[] = 'Source-Artefakte benoetigen einen exakten 40-stelligen Commit-SHA.'; + } + if ($filename === '' || strlen($filename) > 255 || preg_match('/^[A-Za-z0-9._+-]+\.tar\.gz$/', $filename) !== 1) { + $errors[] = 'Source-Artefaktname ist ungueltig.'; + } + if ($commit !== '' && !str_contains(rawurldecode($path), $commit)) { + $errors[] = 'Source-Download-URL ist nicht an den freigegebenen Commit gebunden.'; + } + return $errors; + } } diff --git a/store/src/Domain/Catalog.php b/store/src/Domain/Catalog.php index cd70099..e3f209d 100644 --- a/store/src/Domain/Catalog.php +++ b/store/src/Domain/Catalog.php @@ -94,6 +94,8 @@ final class Catalog 'download_url' => (string) $release['downloadUrl'], 'sha256' => strtolower((string) $release['sha256']), 'artifact_size' => (int) $release['artifactSize'], + 'artifact_kind' => (string) $release['artifactKind'], + 'artifact_filename' => (string) $release['artifactFilename'], 'commit_sha' => (string) ($release['commitSha'] ?? ''), 'min_netbox_version' => (string) (($release['minNetboxVersion'] ?? '') ?: ($plugin['minNetboxVersion'] ?? '')), 'max_netbox_version' => (string) (($release['maxNetboxVersion'] ?? '') ?: ($plugin['maxNetboxVersion'] ?? '')), diff --git a/store/src/Http/Application.php b/store/src/Http/Application.php index 0776c48..23c93c4 100644 --- a/store/src/Http/Application.php +++ b/store/src/Http/Application.php @@ -60,6 +60,9 @@ final class Application if ($request->method === 'GET' && $request->path === '/') { return $this->home($request); } + if ($request->method === 'GET' && $request->path === '/installation') { + return $this->viewResponse('installation', ['title' => 'NetBox Store installieren'], 200, $request); + } if ($request->method === 'GET' && preg_match('#^/plugins/([^/]+)$#', $request->path, $match)) { return $this->plugin($request, $match[1]); } diff --git a/store/src/Sync/Adapter/ForgejoAdapter.php b/store/src/Sync/Adapter/ForgejoAdapter.php index aac9ebd..962acfd 100644 --- a/store/src/Sync/Adapter/ForgejoAdapter.php +++ b/store/src/Sync/Adapter/ForgejoAdapter.php @@ -49,6 +49,7 @@ final class ForgejoAdapter extends AbstractAdapter 'archived' => (bool) ($item['archived'] ?? false), 'fork' => (bool) ($item['fork'] ?? false), 'empty' => (bool) ($item['empty'] ?? false), + 'private' => (bool) ($item['private'] ?? false), ]; $this->validateRepository($repository); return $repository; @@ -135,6 +136,16 @@ final class ForgejoAdapter extends AbstractAdapter return $releases; } + public function sourceArchiveUrl(array $repository, string $commitSha): ?string + { + if (!empty($repository['private'])) { + return null; + } + $this->validateRepository($repository); + $this->assertCommit($commitSha); + return rtrim((string) $repository['htmlUrl'], '/') . '/archive/' . $commitSha . '.tar.gz'; + } + /** @return list>|null */ private function pagedRepositories(string $endpoint): ?array { diff --git a/store/src/Sync/Adapter/GitHubAdapter.php b/store/src/Sync/Adapter/GitHubAdapter.php index 4e9a4dc..024dc9a 100644 --- a/store/src/Sync/Adapter/GitHubAdapter.php +++ b/store/src/Sync/Adapter/GitHubAdapter.php @@ -138,6 +138,16 @@ final class GitHubAdapter extends AbstractAdapter return $releases; } + public function sourceArchiveUrl(array $repository, string $commitSha): ?string + { + if (!empty($repository['private'])) { + return null; + } + $this->validateRepository($repository); + $this->assertCommit($commitSha); + return 'https://codeload.github.com/' . $repository['fullName'] . '/tar.gz/' . $commitSha; + } + /** @return list>|null */ private function paged(string $endpoint): ?array { diff --git a/store/src/Sync/Adapter/SourceAdapter.php b/store/src/Sync/Adapter/SourceAdapter.php index cc1c76f..20df959 100644 --- a/store/src/Sync/Adapter/SourceAdapter.php +++ b/store/src/Sync/Adapter/SourceAdapter.php @@ -24,6 +24,9 @@ interface SourceAdapter /** @param array $repository @return list> */ public function listReleases(array $repository): array; + /** @param array $repository */ + public function sourceArchiveUrl(array $repository, string $commitSha): ?string; + /** @return array{sha256:string,artifactSize:int} */ public function hashArtifact(string $url, string $expectedSha256 = ''): array; } diff --git a/store/src/Sync/SyncService.php b/store/src/Sync/SyncService.php index fa9ecb2..50d1d64 100644 --- a/store/src/Sync/SyncService.php +++ b/store/src/Sync/SyncService.php @@ -19,7 +19,7 @@ use Closure; final class SyncService { private const PLUGIN_SECURITY = ['packageName', 'importName', 'minNetboxVersion', 'maxNetboxVersion']; - private const RELEASE_SECURITY = ['version', 'downloadUrl', 'sha256', 'artifactSize', 'artifactKind', 'commitSha', 'minNetboxVersion', 'maxNetboxVersion', 'withdrawn']; + private const RELEASE_SECURITY = ['version', 'downloadUrl', 'sha256', 'artifactSize', 'artifactKind', 'artifactFilename', 'commitSha', 'minNetboxVersion', 'maxNetboxVersion', 'withdrawn']; public function __construct( private readonly StoreRepository $repository, @@ -199,7 +199,7 @@ final class SyncService // GitHub's browser_download_url is deliberately public-only. // Provider credentials stay on the API origin and the v1 // Host-Agent has no credentials for private asset downloads. - if (($source['provider'] ?? '') === 'github' && !empty($repository['private'])) { + if (!empty($repository['private'])) { $this->withdrawMissingReleases((string) $plugin['id'], []); $run['errors'][] = [ 'repository' => $repository['fullName'], @@ -209,6 +209,29 @@ final class SyncService } $releaseInfos = $adapter->listReleases($result['repository']); + if ($releaseInfos === []) { + $sourceUrl = $adapter->sourceArchiveUrl($result['repository'], (string) $result['repository']['commitSha']); + $version = Support::safeVersion($plugin['latestVersion'] ?? ''); + $commit = (string) $result['repository']['commitSha']; + if ($sourceUrl !== null && $version !== '' && preg_match('/^[a-f0-9]{40}$/', $commit)) { + $package = strtolower((string) preg_replace('/[-_.]+/', '-', (string) $plugin['packageName'])); + $releaseInfos[] = [ + 'externalId' => 'source:' . $commit, + 'version' => $version, + 'title' => 'Source ' . substr($commit, 0, 12), + 'releaseUrl' => rtrim((string) $repository['htmlUrl'], '/') . '/commit/' . $commit, + 'downloadUrl' => $sourceUrl, + 'expectedSha256' => '', + 'commitSha' => $commit, + 'artifactKind' => 'source_archive', + 'artifactFilename' => $package . '-' . $version . '-source.tar.gz', + 'prerelease' => false, + 'draft' => false, + 'changelog' => 'Automatisch aus dem freigegebenen Repository-Commit erzeugter Source-Build.', + 'publishedAt' => null, + ]; + } + } $releaseInfosSeen += count($releaseInfos); $this->http->ensureReleasesCounted($releaseInfosSeen); foreach (array_filter($releaseInfos) as $releaseInfo) { @@ -418,7 +441,8 @@ final class SyncService 'releaseUrl' => $this->publicUrl($releaseInfo['releaseUrl'] ?? ''), 'downloadUrl' => $downloadUrl, 'sha256' => $artifact['sha256'], 'artifactSize' => (int) $artifact['artifactSize'], 'commitSha' => preg_match('/^[a-f0-9]{40}$/', $releaseInfo['commitSha'] ?? '') ? $releaseInfo['commitSha'] : '', - 'artifactKind' => str_ends_with(strtolower(parse_url($downloadUrl, PHP_URL_PATH) ?: ''), '.whl') ? 'wheel' : 'invalid', + 'artifactKind' => (string) ($releaseInfo['artifactKind'] ?? (str_ends_with(strtolower(parse_url($downloadUrl, PHP_URL_PATH) ?: ''), '.whl') ? 'wheel' : 'invalid')), + 'artifactFilename' => (string) ($releaseInfo['artifactFilename'] ?? basename((string) (parse_url($downloadUrl, PHP_URL_PATH) ?: ''))), 'prerelease' => (bool) ($releaseInfo['prerelease'] ?? false), 'draft' => (bool) ($releaseInfo['draft'] ?? false), 'withdrawn' => false, 'changelog' => (string) ($releaseInfo['changelog'] ?? ''), 'publishedAt' => $releaseInfo['publishedAt'] ?? null, diff --git a/store/templates/admin/dashboard.php b/store/templates/admin/dashboard.php index a3597b4..b116480 100644 --- a/store/templates/admin/dashboard.php +++ b/store/templates/admin/dashboard.php @@ -68,7 +68,7 @@ include dirname(__DIR__) . '/partials/head.php'; - · Öffentliche Wheel-Releases + · Wheels oder commitgebundener Source-Fallback
@@ -126,7 +126,7 @@ include dirname(__DIR__) . '/partials/head.php'; - + diff --git a/store/templates/home.php b/store/templates/home.php index 72fbcaf..d34dd1b 100644 --- a/store/templates/home.php +++ b/store/templates/home.php @@ -59,12 +59,12 @@
- +
- Wheel geprüft + Kein installierbares Release diff --git a/store/templates/installation.php b/store/templates/installation.php new file mode 100644 index 0000000..3bc65c3 --- /dev/null +++ b/store/templates/installation.php @@ -0,0 +1,42 @@ + +
+
+

NetBox 4.6.5–4.6.8

+

NetBox Store Plugin installieren

+

Das Plugin verbindet deine NetBox mit https://netbox.mrblake.cc. Docker ist nicht erforderlich.

+
+
+
+ +

1. Plugin aus git.mrblake.cc installieren

+

Trage das Plugin dauerhaft in /opt/netbox/local_requirements.txt ein. Für reproduzierbare Installationen solltest du main durch einen geprüften Commit-SHA ersetzen.

+ 
sudo sh -c 'printf "%s\n" "netbox-plugin-store @ git+https://git.mrblake.cc/MrBlake/Netbox-Store.git@main#subdirectory=netbox_plugin" >> /opt/netbox/local_requirements.txt'
+sudo /opt/netbox/upgrade.sh
+

2. Plugin in NetBox aktivieren

+

Ergänze die NetBox-Konfiguration:

+ 
PLUGINS = [
+    "netbox_plugin_store",
+]
+
+PLUGINS_CONFIG = {
+    "netbox_plugin_store": {
+        "store_url": "https://netbox.mrblake.cc",
+        "allowed_store_urls": ["https://netbox.mrblake.cc"],
+        "allowed_artifact_urls": [
+            "https://git.mrblake.cc",
+            "https://github.com",
+            "https://codeload.github.com",
+        ],
+        "execution_mode": "dry_run",
+    }
+}
+

Führe danach Migrationen und statische Dateien aus und starte NetBox neu:

+ 
sudo /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py migrate
+sudo /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py collectstatic --no-input
+sudo systemctl restart netbox netbox-rq
+

3. Sicheren Host-Agent einrichten

+

dry_run verändert das System nicht. Für Installieren, Aktualisieren, Aktivieren und Entfernen wird der mitgelieferte Linux Host-Agent benötigt. Installiere ihn aus host_agent/, prüfe /etc/netbox-store-agent/agent.toml und stelle anschließend execution_mode auf agent.

+

Source-Kandidaten werden nur nach Admin-Freigabe verarbeitet. Der Agent prüft Commitbindung, Größe und SHA-256, baut daraus lokal ein Wheel und installiert nicht direkt aus einem beweglichen Branch.

+ +
+ diff --git a/store/templates/partials/footer.php b/store/templates/partials/footer.php index d76b238..43195cc 100644 --- a/store/templates/partials/footer.php +++ b/store/templates/partials/footer.php @@ -7,6 +7,7 @@

Freigegebene Metadaten, reproduzierbar geprüfte Artefakte.

+ Installation Catalog API v1 Systemstatus
diff --git a/store/templates/partials/head.php b/store/templates/partials/head.php index 0c7aa31..0e8ac7c 100644 --- a/store/templates/partials/head.php +++ b/store/templates/partials/head.php @@ -21,6 +21,7 @@
Plugin / VersionArtefaktIntegritätStatusAktionen
Keine Release-Artefakte gefunden. Veröffentliche ein Wheel als Forgejo-Release-Asset.
Keine installierbaren Artefakte gefunden. Prüfe Version und Build-Metadaten des Repositorys.