feat: install plugins from approved source commits

host_agent/README.md

@@ -6,7 +6,7 @@ decision is revalidated against the configured JSON API immediately before a lif
 
 The daemon is **dry-run by default**. It serializes operations, persists idempotency and state in
 SQLite, accepts one bounded JSON line per Unix-stream connection, verifies Linux peer credentials,
-downloads only an approved immutable wheel, and invokes subprocesses only as fixed argument arrays
+downloads only an approved immutable wheel or commit-bound source archive, and invokes subprocesses only as fixed argument arrays
 with `shell=False`.
 
 ## Install and one-time operator setup
@@ -27,7 +27,7 @@ active; this switch only permits private/reserved DNS results.
 The agent owns only these two configured files:
 
 - `paths.include_path`, which contains only `STORE_PLUGINS = [...]`;
-- `paths.requirements_path`, which contains the locked direct wheel references.
+- `paths.requirements_path`, which contains locked wheel references. Locally built wheels are cached below `/opt/netbox/.netbox-store-wheels`.
 
 In the operator-owned NetBox `configuration.py`, add once, after the normal `PLUGINS` declaration:
 
@@ -112,7 +112,7 @@ value as an opaque Store marker, and the agent compares it exactly with the fres
 Artifact `sha256`, in contrast, is always a lowercase 64-character SHA-256.
 
 The plugin and release must be approved, the release immutable and compatible with the configured
-NetBox version. The artifact must be a valid wheel whose filename distribution and version match
+NetBox version. A wheel's filename distribution and version must match
 the catalog, and its byte count, digest, host, scheme and DNS addresses are checked while streaming.
 Redirects, source distributions, private/reserved DNS targets (unless explicitly enabled for a test
 environment), and catalog additions outside the v1 schema fail closed.
@@ -151,6 +151,7 @@ netbox-store-agent status 20f4274f-d4e5-42bf-9164-967b1a774481
 
 - `install` and `update` re-fetch plugin and release, exactly match the approval token, download and
   validate the wheel, then use `pip --no-index --no-deps --only-binary=:all: --require-hashes`.
+- Source archives must be tied to an exact commit. The agent rejects unsafe archive members and requires configured `unshare`/`setpriv` executables plus a dedicated build UID/GID. It builds without network access and with dropped privileges via `pip wheel --no-index --no-deps --no-build-isolation`, validates the result, and caches only that wheel for future NetBox upgrades. Reviewing source code remains a privileged trust decision because Python build backends execute code.
 - New installs remain disabled. Updating an enabled plugin runs NetBox `migrate`, `collectstatic`,
   and restarts every configured service (`netbox` and `netbox-rq` in the example).
 - `enable` revalidates its installed release, writes the include, migrates, collects static files,

host_agent/examples/agent.toml

@@ -17,7 +17,7 @@ connection_timeout_seconds = 5
 worker_threads = 1
 
 [store]
-base_url = "https://store.example.invalid"
+base_url = "https://netbox.mrblake.cc"
 plugin_endpoint_template = "/api/v1/plugins/{plugin_slug}/"
 release_endpoint_template = "/api/v1/plugins/{plugin_slug}/releases/{version}/"
 timeout_seconds = 10
@@ -26,7 +26,7 @@ max_artifact_bytes = 268435456
 allow_private_addresses = false
 allow_http_for_testing = false
 # Add an artifact origin only when releases intentionally use that origin.
-allowed_hosts = ["store.example.invalid"]
+allowed_hosts = ["netbox.mrblake.cc", "git.mrblake.cc", "github.com", "codeload.github.com"]
 # bearer_token_file = "/etc/netbox-store-agent/store.token"
 # ca_file = "/etc/ssl/certs/internal-store-ca.pem"
 
@@ -42,6 +42,11 @@ manage_path = "/opt/netbox/netbox/manage.py"
 systemctl_path = "/usr/bin/systemctl"
 services = ["netbox", "netbox-rq"]
 command_timeout_seconds = 900
+unshare_path = "/usr/bin/unshare"
+setpriv_path = "/usr/bin/setpriv"
+# Dedicated unprivileged account used only for reviewed source builds.
+source_build_uid = 65534
+source_build_gid = 65534
 
 [policy]
 netbox_version = "4.6.8"

host_agent/src/netbox_store_agent/catalog.py

@@ -8,6 +8,7 @@ import os
 import socket
 import ssl
 import stat
+import tarfile
 import zipfile
 from dataclasses import dataclass
 from pathlib import Path
@@ -264,6 +265,8 @@ class ReleasePlan:
     artifact_sha256: str
     artifact_size: int
     approved_payload_sha256: str
+    artifact_kind: str = "wheel"
+    artifact_filename: str = ""
 
     def requirement(self) -> dict[str, Any]:
         return {
@@ -275,6 +278,14 @@ class ReleasePlan:
             "size": self.artifact_size,
         }
 
+    def artifact_lock(self) -> dict[str, Any]:
+        return {
+            **self.requirement(),
+            "artifact_kind": self.artifact_kind,
+            "artifact_filename": self.artifact_filename or self.filename,
+            "approved_payload_sha256": self.approved_payload_sha256,
+        }
+
 
 def _object(value: Any, context: str) -> dict[str, Any]:
     if not isinstance(value, dict) or not all(isinstance(key, str) for key in value):
@@ -321,6 +332,8 @@ class StoreClient:
         "download_url",
         "sha256",
         "artifact_size",
+        "artifact_kind",
+        "artifact_filename",
         "commit_sha",
         "min_netbox_version",
         "max_netbox_version",
@@ -432,12 +445,26 @@ class StoreClient:
             raise CatalogError("release.commit_sha is invalid")
         if data["published_at"] is not None:
             _bounded_string(data["published_at"], "release.published_at", 100)
-        parsed_url = urlsplit(download_url)
-        filename = Path(parsed_url.path).name
-        if not filename or not filename.endswith(".whl") or len(filename) > 255:
-            raise CatalogError("approved artifact must be a wheel with a safe filename")
+        artifact_kind = _bounded_string(data["artifact_kind"], "release.artifact_kind", 32)
+        if artifact_kind not in {"wheel", "source_archive"}:
+            raise CatalogError("release.artifact_kind is unsupported")
+        filename = _bounded_string(data["artifact_filename"], "release.artifact_filename", 255)
+        if Path(filename).name != filename:
+            raise CatalogError("release.artifact_filename is unsafe")
+        if artifact_kind == "wheel" and not filename.endswith(".whl"):
+            raise CatalogError("wheel artifact filename is invalid")
+        if artifact_kind == "source_archive":
+            if (
+                not filename.endswith(".tar.gz")
+                or len(commit) != 40
+                or commit not in urlsplit(download_url).path
+            ):
+                raise CatalogError("source archive must be bound to an exact commit")
         # URL host/scheme/DNS are revalidated by the transport at download time.
-        return ReleasePlan(plugin, version_text, download_url, filename, sha256, size, token)
+        return ReleasePlan(
+            plugin, version_text, download_url, filename, sha256, size, token,
+            artifact_kind, filename,
+        )
 
     def get_release(self, slug: str, version: str) -> ReleasePlan:
         plugin = self.get_plugin(slug)
@@ -464,8 +491,15 @@ class StoreClient:
             expected_size=plan.artifact_size,
             expected_sha256=plan.artifact_sha256,
         )
+        if plan.artifact_kind == "source_archive":
+            self._validate_source_archive(destination)
+            return destination
+        self.validate_wheel(plan, destination)
+        return destination
+
+    def validate_wheel(self, plan: ReleasePlan, destination: Path) -> None:
         try:
-            distribution, wheel_version, _build, _tags = parse_wheel_filename(plan.filename)
+            distribution, wheel_version, _build, _tags = parse_wheel_filename(destination.name)
         except (InvalidVersion, ValueError) as exc:
             raise CatalogError("artifact filename is not a valid wheel filename") from exc
         if canonicalize_name(distribution) != canonicalize_name(plan.plugin.package_name):
@@ -494,4 +528,36 @@ class StoreClient:
                     raise CatalogError("wheel archive failed its integrity check")
         except (OSError, zipfile.BadZipFile) as exc:
             raise CatalogError("artifact is not a valid wheel archive") from exc
-        return destination
+
+    def _validate_source_archive(self, archive_path: Path) -> None:
+        try:
+            with tarfile.open(archive_path, "r:gz") as archive:
+                members = archive.getmembers()
+                if not members or len(members) > 10_000:
+                    raise CatalogError("source archive has an invalid member count")
+                expanded = 0
+                roots: set[str] = set()
+                has_build_file = False
+                for member in members:
+                    parts = Path(member.name).parts
+                    if (
+                        not parts
+                        or member.name.startswith(("/", "\\"))
+                        or ".." in parts
+                        or "\x00" in member.name
+                    ):
+                        raise CatalogError("source archive contains an unsafe member")
+                    if member.issym() or member.islnk() or member.isdev() or member.isfifo():
+                        raise CatalogError("source archive contains a forbidden member type")
+                    roots.add(parts[0])
+                    expanded += member.size
+                    if expanded > min(
+                        self.config.store.max_artifact_bytes * 20, 2 * 1024**3
+                    ):
+                        raise CatalogError("source archive expands beyond the safety limit")
+                    if len(parts) == 2 and parts[1] in {"pyproject.toml", "setup.py"}:
+                        has_build_file = True
+                if len(roots) != 1 or not has_build_file:
+                    raise CatalogError("source archive must have one root and a Python build file")
+        except (OSError, tarfile.TarError) as exc:
+            raise CatalogError("artifact is not a valid source archive") from exc

host_agent/src/netbox_store_agent/config.py

@@ -71,6 +71,10 @@ class CommandSettings:
     systemctl_path: Path
     services: tuple[str, ...]
     command_timeout_seconds: int
+    unshare_path: Path | None = None
+    setpriv_path: Path | None = None
+    source_build_uid: int = 65534
+    source_build_gid: int = 65534
 
 
 @dataclass(frozen=True)
@@ -240,7 +244,11 @@ def load_config(path: str | Path, *, allow_insecure_owner: bool = False) -> Conf
     commands_data = _table(
         root,
         "commands",
-        {"python_path", "manage_path", "systemctl_path", "services", "command_timeout_seconds"},
+        {
+            "python_path", "manage_path", "systemctl_path", "services",
+            "command_timeout_seconds", "unshare_path", "setpriv_path",
+            "source_build_uid", "source_build_gid",
+        },
     )
     policy_data = _table(
         root,
@@ -342,6 +350,10 @@ def load_config(path: str | Path, *, allow_insecure_owner: bool = False) -> Conf
         systemctl_path=require_absolute_path(_string(commands_data, "systemctl_path"), "systemctl_path"),
         services=services,
         command_timeout_seconds=_int(commands_data, "command_timeout_seconds", 900, 1, 7200),
+        unshare_path=_optional_path(commands_data, "unshare_path"),
+        setpriv_path=_optional_path(commands_data, "setpriv_path"),
+        source_build_uid=_int(commands_data, "source_build_uid", 65534, 1, 2**31 - 1),
+        source_build_gid=_int(commands_data, "source_build_gid", 65534, 1, 2**31 - 1),
     )
 
     versions: dict[str, str] = {}

host_agent/src/netbox_store_agent/executor.py

@@ -1,6 +1,10 @@
 from __future__ import annotations
 
+import hashlib
+import os
 import secrets
+import shutil
+import tarfile
 import tempfile
 from pathlib import Path
 from typing import Callable
@@ -56,7 +60,11 @@ class OperationProcessor:
 
     @staticmethod
     def _same_release(existing: ManagedPlugin, plan: ReleasePlan) -> None:
-        if existing.version != plan.version or existing.requirements != (plan.requirement(),):
+        expected = plan.artifact_lock() if existing.artifact_lock else plan.requirement()
+        current_lock = existing.artifact_lock or (
+            existing.requirements[0] if existing.requirements else {}
+        )
+        if existing.version != plan.version or current_lock != expected:
             raise PolicyError(
                 "managed artifact lock differs from the current immutable Store release"
             )
@@ -105,6 +113,71 @@ class OperationProcessor:
             ],
         )
 
+    def _build_source_wheel(
+        self, operation_id: str, directory: Path, plan: ReleasePlan, archive: Path
+    ) -> Path:
+        if (
+            os.name != "posix"
+            or self.config.commands.unshare_path is None
+            or self.config.commands.setpriv_path is None
+        ):
+            raise PolicyError("source builds require configured Linux unshare and setpriv executables")
+        source_dir = directory / "source"
+        wheel_dir = directory / "wheel"
+        source_dir.mkdir(mode=0o700)
+        wheel_dir.mkdir(mode=0o700)
+        with tarfile.open(archive, "r:gz") as bundle:
+            bundle.extractall(source_dir, filter="data")
+        roots = list(source_dir.iterdir())
+        if len(roots) != 1 or not roots[0].is_dir():
+            raise PolicyError("source archive extraction produced an invalid root")
+        uid = self.config.commands.source_build_uid
+        gid = self.config.commands.source_build_gid
+        for path in [directory, source_dir, wheel_dir, *source_dir.rglob("*")]:
+            os.chown(path, uid, gid, follow_symlinks=False)
+            if path.is_dir():
+                os.chmod(path, 0o700)
+        self._run(
+            operation_id,
+            "source_build",
+            [
+                str(self.config.commands.unshare_path), "--net", "--",
+                str(self.config.commands.setpriv_path), f"--reuid={uid}", f"--regid={gid}",
+                "--clear-groups", "--no-new-privs", "--",
+                str(self.config.commands.python_path), "-m", "pip", "wheel",
+                "--no-input", "--disable-pip-version-check", "--no-index", "--no-deps",
+                "--no-build-isolation", "--wheel-dir", str(wheel_dir), str(roots[0]),
+            ],
+        )
+        wheels = list(wheel_dir.glob("*.whl"))
+        if len(wheels) != 1:
+            raise PolicyError("source build must produce exactly one wheel")
+        self.store.validate_wheel(plan, wheels[0])
+        return wheels[0]
+
+    def _cache_wheel(self, plan: ReleasePlan, wheel: Path) -> tuple[Path, dict[str, object]]:
+        digest = hashlib.sha256(wheel.read_bytes()).hexdigest()
+        cache = (
+            self.config.paths.allowed_root
+            / ".netbox-store-wheels"
+            / plan.plugin.slug
+            / plan.approved_payload_sha256
+        )
+        cache.mkdir(parents=True, exist_ok=True, mode=0o700)
+        destination = cache / wheel.name
+        if destination.exists() and destination.is_symlink():
+            raise PolicyError("wheel cache destination may not be a symlink")
+        temporary = cache / (wheel.name + ".tmp")
+        shutil.copyfile(wheel, temporary)
+        os.chmod(temporary, 0o600)
+        os.replace(temporary, destination)
+        requirement = {
+            "package_name": plan.plugin.package_name, "version": plan.version,
+            "download_url": destination.as_uri(), "filename": destination.name,
+            "sha256": digest, "size": destination.stat().st_size,
+        }
+        return destination, requirement
+
     def _pip_uninstall(self, operation_id: str, package_name: str) -> None:
         self._run(
             operation_id,
@@ -184,23 +257,26 @@ class OperationProcessor:
             if existing is not None:
                 self._same_identity(existing, plan.plugin)
             enabled = existing.enabled if existing else False
-            managed = ManagedPlugin(
-                slug=request.plugin_slug,
-                package_name=plan.plugin.package_name,
-                import_name=plan.plugin.import_name,
-                version=plan.version,
-                enabled=enabled,
-                requirements=(plan.requirement(),),
-            )
             temp_root = self.config.paths.temp_dir
             if temp_root.is_symlink():
                 raise PolicyError("temporary directory may not be a symlink")
             temp_root.mkdir(parents=True, exist_ok=True, mode=0o700)
             with tempfile.TemporaryDirectory(prefix="operation-", dir=temp_root) as temporary:
-                self._event(operation_id, "artifact", "Downloading and verifying approved wheel")
-                wheel = self.store.download_release(plan, Path(temporary))
+                self._event(operation_id, "artifact", "Downloading and verifying approved artifact")
+                artifact = self.store.download_release(plan, Path(temporary))
                 if self.config.agent.dry_run:
                     return self._dry_result(request, version=plan.version, artifact_verified=True), False
+                wheel = (
+                    self._build_source_wheel(operation_id, Path(temporary), plan, artifact)
+                    if plan.artifact_kind == "source_archive"
+                    else artifact
+                )
+                wheel, requirement = self._cache_wheel(plan, wheel)
+                managed = ManagedPlugin(
+                    slug=request.plugin_slug, package_name=plan.plugin.package_name,
+                    import_name=plan.plugin.import_name, version=plan.version, enabled=enabled,
+                    requirements=(requirement,), artifact_lock=plan.artifact_lock(),
+                )
                 self._mark_mutated(operation_id)
                 host_mutated = True
                 self._pip_install(operation_id, Path(temporary), managed, wheel)
@@ -239,6 +315,7 @@ class OperationProcessor:
                 existing.version,
                 True,
                 existing.requirements,
+                existing.artifact_lock,
             )
             if self.config.agent.dry_run:
                 return self._dry_result(request, version=existing.version, enabled=True), False
@@ -260,6 +337,7 @@ class OperationProcessor:
                 existing.version,
                 False,
                 existing.requirements,
+                existing.artifact_lock,
             )
             if self.config.agent.dry_run:
                 return self._dry_result(request, version=existing.version, enabled=False), False

host_agent/src/netbox_store_agent/journal.py

@@ -31,6 +31,7 @@ class ManagedPlugin:
     version: str
     enabled: bool
     requirements: tuple[dict[str, Any], ...]
+    artifact_lock: dict[str, Any] | None = None
 
 
 class Journal:
@@ -111,10 +112,17 @@ class Journal:
                     version TEXT NOT NULL,
                     enabled INTEGER NOT NULL CHECK(enabled IN (0, 1)),
                     requirements_json TEXT NOT NULL,
+                    artifact_lock_json TEXT NOT NULL DEFAULT '{}',
                     updated_at TEXT NOT NULL
                 );
                 """
             )
+            columns = {row[1] for row in connection.execute("PRAGMA table_info(managed_plugins)")}
+            if "artifact_lock_json" not in columns:
+                connection.execute(
+                    "ALTER TABLE managed_plugins ADD COLUMN artifact_lock_json "
+                    "TEXT NOT NULL DEFAULT '{}'"
+                )
 
     @staticmethod
     def _operation_dict(row: sqlite3.Row, events: list[dict[str, Any]] | None = None) -> dict[str, Any]:
@@ -306,6 +314,7 @@ class Journal:
             version=row["version"],
             enabled=bool(row["enabled"]),
             requirements=tuple(requirements),
+            artifact_lock=json.loads(row["artifact_lock_json"]),
         )
 
     def get_managed_plugin(self, slug: str) -> ManagedPlugin | None:
@@ -322,18 +331,20 @@ class Journal:
 
     def upsert_managed_plugin(self, plugin: ManagedPlugin) -> None:
         requirements_json = canonical_json(list(plugin.requirements))
+        artifact_lock_json = canonical_json(plugin.artifact_lock)
         with self._connect() as connection:
             connection.execute(
                 """
                 INSERT INTO managed_plugins(
-                    slug, package_name, import_name, version, enabled, requirements_json, updated_at
-                ) VALUES (?, ?, ?, ?, ?, ?, ?)
+                    slug, package_name, import_name, version, enabled, requirements_json, artifact_lock_json, updated_at
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                 ON CONFLICT(slug) DO UPDATE SET
                     package_name=excluded.package_name,
                     import_name=excluded.import_name,
                     version=excluded.version,
                     enabled=excluded.enabled,
                     requirements_json=excluded.requirements_json,
+                    artifact_lock_json=excluded.artifact_lock_json,
                     updated_at=excluded.updated_at
                 """,
                 (
@@ -343,6 +354,7 @@ class Journal:
                     plugin.version,
                     int(plugin.enabled),
                     requirements_json,
+                    artifact_lock_json,
                     utc_now(),
                 ),
             )

host_agent/src/netbox_store_agent/managed_files.py

@@ -144,6 +144,7 @@ class ManagedFiles:
     def render_requirements(self, plugins: Iterable[ManagedPlugin]) -> bytes:
         requirements: dict[str, dict[str, object]] = {}
         schemes = {"https", "http"} if self.config.store.allow_http_for_testing else {"https"}
+        schemes.add("file")
         for plugin in plugins:
             for raw in plugin.requirements:
                 if not isinstance(raw, dict):
@@ -166,10 +167,17 @@ class ManagedFiles:
                 if not isinstance(url, str) or any(char in url for char in "\r\n\t "):
                     raise ValidationError("managed requirement URL is invalid")
                 parsed = urlsplit(url)
+                remote_invalid = parsed.scheme != "file" and (
+                    not parsed.hostname or parsed.hostname.lower() not in self.config.store.allowed_hosts
+                )
+                local_path = Path(parsed.path.lstrip("/") if os.name == "nt" else parsed.path)
+                local_invalid = parsed.scheme == "file" and not is_relative_to(
+                    local_path.resolve(strict=False), self.config.paths.allowed_root
+                )
                 if (
                     parsed.scheme not in schemes
-                    or not parsed.hostname
-                    or parsed.hostname.lower() not in self.config.store.allowed_hosts
+                    or remote_invalid
+                    or local_invalid
                     or parsed.username
                     or parsed.password
                     or parsed.fragment

host_agent/tests/support.py

@@ -88,6 +88,8 @@ def release_json(version: str = "1.2.3", **overrides: Any) -> dict[str, Any]:
         "download_url": f"http://artifacts.test/demo_plugin-{version}-py3-none-any.whl",
         "sha256": "a" * 64,
         "artifact_size": 100,
+        "artifact_kind": "wheel",
+        "artifact_filename": f"demo_plugin-{version}-py3-none-any.whl",
         "commit_sha": "",
         "min_netbox_version": "4.6.5",
         "max_netbox_version": "4.6.8",

host_agent/tests/test_catalog.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import hashlib
+import io
+import tarfile
 import tempfile
 import unittest
 from pathlib import Path
@@ -48,6 +50,36 @@ class CatalogTests(unittest.TestCase):
         plan = StoreClient(self.config, transport).get_release("demo-plugin", "1.2.3")
         self.assertEqual(plan.version, "1.2.3")
 
+    def test_commit_bound_source_archive_is_verified(self) -> None:
+        buffer = io.BytesIO()
+        with tarfile.open(fileobj=buffer, mode="w:gz") as archive:
+            content = b"[build-system]\nrequires = []\n"
+            info = tarfile.TarInfo("demo/pyproject.toml")
+            info.size = len(content)
+            archive.addfile(info, io.BytesIO(content))
+        artifact = buffer.getvalue()
+        commit = "d" * 40
+        release = release_json(
+            download_url=f"http://artifacts.test/demo/archive/{commit}.tar.gz",
+            artifact_kind="source_archive",
+            artifact_filename="demo-plugin-1.2.3-source.tar.gz",
+            commit_sha=commit,
+            sha256=hashlib.sha256(artifact).hexdigest(),
+            artifact_size=len(artifact),
+        )
+        transport = FakeTransport(
+            {
+                "http://store.test/api/v1/plugins/demo-plugin": plugin_json(),
+                "http://store.test/api/v1/plugins/demo-plugin/releases/1.2.3": release,
+            },
+            artifact,
+        )
+        client = StoreClient(self.config, transport)
+        plan = client.get_release("demo-plugin", "1.2.3")
+        directory = self.root / "source-operation"
+        directory.mkdir()
+        self.assertEqual(client.download_release(plan, directory).read_bytes(), artifact)
+
     def test_unapproved_or_mutable_release_rejected(self) -> None:
         for change in ({"approved": False}, {"immutable": False}, {"status": "pending"}):
             with self.subTest(change=change):