From 847cc13a8f886dd7a2fb103b7d628fb4dd0ec17f Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 10 Jul 2026 11:06:12 +0200 Subject: [PATCH] Add Proxmox VE sync support --- COMPATIBILITY.md | 1 + README.md | 27 +- netbox_vmware_importer/__init__.py | 6 +- netbox_vmware_importer/choices.py | 20 + netbox_vmware_importer/crypto.py | 2 +- netbox_vmware_importer/filtersets.py | 6 +- netbox_vmware_importer/forms.py | 26 +- netbox_vmware_importer/jobs.py | 10 +- .../0003_add_proxmox_endpoint_fields.py | 69 ++++ netbox_vmware_importer/models.py | 43 +- netbox_vmware_importer/navigation.py | 6 +- netbox_vmware_importer/sync.py | 377 +++++++++++++++++- netbox_vmware_importer/tables.py | 3 + .../vcenterendpoint.html | 16 +- netbox_vmware_importer/views.py | 2 +- pyproject.toml | 5 +- 16 files changed, 569 insertions(+), 50 deletions(-) create mode 100644 netbox_vmware_importer/migrations/0003_add_proxmox_endpoint_fields.py diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index dff731b..7bc78ff 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -11,3 +11,4 @@ | 0.1.6 | 4.4.0 | 4.6.x | | 0.1.7 | 4.4.0 | 4.6.x | | 0.1.8 | 4.4.0 | 4.6.x | +| 0.2.0 | 4.4.0 | 4.6.x | diff --git a/README.md b/README.md index 1f8239e..b944b72 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,21 @@ -# NetBox VMware Importer +# NetBox VMware & Proxmox Importer -NetBox plugin zum Synchronisieren von VMware vSphere/vCenter VMs nach NetBox. +NetBox plugin zum Synchronisieren von VMware vSphere/vCenter und Proxmox VE VMs nach NetBox. ## Funktionen - vCenter/ESXi-Ziele direkt in der NetBox-Weboberflaeche pflegen +- Proxmox VE-Ziele direkt in der NetBox-Weboberflaeche pflegen - Pro Ziel Tenant, Site und NetBox-Cluster hinterlegen - Manuelle Synchronisation per Button - Optionale automatische Synchronisation ueber ein Minutenintervall - Import von VM-Name, Status, vCPU, RAM, Disk, Plattform, Interfaces, MAC-Adressen und IPs - MAC-Adressen werden als NetBox MAC Address Objekte am VM-Interface angelegt und als primaere MAC gesetzt -- Virtuelle VMware-Festplatten werden als NetBox Virtual Disks mit Groesse und Backing-Info dokumentiert +- Virtuelle Festplatten werden als NetBox Virtual Disks mit Groesse und Backing-Info dokumentiert - Primaere IPv4/IPv6 wird anhand der ersten gefundenen Gast-IP gesetzt - Multi-Tenant-sicherer VM-Abgleich ueber `name + cluster` -Das Plugin loescht bewusst keine verwaisten VMs, Interfaces oder IP-Adressen. Damit bleibt der erste Rollout konservativ und vermeidet Datenverlust, wenn VMware-Gastdaten unvollstaendig sind. +Das Plugin loescht bewusst keine verwaisten VMs, Interfaces oder IP-Adressen. Damit bleibt der erste Rollout konservativ und vermeidet Datenverlust, wenn Hypervisor-Gastdaten unvollstaendig sind. ## Installation @@ -43,11 +44,13 @@ sudo systemctl restart netbox netbox-rq ## Nutzung -1. In NetBox unter `VMware Import > vCenter Endpoints` ein Ziel anlegen. -2. vCenter Host, Port, Benutzername und Passwort eintragen. -3. Tenant, Site und Cluster fuer den Kunden auswaehlen. -4. Optional Regex-Filter und Sync-Intervall setzen. -5. Auf der Detailseite `Sync jetzt starten` ausfuehren. +1. In NetBox unter `VM Import > Endpoints` ein Ziel anlegen. +2. Provider auswaehlen. +3. VMware: vCenter Host, Port, Benutzername und Passwort eintragen. +4. Proxmox: Proxmox Host, Port `8006`, Benutzername und Passwort oder API-Token eintragen. +5. Tenant, Site und Cluster fuer den Kunden auswaehlen. +6. Optional Regex-Filter und Sync-Intervall setzen. +7. Auf der Detailseite `Sync jetzt starten` ausfuehren. Wenn `Sync interval minutes` gesetzt ist, prueft ein Systemjob alle fuenf Minuten, welche Endpoints faellig sind, und stellt die eigentlichen Sync-Jobs in die Queue. @@ -55,10 +58,10 @@ Ab Version `0.1.8` wird beim Speichern eines Endpoints zusaetzlich ein wiederkeh ## Sicherheit -Das vCenter-Passwort wird verschluesselt in der Plugin-Tabelle gespeichert. Der Schluessel wird aus `SECRET_KEY` abgeleitet. Wenn `SECRET_KEY` rotiert wird, muessen die vCenter-Passwoerter in den Endpoints erneut gesetzt werden. +Das Endpoint-Passwort bzw. Token-Secret wird verschluesselt in der Plugin-Tabelle gespeichert. Der Schluessel wird aus `SECRET_KEY` abgeleitet. Wenn `SECRET_KEY` rotiert wird, muessen die Secrets in den Endpoints erneut gesetzt werden. Fuer sehr strenge Umgebungen ist ein externer Secret-Store wie Vault als naechster sinnvoller Ausbaupunkt vorgesehen. -## VMware-Datenqualitaet +## Datenqualitaet -IP-Adressen kommen aus den VMware Guest Tools. Ohne laufende bzw. aktuelle VMware Tools koennen Interfaces und IPs fehlen oder veraltet sein. +IP-Adressen kommen bei VMware aus den VMware Guest Tools und bei Proxmox-QEMU aus dem QEMU Guest Agent. Ohne laufende bzw. aktuelle Guest Tools koennen Interfaces und IPs fehlen oder veraltet sein. diff --git a/netbox_vmware_importer/__init__.py b/netbox_vmware_importer/__init__.py index b81242b..df49ae2 100644 --- a/netbox_vmware_importer/__init__.py +++ b/netbox_vmware_importer/__init__.py @@ -3,9 +3,9 @@ from netbox.plugins import PluginConfig class VMwareImporterConfig(PluginConfig): name = "netbox_vmware_importer" - verbose_name = "VMware Importer" - description = "Synchronize VMware vSphere virtual machines into NetBox." - version = "0.1.8" + verbose_name = "Virtualization Importer" + description = "Synchronize VMware vSphere and Proxmox VE virtual machines into NetBox." + version = "0.2.0" author = "Internal NetBox Team" base_url = "vmware-importer" min_version = "4.4.0" diff --git a/netbox_vmware_importer/choices.py b/netbox_vmware_importer/choices.py index 34ef047..5b6b19b 100644 --- a/netbox_vmware_importer/choices.py +++ b/netbox_vmware_importer/choices.py @@ -16,3 +16,23 @@ class SyncStatusChoices(ChoiceSet): (STATUS_SUCCESS, _("Success"), "green"), (STATUS_FAILED, _("Failed"), "red"), ] + + +class EndpointProviderChoices(ChoiceSet): + PROVIDER_VMWARE = "vmware" + PROVIDER_PROXMOX = "proxmox" + + CHOICES = [ + (PROVIDER_VMWARE, _("VMware vSphere"), "blue"), + (PROVIDER_PROXMOX, _("Proxmox VE"), "orange"), + ] + + +class EndpointAuthMethodChoices(ChoiceSet): + METHOD_PASSWORD = "password" + METHOD_API_TOKEN = "api_token" + + CHOICES = [ + (METHOD_PASSWORD, _("Username/password"), "blue"), + (METHOD_API_TOKEN, _("API token"), "green"), + ] diff --git a/netbox_vmware_importer/crypto.py b/netbox_vmware_importer/crypto.py index 4cb2df8..e12108a 100644 --- a/netbox_vmware_importer/crypto.py +++ b/netbox_vmware_importer/crypto.py @@ -22,4 +22,4 @@ def decrypt_secret(value): try: return _fernet().decrypt(value.encode("ascii")).decode("utf-8") except InvalidToken as exc: - raise ValueError("Stored VMware password cannot be decrypted. Re-enter it on the endpoint.") from exc + raise ValueError("Stored endpoint password cannot be decrypted. Re-enter it on the endpoint.") from exc diff --git a/netbox_vmware_importer/filtersets.py b/netbox_vmware_importer/filtersets.py index 07ce34a..799b70d 100644 --- a/netbox_vmware_importer/filtersets.py +++ b/netbox_vmware_importer/filtersets.py @@ -5,7 +5,7 @@ from netbox.filtersets import NetBoxModelFilterSet from tenancy.models import Tenant from virtualization.models import Cluster -from .choices import SyncStatusChoices +from .choices import EndpointProviderChoices, SyncStatusChoices from .models import VCenterEndpoint @@ -25,6 +25,9 @@ class VCenterEndpointFilterSet(NetBoxModelFilterSet): last_status = django_filters.MultipleChoiceFilter( choices=SyncStatusChoices, ) + provider = django_filters.MultipleChoiceFilter( + choices=EndpointProviderChoices, + ) class Meta: model = VCenterEndpoint @@ -33,6 +36,7 @@ class VCenterEndpointFilterSet(NetBoxModelFilterSet): "name", "slug", "enabled", + "provider", "host", "tenant_id", "site_id", diff --git a/netbox_vmware_importer/forms.py b/netbox_vmware_importer/forms.py index 8a1df1d..33a71c5 100644 --- a/netbox_vmware_importer/forms.py +++ b/netbox_vmware_importer/forms.py @@ -12,7 +12,7 @@ from utilities.forms.fields import ( from utilities.forms.rendering import FieldSet from virtualization.models import Cluster -from .choices import SyncStatusChoices +from .choices import EndpointAuthMethodChoices, EndpointProviderChoices, SyncStatusChoices from .models import VCenterEndpoint @@ -29,8 +29,16 @@ class VCenterEndpointForm(NetBoxModelForm): cluster = DynamicModelChoiceField( queryset=Cluster.objects.all(), ) + provider = forms.ChoiceField( + choices=EndpointProviderChoices, + required=True, + ) + auth_method = forms.ChoiceField( + choices=EndpointAuthMethodChoices, + required=True, + ) password = forms.CharField( - label=_("Password"), + label=_("Password / token secret"), required=False, widget=forms.PasswordInput(render_value=False), help_text=_("Leave blank to keep the currently stored password."), @@ -38,7 +46,7 @@ class VCenterEndpointForm(NetBoxModelForm): fieldsets = ( FieldSet("name", "slug", "enabled", "comments", name=_("Endpoint")), - FieldSet("host", "port", "username", "password", "validate_ssl", name=_("VMware connection")), + FieldSet("provider", "host", "port", "username", "auth_method", "token_name", "password", "validate_ssl", name=_("Connection")), FieldSet("tenant", "site", "cluster", name=_("NetBox target")), FieldSet( "include_name_regex", @@ -47,6 +55,7 @@ class VCenterEndpointForm(NetBoxModelForm): "sync_interfaces", "sync_ip_addresses", "sync_primary_ips", + "sync_lxc_containers", "update_existing", name=_("Sync behavior"), ), @@ -60,9 +69,12 @@ class VCenterEndpointForm(NetBoxModelForm): "name", "slug", "enabled", + "provider", "host", "port", "username", + "auth_method", + "token_name", "password", "validate_ssl", "tenant", @@ -74,6 +86,7 @@ class VCenterEndpointForm(NetBoxModelForm): "sync_interfaces", "sync_ip_addresses", "sync_primary_ips", + "sync_lxc_containers", "update_existing", "default_ipv4_prefix_length", "default_ipv6_prefix_length", @@ -133,7 +146,12 @@ class VCenterEndpointFilterForm(NetBoxModelFilterSetForm): required=False, label=_("Last status"), ) + provider = forms.MultipleChoiceField( + choices=EndpointProviderChoices, + required=False, + label=_("Provider"), + ) fieldsets = ( - FieldSet("q", "enabled", "tenant_id", "site_id", "cluster_id", "last_status", name=_("Endpoint")), + FieldSet("q", "enabled", "provider", "tenant_id", "site_id", "cluster_id", "last_status", name=_("Endpoint")), ) diff --git a/netbox_vmware_importer/jobs.py b/netbox_vmware_importer/jobs.py index b6ce8ee..5b45489 100644 --- a/netbox_vmware_importer/jobs.py +++ b/netbox_vmware_importer/jobs.py @@ -33,7 +33,7 @@ def schedule_endpoint_sync(endpoint): class SyncVCenterEndpointJob(JobRunner): class Meta: - name = "VMware VM synchronization" + name = "VM synchronization" def run(self, *args, **kwargs): endpoint = self.job.object @@ -45,7 +45,7 @@ class SyncVCenterEndpointJob(JobRunner): endpoint.mark_failure("Endpoint is disabled.") raise JobFailed("Endpoint is disabled.") - self.logger.info("Starting VMware sync for %s (%s)", endpoint.name, endpoint.host) + self.logger.info("Starting VM sync for %s (%s)", endpoint.name, endpoint.host) endpoint.mark_running() try: @@ -82,7 +82,7 @@ class SyncVCenterEndpointJob(JobRunner): endpoint.mark_success(result) self.logger.info( ( - "VMware sync finished for %s: %s VM(s), %s created, %s updated, %s skipped; " + "VM sync finished for %s: %s VM(s), %s created, %s updated, %s skipped; " "virtual disks: %s created, %s updated, %s deleted; " "MAC addresses: %s created, %s updated, %s deleted." ), @@ -103,7 +103,7 @@ class SyncVCenterEndpointJob(JobRunner): @system_job(interval=5) class ScheduleDueVCenterSyncsJob(JobRunner): class Meta: - name = "Schedule due VMware VM synchronizations" + name = "Schedule due VM synchronizations" def run(self, *args, **kwargs): now = timezone.now() @@ -120,4 +120,4 @@ class ScheduleDueVCenterSyncsJob(JobRunner): if schedule_endpoint_sync(endpoint): scheduled += 1 - self.logger.info("Ensured %s VMware endpoint sync schedule(s).", scheduled) + self.logger.info("Ensured %s endpoint sync schedule(s).", scheduled) diff --git a/netbox_vmware_importer/migrations/0003_add_proxmox_endpoint_fields.py b/netbox_vmware_importer/migrations/0003_add_proxmox_endpoint_fields.py new file mode 100644 index 0000000..75e202f --- /dev/null +++ b/netbox_vmware_importer/migrations/0003_add_proxmox_endpoint_fields.py @@ -0,0 +1,69 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("netbox_vmware_importer", "0002_align_netbox_46_model_state"), + ] + + operations = [ + migrations.AlterField( + model_name="vcenterendpoint", + name="host", + field=models.CharField(help_text="vCenter, ESXi, or Proxmox VE hostname or IP address", max_length=255), + ), + migrations.AlterField( + model_name="vcenterendpoint", + name="validate_ssl", + field=models.BooleanField(default=False, help_text="Validate the endpoint TLS certificate."), + ), + migrations.AlterField( + model_name="vcenterendpoint", + name="cluster", + field=models.ForeignKey( + help_text="NetBox cluster into which hypervisor VMs will be synchronized.", + on_delete=django.db.models.deletion.PROTECT, + related_name="vmware_import_endpoints", + to="virtualization.cluster", + ), + ), + migrations.AddField( + model_name="vcenterendpoint", + name="provider", + field=models.CharField( + choices=[ + ("vmware", "VMware vSphere"), + ("proxmox", "Proxmox VE"), + ], + default="vmware", + max_length=30, + ), + ), + migrations.AddField( + model_name="vcenterendpoint", + name="auth_method", + field=models.CharField( + choices=[ + ("password", "Username/password"), + ("api_token", "API token"), + ], + default="password", + max_length=30, + ), + ), + migrations.AddField( + model_name="vcenterendpoint", + name="token_name", + field=models.CharField( + blank=True, + help_text="Proxmox API token ID, e.g. netbox-import. The token secret is stored in the password field.", + max_length=100, + ), + ), + migrations.AddField( + model_name="vcenterendpoint", + name="sync_lxc_containers", + field=models.BooleanField(default=True, verbose_name="Sync Proxmox LXC containers"), + ), + ] diff --git a/netbox_vmware_importer/models.py b/netbox_vmware_importer/models.py index 1780aa3..d01cc20 100644 --- a/netbox_vmware_importer/models.py +++ b/netbox_vmware_importer/models.py @@ -10,7 +10,7 @@ from django.utils.translation import gettext_lazy as _ from netbox.models import NetBoxModel from netbox.models.features import JobsMixin -from .choices import SyncStatusChoices +from .choices import EndpointAuthMethodChoices, EndpointProviderChoices, SyncStatusChoices from .crypto import decrypt_secret, encrypt_secret @@ -26,9 +26,14 @@ class VCenterEndpoint(JobsMixin, NetBoxModel): enabled = models.BooleanField( default=True, ) + provider = models.CharField( + max_length=30, + choices=EndpointProviderChoices, + default=EndpointProviderChoices.PROVIDER_VMWARE, + ) host = models.CharField( max_length=255, - help_text=_("vCenter or ESXi hostname or IP address"), + help_text=_("vCenter, ESXi, or Proxmox VE hostname or IP address"), ) port = models.PositiveIntegerField( default=443, @@ -37,13 +42,23 @@ class VCenterEndpoint(JobsMixin, NetBoxModel): username = models.CharField( max_length=255, ) + auth_method = models.CharField( + max_length=30, + choices=EndpointAuthMethodChoices, + default=EndpointAuthMethodChoices.METHOD_PASSWORD, + ) + token_name = models.CharField( + max_length=100, + blank=True, + help_text=_("Proxmox API token ID, e.g. netbox-import. The token secret is stored in the password field."), + ) password_ciphertext = models.TextField( blank=True, editable=False, ) validate_ssl = models.BooleanField( default=False, - help_text=_("Validate the VMware endpoint certificate."), + help_text=_("Validate the endpoint TLS certificate."), ) tenant = models.ForeignKey( to="tenancy.Tenant", @@ -63,7 +78,7 @@ class VCenterEndpoint(JobsMixin, NetBoxModel): to="virtualization.Cluster", on_delete=models.PROTECT, related_name="vmware_import_endpoints", - help_text=_("NetBox cluster into which VMware VMs will be synchronized."), + help_text=_("NetBox cluster into which hypervisor VMs will be synchronized."), ) include_name_regex = models.CharField( max_length=255, @@ -88,6 +103,10 @@ class VCenterEndpoint(JobsMixin, NetBoxModel): sync_primary_ips = models.BooleanField( default=True, ) + sync_lxc_containers = models.BooleanField( + default=True, + verbose_name=_("Sync Proxmox LXC containers"), + ) update_existing = models.BooleanField( default=True, help_text=_("Update existing VMs matched by name and cluster."), @@ -154,9 +173,12 @@ class VCenterEndpoint(JobsMixin, NetBoxModel): clone_fields = ( "enabled", + "provider", "host", "port", "username", + "auth_method", + "token_name", "validate_ssl", "tenant", "site", @@ -167,6 +189,7 @@ class VCenterEndpoint(JobsMixin, NetBoxModel): "sync_interfaces", "sync_ip_addresses", "sync_primary_ips", + "sync_lxc_containers", "update_existing", "default_ipv4_prefix_length", "default_ipv6_prefix_length", @@ -175,8 +198,8 @@ class VCenterEndpoint(JobsMixin, NetBoxModel): class Meta: ordering = ("name",) - verbose_name = _("vCenter endpoint") - verbose_name_plural = _("vCenter endpoints") + verbose_name = _("virtualization endpoint") + verbose_name_plural = _("virtualization endpoints") def __str__(self): return self.name @@ -207,6 +230,14 @@ class VCenterEndpoint(JobsMixin, NetBoxModel): if self.cluster.site_id != self.site_id: raise ValidationError({"cluster": _("Selected cluster belongs to a different site.")}) + if self.provider == EndpointProviderChoices.PROVIDER_VMWARE: + if self.auth_method != EndpointAuthMethodChoices.METHOD_PASSWORD: + raise ValidationError({"auth_method": _("VMware endpoints currently support username/password authentication only.")}) + + if self.provider == EndpointProviderChoices.PROVIDER_PROXMOX: + if self.auth_method == EndpointAuthMethodChoices.METHOD_API_TOKEN and not self.token_name: + raise ValidationError({"token_name": _("A token name is required for Proxmox API token authentication.")}) + def mark_queued(self): self.last_status = SyncStatusChoices.STATUS_QUEUED self.last_message = _("Sync job queued.") diff --git a/netbox_vmware_importer/navigation.py b/netbox_vmware_importer/navigation.py index 5feec7a..db361c1 100644 --- a/netbox_vmware_importer/navigation.py +++ b/netbox_vmware_importer/navigation.py @@ -4,7 +4,7 @@ from netbox.plugins import PluginMenu, PluginMenuButton, PluginMenuItem endpoint_item = PluginMenuItem( link="plugins:netbox_vmware_importer:vcenterendpoint_list", - link_text="vCenter Endpoints", + link_text="Endpoints", permissions=["netbox_vmware_importer.view_vcenterendpoint"], buttons=( PluginMenuButton( @@ -18,9 +18,9 @@ endpoint_item = PluginMenuItem( ) menu = PluginMenu( - label="VMware Import", + label="VM Import", groups=( - ("VMware", (endpoint_item,)), + ("Providers", (endpoint_item,)), ), icon_class="mdi mdi-cloud-sync", ) diff --git a/netbox_vmware_importer/sync.py b/netbox_vmware_importer/sync.py index 684242c..5a7a022 100644 --- a/netbox_vmware_importer/sync.py +++ b/netbox_vmware_importer/sync.py @@ -10,6 +10,13 @@ from dcim.models import MACAddress, Platform from ipam.models import IPAddress from virtualization.models import VMInterface, VirtualDisk, VirtualMachine +from .choices import EndpointAuthMethodChoices, EndpointProviderChoices + +try: + import requests +except ImportError: # pragma: no cover - handled at runtime inside NetBox + requests = None + try: from pyVim.connect import Disconnect, SmartConnect from pyVmomi import vim @@ -70,6 +77,10 @@ class VMwareConnectionError(RuntimeError): pass +class ProxmoxConnectionError(RuntimeError): + pass + + class VMwareClient: def __init__(self, endpoint): self.endpoint = endpoint @@ -210,12 +221,352 @@ class VMwareClient: return interfaces +class ProxmoxClient: + DISK_KEY_PATTERN = re.compile(r"^(ide|sata|scsi|virtio)\d+$") + LXC_DISK_KEY_PATTERN = re.compile(r"^(rootfs|mp\d+)$") + NET_KEY_PATTERN = re.compile(r"^net\d+$") + + def __init__(self, endpoint): + self.endpoint = endpoint + self.session = None + self.base_url = f"https://{endpoint.host}:{endpoint.port}/api2/json" + + def __enter__(self): + if requests is None: + raise ProxmoxConnectionError("requests is not installed in the NetBox Python environment.") + + self.session = requests.Session() + self.session.verify = self.endpoint.validate_ssl + + if self.endpoint.auth_method == EndpointAuthMethodChoices.METHOD_API_TOKEN: + token_id = self.endpoint.token_name if "!" in self.endpoint.token_name else f"{self.endpoint.username}!{self.endpoint.token_name}" + self.session.headers.update({"Authorization": f"PVEAPIToken={token_id}={self.endpoint.password}"}) + else: + self._authenticate_with_password() + + return self + + def __exit__(self, exc_type, exc_value, traceback): + if self.session is not None: + self.session.close() + + def _authenticate_with_password(self): + response = self.session.post( + self._url("access/ticket"), + data={ + "username": self.endpoint.username, + "password": self.endpoint.password, + }, + timeout=30, + ) + response.raise_for_status() + data = response.json().get("data") or {} + ticket = data.get("ticket") + csrf_token = data.get("CSRFPreventionToken") + + if not ticket: + raise ProxmoxConnectionError("Proxmox authentication did not return a ticket.") + + self.session.cookies.set("PVEAuthCookie", ticket) + if csrf_token: + self.session.headers.update({"CSRFPreventionToken": csrf_token}) + + def _url(self, path): + return f"{self.base_url}/{path.strip('/')}" + + def _get(self, path, params=None): + response = self.session.get(self._url(path), params=params, timeout=30) + response.raise_for_status() + return response.json().get("data") + + def iter_virtual_machines(self): + resources = self._get("cluster/resources", params={"type": "vm"}) or [] + + for resource in resources: + vm_type = resource.get("type") + if vm_type == "lxc" and not self.endpoint.sync_lxc_containers: + continue + if vm_type not in ("qemu", "lxc"): + continue + + yield self._build_vm_data(resource) + + def _build_vm_data(self, resource): + vmid = resource["vmid"] + node = resource["node"] + vm_type = resource["type"] + config = self._get(f"nodes/{node}/{vm_type}/{vmid}/config") or {} + + name = resource.get("name") or config.get("name") or config.get("hostname") or f"{vm_type}-{vmid}" + status = "active" if resource.get("status") == "running" else "offline" + memory_mb = self._get_memory_mb(resource, config) + vcpus = self._get_vcpus(resource, config) + + if vm_type == "qemu": + disks = self._get_qemu_disks(config) + interfaces = self._get_qemu_interfaces(config) + self._merge_qemu_agent_interfaces(node, vmid, interfaces) + guest_os = self._get_qemu_guest_os(node, vmid) + else: + disks = self._get_lxc_disks(config) + interfaces = self._get_lxc_interfaces(config) + guest_os = "Linux Container" + + disk_mb = sum(disk.size_mb for disk in disks) or self._bytes_to_mb(resource.get("maxdisk")) + + return VMData( + name=str(name), + status=status, + vcpus=vcpus, + memory_mb=memory_mb, + disk_mb=disk_mb, + guest_os=guest_os, + disks=disks, + interfaces=interfaces, + ) + + @staticmethod + def _get_memory_mb(resource, config): + memory = config.get("memory") + if memory not in (None, ""): + try: + return int(memory) + except (TypeError, ValueError): + pass + + return ProxmoxClient._bytes_to_mb(resource.get("maxmem")) + + @staticmethod + def _get_vcpus(resource, config): + try: + sockets = int(config.get("sockets") or 1) + cores = int(config.get("cores") or resource.get("maxcpu") or 0) + return max(sockets * cores, 0) + except (TypeError, ValueError): + return int(resource.get("maxcpu") or 0) + + @staticmethod + def _bytes_to_mb(value): + try: + return round(int(value or 0) / 1024 / 1024) + except (TypeError, ValueError): + return 0 + + def _get_qemu_disks(self, config): + disks = [] + + for key, value in sorted(config.items()): + if not self.DISK_KEY_PATTERN.match(key): + continue + parsed = self._parse_proxmox_value(value) + if parsed.get("media") == "cdrom": + continue + + size_mb = self._parse_size_mb(parsed.get("size")) + if size_mb <= 0: + continue + + disks.append( + DiskData( + name=key, + size_mb=size_mb, + description=f"Proxmox: {str(value)[:191]}", + ) + ) + + return disks + + def _get_lxc_disks(self, config): + disks = [] + + for key, value in sorted(config.items()): + if not self.LXC_DISK_KEY_PATTERN.match(key): + continue + + parsed = self._parse_proxmox_value(value) + size_mb = self._parse_size_mb(parsed.get("size")) + if size_mb <= 0: + continue + + disks.append( + DiskData( + name=key, + size_mb=size_mb, + description=f"Proxmox: {str(value)[:191]}", + ) + ) + + return disks + + def _get_qemu_interfaces(self, config): + interfaces = [] + + for key, value in sorted(config.items()): + if not self.NET_KEY_PATTERN.match(key): + continue + + parsed = self._parse_proxmox_value(value) + mac_address = self._extract_proxmox_mac(parsed) + if not mac_address: + continue + + bridge = parsed.get("bridge") + name = f"{key} ({bridge})" if bridge else key + interfaces.append(InterfaceData(name=name[:64], mac_address=mac_address)) + + return interfaces + + def _get_lxc_interfaces(self, config): + interfaces = [] + + for key, value in sorted(config.items()): + if not self.NET_KEY_PATTERN.match(key): + continue + + parsed = self._parse_proxmox_value(value) + mac_address = parsed.get("hwaddr") + if not mac_address: + continue + + name = parsed.get("name") or key + ip_addresses = [] + for ip_key in ("ip", "ip6"): + ip_value = parsed.get(ip_key) + if ip_value and ip_value not in ("dhcp", "manual", "auto"): + ip_addresses.append(ip_value) + + interfaces.append( + InterfaceData( + name=str(name)[:64], + mac_address=mac_address, + ip_addresses=ip_addresses, + ) + ) + + return interfaces + + def _merge_qemu_agent_interfaces(self, node, vmid, interfaces): + try: + response = self._get(f"nodes/{node}/qemu/{vmid}/agent/network-get-interfaces") or {} + except requests.RequestException: + return + + agent_interfaces = response.get("result") if isinstance(response, dict) else response + if not isinstance(agent_interfaces, list): + return + + interfaces_by_mac = {self._normalize_mac(interface.mac_address): interface for interface in interfaces} + + for agent_interface in agent_interfaces: + mac_address = agent_interface.get("hardware-address") + normalized_mac = self._normalize_mac(mac_address) + if not normalized_mac: + continue + + ip_addresses = [] + for ip_data in agent_interface.get("ip-addresses") or []: + ip_address = ip_data.get("ip-address") + prefix = ip_data.get("prefix") + if not ip_address: + continue + if prefix is not None: + ip_addresses.append(f"{ip_address}/{prefix}") + else: + ip_addresses.append(ip_address) + + if normalized_mac in interfaces_by_mac: + interfaces_by_mac[normalized_mac].ip_addresses = ip_addresses + else: + interfaces.append( + InterfaceData( + name=str(agent_interface.get("name") or f"NIC-{normalized_mac[-5:]}")[:64], + mac_address=normalized_mac, + ip_addresses=ip_addresses, + ) + ) + + def _get_qemu_guest_os(self, node, vmid): + try: + response = self._get(f"nodes/{node}/qemu/{vmid}/agent/get-osinfo") or {} + except requests.RequestException: + return "" + + result = response.get("result") if isinstance(response, dict) else None + if not isinstance(result, dict): + return "" + + return result.get("pretty-name") or result.get("name") or "" + + @staticmethod + def _parse_proxmox_value(value): + parsed = {"_raw": str(value)} + parts = str(value).split(",") + if parts: + parsed["_first"] = parts[0] + + for part in parts: + if "=" in part: + key, part_value = part.split("=", 1) + parsed[key.strip()] = part_value.strip() + + return parsed + + @staticmethod + def _extract_proxmox_mac(parsed): + for key in ("virtio", "e1000", "e1000e", "rtl8139", "vmxnet3", "hwaddr"): + if parsed.get(key): + return parsed[key] + return None + + @staticmethod + def _parse_size_mb(value): + if value in (None, ""): + return 0 + + match = re.match(r"^\s*([0-9.]+)\s*([KMGTPE]?)(?:i?B)?\s*$", str(value), re.IGNORECASE) + if not match: + return 0 + + amount = float(match.group(1)) + unit = match.group(2).upper() + factors = { + "": 1, + "K": 1 / 1024, + "M": 1, + "G": 1024, + "T": 1024 * 1024, + "P": 1024 * 1024 * 1024, + "E": 1024 * 1024 * 1024 * 1024, + } + + if not unit and amount > 1024 * 1024: + return round(amount / 1024 / 1024) + + return round(amount * factors.get(unit, 1)) + + @staticmethod + def _normalize_mac(mac_address): + if not mac_address: + return None + + value = str(mac_address).strip().replace("-", ":").lower() + parts = value.split(":") + if len(parts) != 6: + return None + + try: + return ":".join(f"{int(part, 16):02x}" for part in parts) + except ValueError: + return None + + class VMwareImporter: def __init__(self, endpoint, logger): self.endpoint = endpoint self.logger = logger self.include_pattern = re.compile(endpoint.include_name_regex) if endpoint.include_name_regex else None self.exclude_pattern = re.compile(endpoint.exclude_name_regex) if endpoint.exclude_name_regex else None + self.provider_name = "Proxmox" if endpoint.provider == EndpointProviderChoices.PROVIDER_PROXMOX else "VMware" self.interface_content_type = None self.mac_address_content_type = None @@ -224,8 +575,9 @@ class VMwareImporter: platforms = list(Platform.objects.all()) self.interface_content_type = ContentType.objects.get_for_model(VMInterface) self.mac_address_content_type = ContentType.objects.get_for_model(VMInterface) + client_class = ProxmoxClient if self.endpoint.provider == EndpointProviderChoices.PROVIDER_PROXMOX else VMwareClient - with VMwareClient(self.endpoint) as client: + with client_class(self.endpoint) as client: for vm_data in client.iter_virtual_machines(): result.seen += 1 @@ -337,15 +689,18 @@ class VMwareImporter: result.virtual_disks_updated += 1 self.logger.info("Updated virtual disk %s on %s", disk_data.name, nb_vm.name) - stale_disks = VirtualDisk.objects.filter( - virtual_machine=nb_vm, - description__startswith="VMware:", - ).exclude(name__in=synced_names) - stale_count = stale_disks.count() - if stale_count: + stale_count = 0 + for description_prefix in (f"{self.provider_name}:",): + stale_disks = VirtualDisk.objects.filter( + virtual_machine=nb_vm, + description__startswith=description_prefix, + ).exclude(name__in=synced_names) + stale_count += stale_disks.count() stale_disks.delete() + + if stale_count: result.virtual_disks_deleted += stale_count - self.logger.info("Deleted %s stale VMware virtual disk(s) from %s", stale_count, nb_vm.name) + self.logger.info("Deleted %s stale %s virtual disk(s) from %s", stale_count, self.provider_name, nb_vm.name) disk_total = nb_vm.virtualdisks.aggregate(total=Sum("size", default=0))["total"] or vm_data.disk_mb if nb_vm.disk != disk_total: @@ -422,7 +777,7 @@ class VMwareImporter: assigned_object=vm_interface, ) - mac_obj.description = "VMware reported MAC address" + mac_obj.description = f"{self.provider_name} reported MAC address" mac_obj.full_clean() mac_obj.save() @@ -442,13 +797,13 @@ class VMwareImporter: stale_macs = MACAddress.objects.filter( assigned_object_type=self.mac_address_content_type, assigned_object_id=vm_interface.pk, - description="VMware reported MAC address", + description=f"{self.provider_name} reported MAC address", ).exclude(pk=mac_obj.pk) stale_count = stale_macs.count() if stale_count: stale_macs.delete() result.mac_addresses_deleted += stale_count - self.logger.info("Deleted %s stale VMware MAC address(es) from interface %s", stale_count, vm_interface.name) + self.logger.info("Deleted %s stale %s MAC address(es) from interface %s", stale_count, self.provider_name, vm_interface.name) return mac_obj diff --git a/netbox_vmware_importer/tables.py b/netbox_vmware_importer/tables.py index ff02938..0866556 100644 --- a/netbox_vmware_importer/tables.py +++ b/netbox_vmware_importer/tables.py @@ -11,6 +11,7 @@ class VCenterEndpointTable(NetBoxTable): linkify=True, ) enabled = BooleanColumn() + provider = ChoiceFieldColumn() tenant = tables.Column( linkify=True, ) @@ -31,6 +32,7 @@ class VCenterEndpointTable(NetBoxTable): "id", "name", "enabled", + "provider", "host", "tenant", "site", @@ -45,6 +47,7 @@ class VCenterEndpointTable(NetBoxTable): "pk", "name", "enabled", + "provider", "host", "tenant", "cluster", diff --git a/netbox_vmware_importer/templates/netbox_vmware_importer/vcenterendpoint.html b/netbox_vmware_importer/templates/netbox_vmware_importer/vcenterendpoint.html index 982f7a2..dfb84f6 100644 --- a/netbox_vmware_importer/templates/netbox_vmware_importer/vcenterendpoint.html +++ b/netbox_vmware_importer/templates/netbox_vmware_importer/vcenterendpoint.html @@ -4,12 +4,16 @@
-
vCenter
+
Endpoint
+ + + + @@ -18,6 +22,16 @@ + + + + + {% if object.token_name %} + + + + + {% endif %} diff --git a/netbox_vmware_importer/views.py b/netbox_vmware_importer/views.py index bf1ed68..513d2c0 100644 --- a/netbox_vmware_importer/views.py +++ b/netbox_vmware_importer/views.py @@ -46,6 +46,6 @@ class VCenterEndpointSyncView(View): SyncVCenterEndpointJob.enqueue(instance=endpoint) endpoint.mark_queued() - messages.success(request, _("VMware sync job queued for %(endpoint)s.") % {"endpoint": endpoint}) + messages.success(request, _("VM sync job queued for %(endpoint)s.") % {"endpoint": endpoint}) return redirect(endpoint) diff --git a/pyproject.toml b/pyproject.toml index 913590d..eedf076 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta" [project] name = "netbox-vmware-importer" -version = "0.1.8" -description = "NetBox plugin to synchronize VMware vSphere virtual machines into NetBox." +version = "0.2.0" +description = "NetBox plugin to synchronize VMware vSphere and Proxmox VE virtual machines into NetBox." readme = "README.md" requires-python = ">=3.12" authors = [ @@ -22,6 +22,7 @@ classifiers = [ dependencies = [ "pyvmomi>=8.0.3", "cryptography>=42.0", + "requests>=2.32", ] [tool.setuptools.packages.find]
Name {{ object.name }}
Provider{{ object.get_provider_display }}
Host {{ object.host }}:{{ object.port }}Username {{ object.username }}
Auth method{{ object.get_auth_method_display }}
Token name{{ object.token_name }}
Enabled {{ object.enabled|yesno:"Yes,No" }}