diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..87b0196 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +netbox_vmware_importer/migrations \ No newline at end of file diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index b782b01..49cdde9 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -17,3 +17,4 @@ | 0.2.3 | 4.4.0 | 4.6.x | | 0.2.4 | 4.4.0 | 4.6.x | | 0.2.5 | 4.4.0 | 4.6.x | +| 0.2.6 | 4.4.0 | 4.6.x | diff --git a/README.md b/README.md index 3a5c4cb..073f9c7 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Im Python-Environment der NetBox-Installation: pip install git+https://git.mrblake.cc/MrBlake/NetBox-VM-Import.git ``` -NetBox selbst ist keine `pip`-Dependency des Plugins. Das Plugin wird im vorhandenen NetBox-Virtualenv installiert und bringt nur externe Bibliotheken wie `pyvmomi` und `cryptography` mit. +NetBox selbst ist keine `pip`-Dependency des Plugins. Das Plugin wird im vorhandenen NetBox-Virtualenv installiert und bringt nur externe Bibliotheken wie `pyvmomi`, `cryptography`, `requests` und `proxmox-sdk` mit. In `configuration.py` aktivieren: @@ -54,12 +54,14 @@ sudo systemctl restart netbox netbox-rq Bei selbstsignierten Proxmox-Zertifikaten `Validate SSL` deaktiviert lassen. Wenn Proxmox langsam antwortet, kann `API timeout` am Endpoint erhoeht werden. -Der Proxmox-Client ignoriert Proxy-Umgebungsvariablen des NetBox-Dienstes, damit interne Proxmox-Adressen nicht versehentlich ueber einen HTTPS-Proxy laufen. +Der Proxmox-Client nutzt bevorzugt `proxmox-sdk` wie `netbox-proxbox`/`proxbox-api`. Falls dieses Paket nicht installiert ist, faellt er auf den bisherigen `requests`-Client zurueck. Proxy-Umgebungsvariablen des NetBox-Dienstes werden im Fallback ignoriert, damit interne Proxmox-Adressen nicht versehentlich ueber einen HTTPS-Proxy laufen. -Der unauthentifizierte Proxmox-`version`-Probe ist nur diagnostisch. Wenn er haengt, versucht das Plugin den eigentlichen Login trotzdem. +Im `requests`-Fallback ist der unauthentifizierte Proxmox-`version`-Probe nur diagnostisch. Wenn er haengt, versucht das Plugin den eigentlichen Login trotzdem. Im bevorzugten SDK-Pfad wird `version` als erste SDK-Anfrage genutzt und authentifiziert dabei automatisch. Proxmox-API-Verbindungen nutzen TLS 1.2-Kompatibilitaetsmodus, weil einige Firewalls oder TLS-Inspektionspfade moderne TLS-1.3-ClientHellos blockieren. +Bei Proxmox koennen `Connect timeout`, `API timeout`, `GET retries` und `Retry backoff` pro Endpoint gesetzt werden. Das entspricht dem Ansatz von Proxbox, Verbindungs- und Retry-Parameter pro Proxmox-Ziel zu speichern. + Wenn Passwort-Login auf `access/ticket` haengt, zuerst den Realm im Benutzernamen pruefen (`root@pam`, `user@pve`, `user@ldaprealm`). Fuer produktive Imports ist ein Proxmox API-Token meistens stabiler als Passwort-Login. Wenn `Sync interval minutes` gesetzt ist, prueft ein Systemjob alle fuenf Minuten, welche Endpoints faellig sind, und stellt die eigentlichen Sync-Jobs in die Queue. diff --git a/netbox_vmware_importer/__init__.py b/netbox_vmware_importer/__init__.py index ac09b90..09c7ce3 100644 --- a/netbox_vmware_importer/__init__.py +++ b/netbox_vmware_importer/__init__.py @@ -5,7 +5,7 @@ class VMwareImporterConfig(PluginConfig): name = "netbox_vmware_importer" verbose_name = "Virtualization Importer" description = "Synchronize VMware vSphere and Proxmox VE virtual machines into NetBox." - version = "0.2.5" + version = "0.2.6" author = "Internal NetBox Team" base_url = "vmware-importer" min_version = "4.4.0" diff --git a/netbox_vmware_importer/forms.py b/netbox_vmware_importer/forms.py index 40d4e51..5e598ec 100644 --- a/netbox_vmware_importer/forms.py +++ b/netbox_vmware_importer/forms.py @@ -56,6 +56,9 @@ class VCenterEndpointForm(NetBoxModelForm): "password", "validate_ssl", "request_timeout_seconds", + "connect_timeout_seconds", + "max_retries", + "retry_backoff_seconds", name=_("Connection"), ), FieldSet("tenant", "site", "cluster", name=_("NetBox target")), @@ -89,6 +92,9 @@ class VCenterEndpointForm(NetBoxModelForm): "password", "validate_ssl", "request_timeout_seconds", + "connect_timeout_seconds", + "max_retries", + "retry_backoff_seconds", "tenant", "site", "cluster", diff --git a/netbox_vmware_importer/models.py b/netbox_vmware_importer/models.py index c05e218..23fd720 100644 --- a/netbox_vmware_importer/models.py +++ b/netbox_vmware_importer/models.py @@ -1,4 +1,5 @@ import re +from decimal import Decimal from datetime import timedelta from django.core.exceptions import ValidationError @@ -65,6 +66,23 @@ class VCenterEndpoint(JobsMixin, NetBoxModel): validators=[MinValueValidator(5), MaxValueValidator(600)], help_text=_("HTTP/API read timeout in seconds."), ) + connect_timeout_seconds = models.PositiveIntegerField( + default=10, + validators=[MinValueValidator(1), MaxValueValidator(120)], + help_text=_("TCP/TLS connection timeout in seconds."), + ) + max_retries = models.PositiveSmallIntegerField( + default=0, + validators=[MinValueValidator(0), MaxValueValidator(10)], + help_text=_("Retries for safe Proxmox GET requests when the endpoint returns transient errors."), + ) + retry_backoff_seconds = models.DecimalField( + max_digits=5, + decimal_places=2, + default=Decimal("0.50"), + validators=[MinValueValidator(Decimal("0.00")), MaxValueValidator(Decimal("60.00"))], + help_text=_("Exponential retry backoff base delay in seconds."), + ) tenant = models.ForeignKey( to="tenancy.Tenant", on_delete=models.PROTECT, @@ -186,6 +204,9 @@ class VCenterEndpoint(JobsMixin, NetBoxModel): "token_name", "validate_ssl", "request_timeout_seconds", + "connect_timeout_seconds", + "max_retries", + "retry_backoff_seconds", "tenant", "site", "cluster", diff --git a/netbox_vmware_importer/sync.py b/netbox_vmware_importer/sync.py index 2bc1214..7c8eaf1 100644 --- a/netbox_vmware_importer/sync.py +++ b/netbox_vmware_importer/sync.py @@ -1,7 +1,9 @@ import ipaddress import re import ssl +import time from dataclasses import dataclass, field +from urllib.parse import urlsplit from django.contrib.contenttypes.models import ContentType from django.db import transaction @@ -21,6 +23,19 @@ except ImportError: # pragma: no cover - handled at runtime inside NetBox requests = None InsecureRequestWarning = None +try: + from proxmox_sdk import ProxmoxSDK + from proxmox_sdk.sdk.exceptions import ( + ProxmoxConnectionError as SDKConnectionError, + ProxmoxTimeoutError as SDKTimeoutError, + ResourceException as SDKResourceException, + ) +except ImportError: # pragma: no cover - optional compatibility fallback + ProxmoxSDK = None + SDKConnectionError = None + SDKTimeoutError = None + SDKResourceException = None + try: from pyVim.connect import Disconnect, SmartConnect from pyVmomi import vim @@ -253,20 +268,114 @@ class ProxmoxClient: DISK_KEY_PATTERN = re.compile(r"^(ide|sata|scsi|virtio)\d+$") LXC_DISK_KEY_PATTERN = re.compile(r"^(rootfs|mp\d+)$") NET_KEY_PATTERN = re.compile(r"^net\d+$") + FULL_TOKEN_PATTERN = re.compile(r"^(?:PVEAPIToken=)?(?P[^!]+)!(?P[^=]+)=(?P.+)$") + RETRY_STATUSES = {502, 503, 504} def __init__(self, endpoint): self.endpoint = endpoint + self.host, self.port = self._parse_endpoint_address(endpoint.host, endpoint.port) + self.url_host = self._format_host_for_url(self.host) self.session = None - self.base_url = f"https://{endpoint.host}:{endpoint.port}/api2/json" - self.timeout = (10, endpoint.request_timeout_seconds or 120) + self.sdk = None + self.uses_sdk = False + self.base_url = f"https://{self.url_host}:{self.port}/api2/json" + self.connect_timeout = endpoint.connect_timeout_seconds or 10 + self.read_timeout = endpoint.request_timeout_seconds or 120 + self.timeout = (self.connect_timeout, self.read_timeout) + self.max_retries = endpoint.max_retries or 0 + self.retry_backoff = float(endpoint.retry_backoff_seconds or 0.5) self.api_probe_succeeded = False + self.api_probe_error = "" def __enter__(self): - if requests is None: - raise ProxmoxConnectionError("requests is not installed in the NetBox Python environment.") if "@" not in self.endpoint.username: raise ProxmoxConnectionError("Proxmox username must include a realm, e.g. root@pam or user@pve.") + if ProxmoxSDK is not None: + self._connect_with_sdk() + return self + + self._connect_with_requests() + return self + + def __exit__(self, exc_type, exc_value, traceback): + if self.sdk is not None: + self.sdk.close() + if self.session is not None: + self.session.close() + + @staticmethod + def _parse_endpoint_address(host, port): + raw_host = str(host or "").strip() + resolved_port = int(port or 8006) + if not raw_host: + return raw_host, resolved_port + + parsed = urlsplit(raw_host if "://" in raw_host else f"//{raw_host}") + parsed_host = parsed.hostname + if parsed.port: + resolved_port = parsed.port + + if parsed_host: + return parsed_host.strip("[]"), resolved_port + + return raw_host.strip("[]"), resolved_port + + @staticmethod + def _format_host_for_url(host): + if ":" in host and not host.startswith("["): + return f"[{host}]" + return host + + def _connect_with_sdk(self): + user = (self.endpoint.username or "").strip() + kwargs = { + "host": self.host, + "backend": "https", + "service": "PVE", + "port": self.port, + "user": user, + "verify_ssl": self.endpoint.validate_ssl, + "timeout": self.read_timeout, + "connect_timeout": self.connect_timeout, + "max_retries": self.max_retries, + "retry_backoff": self.retry_backoff, + } + + if self.endpoint.auth_method == EndpointAuthMethodChoices.METHOD_API_TOKEN: + user, token_name, token_value = self._normalized_token_credentials() + kwargs["user"] = user + kwargs.update( + { + "token_name": token_name, + "token_value": token_value, + } + ) + else: + kwargs["password"] = self.endpoint.password + + try: + self.sdk = ProxmoxSDK.sync(**kwargs) + self.uses_sdk = True + self._get("version") + self.api_probe_succeeded = True + except ProxmoxConnectionError: + if self.sdk is not None: + self.sdk.close() + self.sdk = None + raise + except Exception as exc: + if self.sdk is not None: + self.sdk.close() + self.sdk = None + raise ProxmoxConnectionError(self._describe_sdk_error("GET", "version", exc)) from exc + + def _connect_with_requests(self): + if requests is None: + raise ProxmoxConnectionError( + "Neither proxmox-sdk nor requests is installed in the NetBox Python environment." + ) + self.session = requests.Session() self.session.trust_env = False self.session.verify = self.endpoint.validate_ssl @@ -285,17 +394,11 @@ class ProxmoxClient: self._probe_api_availability() if self.endpoint.auth_method == EndpointAuthMethodChoices.METHOD_API_TOKEN: - token_id = self.endpoint.token_name if "!" in self.endpoint.token_name else f"{self.endpoint.username}!{self.endpoint.token_name}" - self.session.headers.update({"Authorization": f"PVEAPIToken={token_id}={self.endpoint.password}"}) + user, token_name, token_value = self._normalized_token_credentials() + self.session.headers.update({"Authorization": f"PVEAPIToken={user}!{token_name}={token_value}"}) else: self._authenticate_with_password() - return self - - def __exit__(self, exc_type, exc_value, traceback): - if self.session is not None: - self.session.close() - def _probe_api_availability(self): try: self._request("GET", "version", timeout=(5, 10)) @@ -305,6 +408,45 @@ class ProxmoxClient: self.api_probe_succeeded = False self.api_probe_error = str(exc) + def _normalized_token_credentials(self): + user = (self.endpoint.username or "").strip() + token_name = (self.endpoint.token_name or "").strip() + token_value = (self.endpoint.password or "").strip() + + parsed = self._parse_full_token(token_value) + if parsed is not None: + user, token_name, token_value = parsed + else: + parsed = self._parse_full_token(token_name) + if parsed is not None: + parsed_user, parsed_name, parsed_value = parsed + user = parsed_user or user + token_name = parsed_name + if parsed_value and not token_value: + token_value = parsed_value + elif token_name.startswith("PVEAPIToken="): + token_name = token_name.removeprefix("PVEAPIToken=").strip() + + if "!" in token_name: + parsed_user, parsed_name = token_name.rsplit("!", 1) + if parsed_user: + user = parsed_user + token_name = parsed_name + + if not token_name or not token_value: + raise ProxmoxConnectionError( + "Proxmox API token authentication requires a token name and token secret." + ) + + return user, token_name, token_value + + @classmethod + def _parse_full_token(cls, value): + match = cls.FULL_TOKEN_PATTERN.match(str(value or "").strip()) + if not match: + return None + return match.group("user").strip(), match.group("name").strip(), match.group("value").strip() + def _authenticate_with_password(self): response = self._request( "POST", @@ -331,29 +473,86 @@ class ProxmoxClient: def _request(self, method, path, timeout=None, **kwargs): url = self._url(path) request_timeout = timeout or self.timeout + attempts = self.max_retries + 1 if method.upper() == "GET" else 1 + last_error = None + + for attempt in range(attempts): + if attempt: + time.sleep(min(self.retry_backoff * (2 ** (attempt - 1)), 30.0)) + try: + response = self.session.request(method, url, timeout=request_timeout, **kwargs) + if response.status_code in self.RETRY_STATUSES and attempt < attempts - 1: + last_error = ProxmoxConnectionError( + f"Proxmox API {method} {path} returned transient HTTP {response.status_code}." + ) + continue + response.raise_for_status() + return response + except requests.ConnectTimeout as exc: + last_error = ProxmoxConnectionError( + f"Proxmox API {method} {path} connect timed out after {request_timeout[0]} seconds." + ) + if attempt >= attempts - 1: + raise last_error from exc + except requests.ReadTimeout as exc: + detail = f"Proxmox API {method} {path} read timed out after {request_timeout[1]} seconds." + if path == "access/ticket" and self.api_probe_succeeded: + detail += " API version endpoint responded, so the delay is likely in Proxmox authentication. Check username realm (e.g. root@pam), PAM/LDAP auth, two-factor auth, or use an API token." + elif getattr(self, "api_probe_error", "") and path != "version": + detail += f" Earlier unauthenticated version probe also failed: {self.api_probe_error}" + last_error = ProxmoxConnectionError(detail) + if attempt >= attempts - 1: + raise last_error from exc + except requests.exceptions.SSLError as exc: + raise ProxmoxConnectionError( + f"Proxmox API {method} {path} TLS validation failed. Disable Validate SSL for self-signed certificates or install the CA certificate. Details: {exc}" + ) from exc + except requests.RequestException as exc: + last_error = ProxmoxConnectionError(f"Proxmox API {method} {path} failed: {exc}") + if attempt >= attempts - 1: + raise last_error from exc + + if last_error is not None: + raise last_error + raise ProxmoxConnectionError(f"Proxmox API {method} {path} failed without a response.") + + def _sdk_get(self, path, params=None): try: - response = self.session.request(method, url, timeout=request_timeout, **kwargs) - response.raise_for_status() - return response - except requests.ConnectTimeout as exc: - raise ProxmoxConnectionError( - f"Proxmox API {method} {path} connect timed out after {request_timeout[0]} seconds." - ) from exc - except requests.ReadTimeout as exc: - detail = f"Proxmox API {method} {path} read timed out after {request_timeout[1]} seconds." - if path == "access/ticket" and self.api_probe_succeeded: - detail += " API version endpoint responded, so the delay is likely in Proxmox authentication. Check username realm (e.g. root@pam), PAM/LDAP auth, two-factor auth, or use an API token." - elif getattr(self, "api_probe_error", "") and path != "version": - detail += f" Earlier unauthenticated version probe also failed: {self.api_probe_error}" - raise ProxmoxConnectionError(detail) from exc - except requests.exceptions.SSLError as exc: - raise ProxmoxConnectionError( - f"Proxmox API {method} {path} TLS validation failed. Disable Validate SSL for self-signed certificates or install the CA certificate. Details: {exc}" - ) from exc - except requests.RequestException as exc: - raise ProxmoxConnectionError(f"Proxmox API {method} {path} failed: {exc}") from exc + return self.sdk(path.strip("/")).get(**(params or {})) + except Exception as exc: + raise ProxmoxConnectionError(self._describe_sdk_error("GET", path, exc)) from exc + + def _describe_sdk_error(self, method, path, exc): + if SDKTimeoutError is not None and isinstance(exc, SDKTimeoutError): + return f"Proxmox SDK {method} {path} timed out after {self.read_timeout} seconds." + + if SDKConnectionError is not None and isinstance(exc, SDKConnectionError): + detail = str(exc).strip() or "connection failed" + return f"Proxmox SDK {method} {path} connection failed: {detail}" + + if SDKResourceException is not None and isinstance(exc, SDKResourceException): + status_code = getattr(exc, "status_code", None) + status_message = getattr(exc, "status_message", "") or "" + content = (getattr(exc, "content", "") or "").strip() + errors = getattr(exc, "errors", None) + parts = [] + if status_code: + parts.append(f"HTTP {status_code} {status_message}".strip()) + elif status_message: + parts.append(status_message) + if content: + parts.append(content) + if errors: + parts.append(str(errors)) + detail = " - ".join(parts) if parts else str(exc) + return f"Proxmox SDK {method} {path} failed: {detail}" + + detail = str(exc).strip() or "unknown error" + return f"Proxmox SDK {method} {path} failed: {detail}" def _get(self, path, params=None): + if self.uses_sdk: + return self._sdk_get(path, params=params) response = self._request("GET", path, params=params) return response.json().get("data") diff --git a/netbox_vmware_importer/templates/netbox_vmware_importer/vcenterendpoint.html b/netbox_vmware_importer/templates/netbox_vmware_importer/vcenterendpoint.html index 98eb6b5..c17cba3 100644 --- a/netbox_vmware_importer/templates/netbox_vmware_importer/vcenterendpoint.html +++ b/netbox_vmware_importer/templates/netbox_vmware_importer/vcenterendpoint.html @@ -44,6 +44,14 @@ API timeout {{ object.request_timeout_seconds }} seconds + + Connect timeout + {{ object.connect_timeout_seconds }} seconds + + + GET retries + {{ object.max_retries }} retries, {{ object.retry_backoff_seconds }}s backoff + diff --git a/pyproject.toml b/pyproject.toml index 15c0e5b..f06e11b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "netbox-vmware-importer" -version = "0.2.5" +version = "0.2.6" description = "NetBox plugin to synchronize VMware vSphere and Proxmox VE virtual machines into NetBox." readme = "README.md" requires-python = ">=3.12" @@ -23,6 +23,7 @@ dependencies = [ "pyvmomi>=8.0.3", "cryptography>=42.0", "requests>=2.32", + "proxmox-sdk==0.0.12", ] [tool.setuptools.packages.find]