From 83b662eac376a49b2049340f690a52d73141183c Mon Sep 17 00:00:00 2001 From: pycolas <104468038+pycolas@users.noreply.github.com> Date: Thu, 30 Jun 2022 14:13:04 -0300 Subject: [PATCH] End to end connections (#113) * Add hide-device-role feature * Add hide-device-role feature * Remove end^Cevices not in queryset * clean up * Fix destination.device.id test * Query optimization * multiple updates --- README.md | 1 + netbox_topology_views/__init__.py | 2 + netbox_topology_views/forms.py | 15 +- netbox_topology_views/views.py | 362 +++++++++++++++++++----------- 4 files changed, 242 insertions(+), 138 deletions(-) diff --git a/README.md b/README.md index c091208..eb3a0a4 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ PLUGINS_CONFIG = { | ------------- |-------------| -----| | device_img |['access-switch', 'core-switch', 'firewall', 'router', 'distribution-switch', 'backup', 'storage,wan-network', 'wireless-ap', 'server', 'internal-switch', 'isp-cpe-material', 'non-racked-devices', 'power-units'] | The slug of the device roles that you have a image for. | | preselected_device_roles | ['Firewall', 'Router', 'Distribution Switch', 'Core Switch', 'Internal Switch', 'Access Switch', 'Server', 'Storage', 'Backup', 'Wireless AP'] | The full name of the device roles you want to pre select in the global view. Note that this is case sensitive| +| preselected_intermediate_dev_roles | ['Patch-panel'] | The full name of the device roles you want to display in the global view when using end-to-end connections mode. Note that this is case sensitive| | allow_coordinates_saving | False | (bool) Set to true if you use the custom coordinates fields and want to save the coordinates | | ignore_cable_type | ['power outlet', 'power port'] | The cable types that you want to ignore in the views | | preselected_tags | '[]' | The name of tags you want to preload | diff --git a/netbox_topology_views/__init__.py b/netbox_topology_views/__init__.py index d3c4ed8..98068db 100644 --- a/netbox_topology_views/__init__.py +++ b/netbox_topology_views/__init__.py @@ -11,6 +11,8 @@ class TopologyViewsConfig(PluginConfig): required_settings = [] default_settings = { 'preselected_device_roles': ['Firewall', 'Router', 'Distribution Switch', 'Core Switch', 'Internal Switch', 'Access Switch', 'Server', 'Storage', 'Backup', 'Wireless AP'], + 'preselected_intermediate_dev_roles' : [], + 'end2end_connections': False, 'ignore_cable_type': ['power outlet','power port'], 'device_img': ['access-switch', 'core-switch', 'firewall', 'router', 'distribution-switch', 'backup', 'storage', 'wan-network', 'wireless-ap', 'server', 'internal-switch', 'isp-cpe-material', 'non-racked-devices', 'power-units'], 'allow_coordinates_saving': False, diff --git a/netbox_topology_views/forms.py b/netbox_topology_views/forms.py index f8379c9..d4f9f30 100644 --- a/netbox_topology_views/forms.py +++ b/netbox_topology_views/forms.py @@ -19,10 +19,10 @@ allow_coordinates_saving = settings.PLUGINS_CONFIG["netbox_topology_views"]["all class DeviceFilterForm(TenancyFilterForm, NetBoxModelFilterSetForm): model = Device fieldsets = ( - (None, ('q', 'hide_unconnected', 'save_coords')), + (None, ('q', 'hide_unconnected', 'save_coords', 'end2end_connections')), (None, ('tenant_group_id', 'tenant_id')), (None, ('region_id', 'site_id', 'location_id')), - (None, ('device_role_id',)), + (None, ('device_role_id', 'intermediate_dev_role_id')), (None, ('tag',)), ) @@ -36,6 +36,17 @@ class DeviceFilterForm(TenancyFilterForm, NetBoxModelFilterSetForm): required=False, label=_('Device Role') ) + end2end_connections = forms.BooleanField( + label=_("Display end-to-end connections"), + required=False, + initial=False + ) + intermediate_dev_role_id = DynamicModelMultipleChoiceField( + queryset=DeviceRole.objects.all(), + required=False, + label=_('Intermediate Devices Role'), + help_text='Intermediate devices to display when using end-to-end connections mode, even if they do not match the query' + ) site_id = DynamicModelMultipleChoiceField( queryset=Site.objects.all(), required=False, diff --git a/netbox_topology_views/views.py b/netbox_topology_views/views.py index 4e8c424..ee51fe8 100644 --- a/netbox_topology_views/views.py +++ b/netbox_topology_views/views.py @@ -11,12 +11,82 @@ from .filters import DeviceFilterSet import json -from dcim.models import Device, Cable, DeviceRole, DeviceType +from dcim.models import Device, Cable, DeviceRole, PathEndpoint +from circuits.models import CircuitTermination, ProviderNetwork from extras.models import Tag -def get_topology_data(queryset, hide_unconnected): - nodes = [] - nodes_ids = [] + +def create_node(device): + dev_name = device.name + if dev_name is None: + dev_name = "device name unknown" + + node_content = "" + + if device.device_type is not None: + node_content += "Type: " + device.device_type.model + "" + if device.device_role.name is not None: + node_content += "Role: " + device.device_role.name + "" + if device.serial != "": + node_content += "Serial: " + device.serial + "" + if device.primary_ip is not None: + node_content += "IP Address: " + str(device.primary_ip.address) + "" + + dev_title = " %s
" % (node_content) + + node = {} + node["id"] = device.id + node["name"] = dev_name + node["label"] = dev_name + node["title"] = dev_title + node["shape"] = "image" + if device.device_role.slug in settings.PLUGINS_CONFIG["netbox_topology_views"]["device_img"]: + node["image"] = "../../static/netbox_topology_views/img/" + device.device_role.slug + ".png" + else: + node["image"] = "../../static/netbox_topology_views/img/role-unknown.png" + + if device.device_role.color != "": + node["color.border"] = "#" + device.device_role.color + + if "coordinates" in device.custom_field_data: + if device.custom_field_data["coordinates"] is not None: + if ";" in device.custom_field_data["coordinates"]: + cords = device.custom_field_data["coordinates"].split(";") + node["x"] = int(cords[0]) + node["y"] = int(cords[1]) + node["physics"] = False + return node + +def create_edge(edge_id, cable, termination_a, termination_b, path = None, circuit = None): + cable_a_dev_name = "device A name unknown" if termination_a.device.name is None else termination_a.device.name + cable_a_name = "device A name unknown" if termination_a.name is None else termination_a.name + cable_b_dev_name = "device A name unknown" if termination_b.device.name is None else termination_b.device.name + cable_b_name = "cable B name unknown" if termination_b.name is None else termination_b.name + + edge = {} + edge["id"] = edge_id + edge["from"] = termination_a.device.id + edge["to"] = termination_b.device.id + + if circuit is not None: + edge["dashes"] = True + edge["title"] = "Circuit provider: " + circuit.provider.name + "
" + edge["title"] += "Termination between
" + edge["title"] += cable_b_dev_name + " [" + cable_b_name + "]
" + edge["title"] += cable_a_dev_name + " [" + cable_a_name + "]" + else: + edge["title"] = "Cable between
" + cable_a_dev_name + " [" + cable_a_name + "]
" + cable_b_dev_name + " [" + cable_b_name + "]" + + if path is not None: + edge["title"] += "" if len(path) <= 0 else "
Through " + "/".join(path) + + if cable is not None and cable.color != "": + edge["color"] = "#" + cable.color + + return edge + +def get_topology_data(queryset, hide_unconnected, intermediate_dev_role_ids, end2end_connections): + nodes_devices = {} edges = [] edge_ids = 0 cable_ids = [] @@ -25,184 +95,204 @@ def get_topology_data(queryset, hide_unconnected): return None ignore_cable_type = settings.PLUGINS_CONFIG["netbox_topology_views"]["ignore_cable_type"] + enable_circuit_terminations = settings.PLUGINS_CONFIG["netbox_topology_views"]["enable_circuit_terminations"] device_ids = [d.id for d in queryset] - for qs_device in queryset: - device_has_connections = False + links = Cable.objects.filter( Q(_termination_a_device_id__in=device_ids) | Q(_termination_b_device_id__in=device_ids) ) \ + .select_related("termination_a_type", "termination_b_type") \ + .prefetch_related("termination_a", "termination_b") + + for link in links: + + if link.termination_a_type.name in ignore_cable_type or link.termination_b_type.name in ignore_cable_type \ + or link.id in cable_ids: + continue + + a_is_path_endoint = isinstance(link.termination_a, PathEndpoint) + b_is_path_endoint = isinstance(link.termination_b, PathEndpoint) + + if not end2end_connections or (not a_is_path_endoint and not b_is_path_endoint): + + if isinstance(link.termination_a, CircuitTermination): + if enable_circuit_terminations and link.termination_a.circuit.id not in circuit_ids: + circuit = link.termination_a.circuit + circuit_ids.append(circuit.id) + cable_ids.append(link.id) + + path_destination = circuit.termination_z if link.termination_a.term_side == "A" else circuit.termination_a + + if path_destination is not None and path_destination.provider_network is None: + # ProviderNetwork not supported at the moment + # $path_destination.cable would be none : there is no cable between a CircuitTermination and ProviderNetwork + origin_device = link.termination_b.device + destination_device = path_destination.cable.termination_b.device + + if origin_device.id in device_ids and destination_device.id in device_ids: + if origin_device.id not in nodes_devices: + nodes_devices[origin_device.id] = origin_device + if destination_device.id not in nodes_devices: + nodes_devices[destination_device.id] = destination_device - links_device = Cable.objects.filter(Q(_termination_a_device_id=qs_device.id) | Q(_termination_b_device_id=qs_device.id) ) - for link_from in links_device: - if link_from.termination_a_type.name != "circuit termination" and link_from.termination_b_type.name != "circuit termination": - if link_from.termination_a_type.name not in ignore_cable_type and link_from.termination_b_type.name not in ignore_cable_type: - if link_from.id not in cable_ids: - if link_from.termination_a.device.id in device_ids and link_from.termination_b.device.id in device_ids: - device_has_connections = True - cable_ids.append(link_from.id) edge_ids += 1 - cable_a_dev_name = link_from.termination_a.device.name - if cable_a_dev_name is None: - cable_a_dev_name = "device A name unknown" - cable_a_name = link_from.termination_a.name - if cable_a_name is None: - cable_a_name = "cable A name unknown" - cable_b_dev_name = link_from.termination_b.device.name - if cable_b_dev_name is None: - cable_b_dev_name = "device B name unknown" - cable_b_name = link_from.termination_b.name - if cable_b_name is None: - cable_b_name = "cable B name unknown" + edges.append(create_edge(edge_ids, link, link.termination_b, path_destination.cable.termination_b, [circuit.cid], circuit)) - edge = {} - edge["id"] = edge_ids - edge["from"] = link_from.termination_a.device.id - edge["to"] = link_from.termination_b.device.id - edge["title"] = "Cable between
" + cable_a_dev_name + " [" + cable_a_name + "]
" + cable_b_dev_name + " [" + cable_b_name + "]" - if link_from.color != "": - edge["color"] = "#" + link_from.color - edges.append(edge) - else: - if link_from.termination_a.device.id in device_ids and link_from.termination_b.device.id in device_ids: - device_has_connections = True + elif link.termination_a.device.id in device_ids and link.termination_b.device.id in device_ids: + + if link.termination_a.device.id not in nodes_devices: + nodes_devices[link.termination_a.device.id] = link.termination_a.device + if link.termination_b.device.id not in nodes_devices: + nodes_devices[link.termination_b.device.id] = link.termination_b.device + + cable_ids.append(link.id) + edge_ids += 1 + edges.append(create_edge(edge_ids, link, link.termination_a, link.termination_b)) + else: + # termination_a can be a CircuitTermination (no trace()) while termination_b is a PathEndpoint + # If so, we swap them so we can follow the path using PathEndpoint#trace() + path_start = link.termination_a if a_is_path_endoint else link.termination_b + if path_start.device is not None and path_start.device.id not in device_ids: + # device not in queryset, skip + continue + + # Ignore incomplete path when in end-to-end connection mode + if path_start.path is not None: + if not path_start.path.is_active or path_start.path.is_split: + continue else: - if settings.PLUGINS_CONFIG["netbox_topology_views"]["enable_circuit_terminations"]: - if link_from.termination_a_type.name == "circuit termination": - if link_from.termination_a.circuit.id not in circuit_ids: - circuit_ids.append(link_from.termination_a.circuit.id) - edge_ids += 1 + continue - cable_b_dev_name = link_from.termination_b.device.name - if cable_b_dev_name is None: - cable_b_dev_name = "device B name unknown" - cable_b_name = link_from.termination_b.name - if cable_b_name is None: - cable_b_name = "cable B name unknown" + path_destination = path_start.path.destination + if hasattr(path_destination, "device") and path_destination.device is not None and path_destination.device.id not in device_ids: + # device not in queryset, skip + continue - edge = {} - edge["id"] = edge_ids - edge["to"] = link_from.termination_b.device.id - edge["dashes"] = True - title = "" + valid_path = False - title += "Circuit provider: " + link_from.termination_a.circuit.provider.name + "
" - title += "Termination between
" - title += cable_b_dev_name + " [" + cable_b_name + "]
" + # variables needed for trace() browsing + origin = None + circuit = None + tmp_path = [] - if link_from.termination_a.circuit.termination_a is not None and link_from.termination_a.circuit.termination_a.cable is not None and link_from.termination_a.circuit.termination_a.cable.id != link_from.id and link_from.termination_a.circuit.termination_a.cable.termination_b is not None and link_from.termination_a.circuit.termination_a.cable.termination_b.device is not None: - edge["from"] = link_from.termination_a.circuit.termination_a.cable.termination_b.device.id + # trace() : Return the path as a list of three-tuples (A termination, cable, B termination) + # TODO device qui ne s"affichent pas si hide_unconnected + for (termination_a, cable, termination_b) in path_start.trace(): + + if origin is None: + # New part of the link + origin = termination_a + circuit = None + tmp_path = [] - cable_a_dev_name = link_from.termination_a.circuit.termination_a.cable.termination_b.device.name - if cable_a_dev_name is None: - cable_a_dev_name = "device B name unknown" - cable_b_name = link_from.termination_a.circuit.termination_a.cable.termination_b.name - if cable_a_name is None: - cable_a_name = "cable B name unknown" - title += cable_a_dev_name + " [" + cable_a_name + "]
" - edge["title"] = title - edges.append(edge) + if isinstance(termination_b, ProviderNetwork): + # ProviderNetwork not supported at the moment + # It would need to manage several kind of nodes (Device, ProviderNetwork) + # and maps javascript nodes ID with Devices IDs and ProviderNetworks IDs + # When $termination_b is a ProviderNetwork instance, $cable will be None + break - if link_from.termination_a.circuit.termination_z is not None and link_from.termination_a.circuit.termination_z.cable is not None and link_from.termination_a.circuit.termination_z.cable.id != link_from.id and link_from.termination_a.circuit.termination_z.cable.termination_b is not None and link_from.termination_a.circuit.termination_z.cable.termination_b.device is not None: - edge["from"] = link_from.termination_a.circuit.termination_z.cable.termination_b.device.id + if cable is not None and (cable.termination_a_type in ignore_cable_type or cable.termination_b_type in ignore_cable_type): + # Ignore this path + break - cable_a_dev_name = link_from.termination_a.circuit.termination_z.cable.termination_b.device.name - if cable_a_dev_name is None: - cable_a_dev_name = "device B name unknown" - cable_a_name = link_from.termination_a.circuit.termination_z.cable.termination_b.name - if cable_a_name is None: - cable_a_name = "cable B name unknown" - title += cable_a_dev_name + " [" + cable_a_name + "]
" - edge["title"] = title - edges.append(edge) + if isinstance(termination_b, CircuitTermination): + if enable_circuit_terminations: + circuit = termination_b.circuit + tmp_path.append(circuit.cid) + if circuit.id not in circuit_ids: + circuit_ids.append(circuit.id) + else: + # Ignore this path + break + elif end2end_connections and termination_b != path_destination \ + and termination_b.device.device_role.id not in intermediate_dev_role_ids \ + and termination_b.device.id not in device_ids: + # Skip this intermediate device, origin device is keep in $origin for next iteration + tmp_path.append(termination_b.device.name) + elif cable is not None and termination_b is not None: + if cable.id not in cable_ids: + # New part of the link we want to display + cable_ids.append(cable.id) + edge_ids += 1 + edges.append(create_edge(edge_ids, cable, origin, termination_b, tmp_path, circuit)) - if qs_device.id not in nodes_ids: - if hide_unconnected == None or (hide_unconnected is True and device_has_connections is True): - nodes_ids.append(qs_device.id) + if not valid_path: + valid_path = True - dev_name = qs_device.name - if dev_name is None: - dev_name = "device name unknown" + if termination_b.device.id not in nodes_devices and \ + (termination_b.device.device_role.id in intermediate_dev_role_ids or termination_b.device.id in device_ids) : + nodes_devices[termination_b.device.id] = termination_b.device + # Reset for next iteration + origin = None + # endfor (trace) - node_content = "" + if valid_path and path_start.device.id not in nodes_devices: + nodes_devices[path_start.device.id] = path_start.device - if qs_device.device_type is not None: - node_content += "Type: " + qs_device.device_type.model + "" - if qs_device.device_role.name is not None: - node_content += "Role: " + qs_device.device_role.name + "" - if qs_device.serial != "": - node_content += "Serial: " + qs_device.serial + "" - if qs_device.primary_ip is not None: - node_content += "IP Address: " + str(qs_device.primary_ip.address) + "" - - dev_title = " %s
" % (node_content) - - node = {} - node["id"] = qs_device.id - node["name"] = dev_name - node["label"] = dev_name - node["title"] = dev_title - node["shape"] = 'image' - if qs_device.device_role.slug in settings.PLUGINS_CONFIG["netbox_topology_views"]["device_img"]: - node["image"] = '../../static/netbox_topology_views/img/' + qs_device.device_role.slug + ".png" - else: - node["image"] = "../../static/netbox_topology_views/img/role-unknown.png" - - if qs_device.device_role.color != "": - node["color.border"] = "#" + qs_device.device_role.color - - if "coordinates" in qs_device.custom_field_data: - if qs_device.custom_field_data["coordinates"] is not None: - if ';' in qs_device.custom_field_data["coordinates"]: - cords = qs_device.custom_field_data["coordinates"].split(";") - node["x"] = int(cords[0]) - node["y"] = int(cords[1]) - node["physics"] = False - nodes.append(node) + # endfor (link) + + for qs_device in queryset: + if qs_device.id not in nodes_devices and not hide_unconnected: + nodes_devices[qs_device.id] = qs_device results = {} - results["nodes"] = nodes + results["nodes"] = [create_node(d) for d in nodes_devices.values()] results["edges"] = edges return results - class TopologyHomeView(PermissionRequiredMixin, View): - permission_required = ('dcim.view_site', 'dcim.view_device') + permission_required = ("dcim.view_site", "dcim.view_device") """ Show the home page """ def get(self, request): self.filterset = DeviceFilterSet - self.queryset = Device.objects.all() + self.queryset = Device.objects.all().select_related("device_type", "device_role") self.queryset = self.filterset(request.GET, self.queryset).qs topo_data = None if request.GET: - hide_unconnected = None - if 'hide_unconnected' in request.GET: + hide_unconnected = False + if "hide_unconnected" in request.GET: if request.GET["hide_unconnected"] == "on" : hide_unconnected = True - if 'draw_init' in request.GET: - if request.GET["draw_init"].lower() == 'true': - topo_data = get_topology_data(self.queryset, hide_unconnected) + if "intermediate_dev_role_id" in request.GET: + intermediate_dev_role_ids = list(map(int, request.GET.getlist("intermediate_dev_role_id"))) else: - topo_data = get_topology_data(self.queryset, hide_unconnected) + intermediate_dev_role_ids = [] + + end2end_connections = False + if "end2end_connections" in request.GET: + if request.GET["end2end_connections"] == "on" : + end2end_connections = True + + if "draw_init" in request.GET: + if request.GET["draw_init"].lower() == "true": + topo_data = get_topology_data(self.queryset, hide_unconnected, intermediate_dev_role_ids, end2end_connections) + else: + topo_data = get_topology_data(self.queryset, hide_unconnected, intermediate_dev_role_ids, end2end_connections) else: preselected_device_roles = settings.PLUGINS_CONFIG["netbox_topology_views"]["preselected_device_roles"] + preselected_intermediate_dev_roles = settings.PLUGINS_CONFIG["netbox_topology_views"]["preselected_intermediate_dev_roles"] preselected_tags = settings.PLUGINS_CONFIG["netbox_topology_views"]["preselected_tags"] - q_device_role_id = DeviceRole.objects.filter(name__in=preselected_device_roles).values_list('id', flat=True) - q_tags = Tag.objects.filter(name__in=preselected_tags).values_list('name', flat=True) + q_device_role_id = DeviceRole.objects.filter(name__in=preselected_device_roles).values_list("id", flat=True) + q_intermediate_dev_role_id = DeviceRole.objects.filter(name__in=preselected_intermediate_dev_roles).values_list("id", flat=True) + q_tags = Tag.objects.filter(name__in=preselected_tags).values_list("name", flat=True) q = QueryDict(mutable=True) - q.setlist('device_role_id', list(q_device_role_id)) - q.setlist('tag', list(q_tags)) - q['draw_init'] = settings.PLUGINS_CONFIG["netbox_topology_views"]["draw_default_layout"] + q.setlist("device_role_id", list(q_device_role_id)) + q.setlist("intermediate_dev_role_id", list(q_intermediate_dev_role_id)) + q.setlist("tag", list(q_tags)) + q["end2end_connections"] = settings.PLUGINS_CONFIG["netbox_topology_views"]["end2end_connections"] + q["draw_init"] = settings.PLUGINS_CONFIG["netbox_topology_views"]["draw_default_layout"] query_string = q.urlencode() return HttpResponseRedirect(request.path + "?" + query_string) - return render(request, 'netbox_topology_views/index.html' , { - 'filter_form': DeviceFilterForm(request.GET, label_suffix=''), - 'topology_data': json.dumps(topo_data) + return render(request, "netbox_topology_views/index.html" , { + "filter_form": DeviceFilterForm(request.GET, label_suffix=""), + "topology_data": json.dumps(topo_data) } )