End to end connections (#113)

* Add hide-device-role feature

* Add hide-device-role feature

* Remove end^Cevices not in queryset

* clean up

* Fix destination.device.id test

* Query optimization

* multiple updates

README.md

@@ -67,6 +67,7 @@ PLUGINS_CONFIG = {
 | ------------- |-------------| -----|
 | device_img      |['access-switch', 'core-switch', 'firewall', 'router', 'distribution-switch', 'backup', 'storage,wan-network', 'wireless-ap', 'server', 'internal-switch', 'isp-cpe-material', 'non-racked-devices', 'power-units'] | The slug of the device roles that you have a image for. |
 | preselected_device_roles      | ['Firewall', 'Router', 'Distribution Switch', 'Core Switch', 'Internal Switch', 'Access Switch', 'Server', 'Storage', 'Backup', 'Wireless AP'] | The full name of the device roles you want to pre select in the global view.  Note that this is case sensitive|
+| preselected_intermediate_dev_roles      | ['Patch-panel'] | The full name of the device roles you want to display in the global view when using end-to-end connections mode.  Note that this is case sensitive|
 | allow_coordinates_saving      | False | (bool) Set to true if you use the custom coordinates fields and want to save the coordinates |
 | ignore_cable_type      | ['power outlet', 'power port'] | The cable types that you want to ignore in the views  |
 | preselected_tags      | '[]' | The name of tags you want to preload  |

netbox_topology_views/__init__.py

@@ -11,6 +11,8 @@ class TopologyViewsConfig(PluginConfig):
     required_settings = []
     default_settings = {
         'preselected_device_roles': ['Firewall', 'Router', 'Distribution Switch', 'Core Switch', 'Internal Switch', 'Access Switch', 'Server', 'Storage', 'Backup', 'Wireless AP'],
+        'preselected_intermediate_dev_roles' : [],
+        'end2end_connections': False,
         'ignore_cable_type': ['power outlet','power port'],
         'device_img': ['access-switch', 'core-switch', 'firewall', 'router', 'distribution-switch', 'backup', 'storage', 'wan-network', 'wireless-ap', 'server', 'internal-switch', 'isp-cpe-material', 'non-racked-devices', 'power-units'],
         'allow_coordinates_saving': False,

netbox_topology_views/forms.py

@@ -19,10 +19,10 @@ allow_coordinates_saving = settings.PLUGINS_CONFIG["netbox_topology_views"]["all
 class DeviceFilterForm(TenancyFilterForm, NetBoxModelFilterSetForm):
     model = Device
     fieldsets = (
-        (None, ('q', 'hide_unconnected', 'save_coords')),
+        (None, ('q', 'hide_unconnected', 'save_coords', 'end2end_connections')),
         (None, ('tenant_group_id', 'tenant_id')),
         (None, ('region_id', 'site_id', 'location_id')),
-        (None, ('device_role_id',)),
+        (None, ('device_role_id', 'intermediate_dev_role_id')),
         (None, ('tag',)),
     )
 
@@ -36,6 +36,17 @@ class DeviceFilterForm(TenancyFilterForm, NetBoxModelFilterSetForm):
         required=False,
         label=_('Device Role')
     )
+    end2end_connections = forms.BooleanField(
+        label=_("Display end-to-end connections"),
+        required=False,
+        initial=False
+    )
+    intermediate_dev_role_id = DynamicModelMultipleChoiceField(
+        queryset=DeviceRole.objects.all(),
+        required=False,
+        label=_('Intermediate Devices Role'),
+        help_text='Intermediate devices to display when using end-to-end connections mode, even if they do not match the query'
+    )
     site_id = DynamicModelMultipleChoiceField(
         queryset=Site.objects.all(),
         required=False,

netbox_topology_views/views.py

@@ -11,12 +11,82 @@ from .filters import DeviceFilterSet
 
 import json
 
-from dcim.models import Device, Cable, DeviceRole, DeviceType
+from dcim.models import Device, Cable, DeviceRole, PathEndpoint
+from circuits.models import CircuitTermination, ProviderNetwork
 from extras.models import Tag
 
-def get_topology_data(queryset, hide_unconnected):
+
-    nodes = []
+def create_node(device):
-    nodes_ids = []
+    dev_name = device.name
+    if dev_name is None:
+        dev_name = "device name unknown"
+
+    node_content = ""
+
+    if device.device_type is not None:
+        node_content += "<tr><th>Type: </th><td>" + device.device_type.model + "</td></tr>"
+    if device.device_role.name is not None:
+        node_content +=  "<tr><th>Role: </th><td>" + device.device_role.name + "</td></tr>"
+    if device.serial != "":
+        node_content += "<tr><th>Serial: </th><td>" + device.serial + "</td></tr>"
+    if device.primary_ip is not None:
+        node_content += "<tr><th>IP Address: </th><td>" + str(device.primary_ip.address) + "</td></tr>"
+
+    dev_title = "<table> %s </table>" % (node_content)
+
+    node = {}
+    node["id"] = device.id
+    node["name"] = dev_name
+    node["label"] = dev_name
+    node["title"] = dev_title
+    node["shape"] = "image"
+    if device.device_role.slug in settings.PLUGINS_CONFIG["netbox_topology_views"]["device_img"]:
+        node["image"] = "../../static/netbox_topology_views/img/"  + device.device_role.slug + ".png"
+    else:
+        node["image"] = "../../static/netbox_topology_views/img/role-unknown.png"
+
+    if device.device_role.color != "":
+        node["color.border"] = "#" + device.device_role.color
+
+    if "coordinates" in device.custom_field_data:
+        if device.custom_field_data["coordinates"] is not None:
+            if ";" in device.custom_field_data["coordinates"]:
+                cords =  device.custom_field_data["coordinates"].split(";")
+                node["x"] = int(cords[0])
+                node["y"] = int(cords[1])
+                node["physics"] = False
+    return node
+
+def create_edge(edge_id, cable, termination_a, termination_b, path = None, circuit = None):
+    cable_a_dev_name = "device A name unknown" if termination_a.device.name is None else termination_a.device.name
+    cable_a_name = "device A name unknown" if termination_a.name is None else termination_a.name
+    cable_b_dev_name = "device A name unknown" if termination_b.device.name is None else termination_b.device.name
+    cable_b_name = "cable B name unknown" if termination_b.name is None else termination_b.name
+
+    edge = {}
+    edge["id"] = edge_id
+    edge["from"] = termination_a.device.id
+    edge["to"] = termination_b.device.id
+
+    if circuit is not None:
+        edge["dashes"] = True
+        edge["title"] = "Circuit provider: "  + circuit.provider.name + "<br>"
+        edge["title"] += "Termination between <br>"
+        edge["title"] += cable_b_dev_name + " [" + cable_b_name +  "]<br>"
+        edge["title"] += cable_a_dev_name + " [" + cable_a_name +  "]"
+    else:
+        edge["title"] = "Cable between <br> " + cable_a_dev_name + " [" + cable_a_name +  "]<br>" + cable_b_dev_name + " [" + cable_b_name + "]"
+    
+    if path is not None:
+        edge["title"] += "" if len(path) <= 0 else "<br>Through " + "/".join(path)
+        
+    if cable is not None and cable.color != "":
+        edge["color"] = "#" + cable.color
+    
+    return edge
+
+def get_topology_data(queryset, hide_unconnected, intermediate_dev_role_ids, end2end_connections):
+    nodes_devices = {}
     edges = []
     edge_ids = 0
     cable_ids = []
@@ -25,184 +95,204 @@ def get_topology_data(queryset, hide_unconnected):
         return None
 
     ignore_cable_type = settings.PLUGINS_CONFIG["netbox_topology_views"]["ignore_cable_type"]
+    enable_circuit_terminations = settings.PLUGINS_CONFIG["netbox_topology_views"]["enable_circuit_terminations"]
 
     device_ids = [d.id for d in queryset]
 
-    for qs_device in queryset:
+    links = Cable.objects.filter( Q(_termination_a_device_id__in=device_ids) | Q(_termination_b_device_id__in=device_ids) ) \
-        device_has_connections = False
+                        .select_related("termination_a_type", "termination_b_type") \
+                        .prefetch_related("termination_a", "termination_b")
+
+    for link in links:
+
+        if link.termination_a_type.name in ignore_cable_type or link.termination_b_type.name in ignore_cable_type \
+            or link.id in cable_ids:
+            continue
+        
+        a_is_path_endoint = isinstance(link.termination_a, PathEndpoint)
+        b_is_path_endoint = isinstance(link.termination_b, PathEndpoint)
+
+        if not end2end_connections or (not a_is_path_endoint and not b_is_path_endoint):
+
+            if isinstance(link.termination_a, CircuitTermination):
+                if enable_circuit_terminations and link.termination_a.circuit.id not in circuit_ids:
+                    circuit = link.termination_a.circuit
+                    circuit_ids.append(circuit.id)
+                    cable_ids.append(link.id)
+                    
+                    path_destination = circuit.termination_z if link.termination_a.term_side == "A" else circuit.termination_a
+
+                    if path_destination is not None and path_destination.provider_network is None:
+                        # ProviderNetwork not supported at the moment
+                        # $path_destination.cable would be none : there is no cable between a CircuitTermination and ProviderNetwork
+                        origin_device = link.termination_b.device
+                        destination_device = path_destination.cable.termination_b.device
+
+                        if origin_device.id in device_ids and destination_device.id in device_ids:
+                            if origin_device.id not in nodes_devices:
+                                nodes_devices[origin_device.id] = origin_device
+                            if destination_device.id not in nodes_devices:
+                                nodes_devices[destination_device.id] = destination_device
 
-        links_device = Cable.objects.filter(Q(_termination_a_device_id=qs_device.id) | Q(_termination_b_device_id=qs_device.id) )
-        for link_from in links_device:
-            if link_from.termination_a_type.name != "circuit termination" and link_from.termination_b_type.name != "circuit termination":
-                if link_from.termination_a_type.name not in ignore_cable_type and link_from.termination_b_type.name not in ignore_cable_type:
-                    if link_from.id not in cable_ids:
-                        if link_from.termination_a.device.id in device_ids and link_from.termination_b.device.id in device_ids:
-                            device_has_connections = True
-                            cable_ids.append(link_from.id)
                             edge_ids += 1
-                            cable_a_dev_name = link_from.termination_a.device.name
+                            edges.append(create_edge(edge_ids, link, link.termination_b, path_destination.cable.termination_b, [circuit.cid], circuit))
-                            if cable_a_dev_name is None:
-                                cable_a_dev_name = "device A name unknown"
-                            cable_a_name = link_from.termination_a.name
-                            if cable_a_name is None:
-                                cable_a_name = "cable A name unknown"
-                            cable_b_dev_name = link_from.termination_b.device.name
-                            if cable_b_dev_name is None:
-                                cable_b_dev_name = "device B name unknown"
-                            cable_b_name = link_from.termination_b.name
-                            if cable_b_name is None:
-                                cable_b_name = "cable B name unknown"
 
-                            edge = {}
+            elif link.termination_a.device.id in device_ids and link.termination_b.device.id in device_ids:
-                            edge["id"] = edge_ids
+
-                            edge["from"] = link_from.termination_a.device.id
+                    if link.termination_a.device.id not in nodes_devices:
-                            edge["to"] = link_from.termination_b.device.id
+                        nodes_devices[link.termination_a.device.id] = link.termination_a.device
-                            edge["title"] = "Cable between <br> " + cable_a_dev_name + " [" + cable_a_name +  "]<br>" + cable_b_dev_name + " [" + cable_b_name + "]"
+                    if link.termination_b.device.id not in nodes_devices:
-                            if link_from.color != "":
+                        nodes_devices[link.termination_b.device.id] = link.termination_b.device
-                                edge["color"] = "#" + link_from.color
+
-                            edges.append(edge)
+                    cable_ids.append(link.id)
-                    else:
+                    edge_ids += 1
-                        if link_from.termination_a.device.id in device_ids and link_from.termination_b.device.id in device_ids:
+                    edges.append(create_edge(edge_ids, link, link.termination_a, link.termination_b))
-                            device_has_connections = True
+        else:
+            # termination_a can be a CircuitTermination (no trace()) while termination_b is a PathEndpoint
+            # If so, we swap them so we can follow the path using PathEndpoint#trace()
+            path_start = link.termination_a if a_is_path_endoint else link.termination_b
+            if path_start.device is not None and path_start.device.id not in device_ids:
+                # device not in queryset, skip
+                continue
+
+            # Ignore incomplete path when in end-to-end connection mode
+            if path_start.path is not None:
+                if not path_start.path.is_active or path_start.path.is_split:
+                    continue
             else:
-                if settings.PLUGINS_CONFIG["netbox_topology_views"]["enable_circuit_terminations"]:
+                continue
-                    if link_from.termination_a_type.name == "circuit termination":
-                        if link_from.termination_a.circuit.id not in circuit_ids:
-                            circuit_ids.append(link_from.termination_a.circuit.id)
-                            edge_ids += 1
 
-                            cable_b_dev_name = link_from.termination_b.device.name
+            path_destination = path_start.path.destination
-                            if cable_b_dev_name is None:
+            if hasattr(path_destination, "device") and path_destination.device is not None and path_destination.device.id not in device_ids:
-                                cable_b_dev_name = "device B name unknown"
+                # device not in queryset, skip
-                            cable_b_name = link_from.termination_b.name
+                continue
-                            if cable_b_name is None:
-                                cable_b_name = "cable B name unknown"
 
-                            edge = {}
+            valid_path = False
-                            edge["id"] = edge_ids
-                            edge["to"] = link_from.termination_b.device.id
-                            edge["dashes"] = True
-                            title = ""
 
-                            title += "Circuit provider: "  + link_from.termination_a.circuit.provider.name + "<br>"
+            # variables needed for trace() browsing
-                            title += "Termination between <br>"
+            origin = None
-                            title += cable_b_dev_name + " [" + cable_b_name +  "]<br>"
+            circuit = None
+            tmp_path = []
 
-                            if link_from.termination_a.circuit.termination_a is not None and link_from.termination_a.circuit.termination_a.cable is not None and link_from.termination_a.circuit.termination_a.cable.id != link_from.id and link_from.termination_a.circuit.termination_a.cable.termination_b is not None and link_from.termination_a.circuit.termination_a.cable.termination_b.device is not None:
+            # trace() : Return the path as a list of three-tuples (A termination, cable, B termination)
-                                edge["from"] = link_from.termination_a.circuit.termination_a.cable.termination_b.device.id
+            # TODO device qui ne s"affichent pas si hide_unconnected
+            for (termination_a, cable, termination_b) in path_start.trace():
+                
+                if origin is None:
+                    # New part of the link
+                    origin = termination_a
+                    circuit = None
+                    tmp_path = []
 
-                                cable_a_dev_name = link_from.termination_a.circuit.termination_a.cable.termination_b.device.name
+                if isinstance(termination_b, ProviderNetwork):
-                                if cable_a_dev_name is None:
+                    # ProviderNetwork not supported at the moment
-                                    cable_a_dev_name = "device B name unknown"
+                    # It would need to manage several kind of nodes (Device, ProviderNetwork)
-                                cable_b_name = link_from.termination_a.circuit.termination_a.cable.termination_b.name
+                    # and maps javascript nodes ID with Devices IDs and ProviderNetworks IDs
-                                if cable_a_name is None:
+                    # When $termination_b is a ProviderNetwork instance, $cable will be None
-                                    cable_a_name = "cable B name unknown"
+                    break
-                                title += cable_a_dev_name + " [" + cable_a_name +  "]<br>"
-                                edge["title"] = title
-                                edges.append(edge)
 
-                            if link_from.termination_a.circuit.termination_z is not None and link_from.termination_a.circuit.termination_z.cable is not None and link_from.termination_a.circuit.termination_z.cable.id != link_from.id and link_from.termination_a.circuit.termination_z.cable.termination_b is not None and link_from.termination_a.circuit.termination_z.cable.termination_b.device is not None:
+                if cable is not None and (cable.termination_a_type in ignore_cable_type or cable.termination_b_type in ignore_cable_type):
-                                edge["from"] = link_from.termination_a.circuit.termination_z.cable.termination_b.device.id
+                    # Ignore this path
+                    break
 
-                                cable_a_dev_name = link_from.termination_a.circuit.termination_z.cable.termination_b.device.name
+                if isinstance(termination_b, CircuitTermination):
-                                if cable_a_dev_name is None:
+                    if enable_circuit_terminations:
-                                    cable_a_dev_name = "device B name unknown"
+                        circuit = termination_b.circuit
-                                cable_a_name = link_from.termination_a.circuit.termination_z.cable.termination_b.name
+                        tmp_path.append(circuit.cid)
-                                if cable_a_name is None:
+                        if circuit.id not in circuit_ids:
-                                    cable_a_name = "cable B name unknown"
+                            circuit_ids.append(circuit.id)
-                                title += cable_a_dev_name + " [" + cable_a_name +  "]<br>"
+                    else:
-                                edge["title"] = title
+                        # Ignore this path
-                                edges.append(edge)
+                        break
+                elif end2end_connections and termination_b != path_destination \
+                        and termination_b.device.device_role.id not in intermediate_dev_role_ids \
+                        and termination_b.device.id not in device_ids:
+                    # Skip this intermediate device, origin device is keep in $origin for next iteration
+                    tmp_path.append(termination_b.device.name)
+                elif cable is not None and termination_b is not None:
+                    if cable.id not in cable_ids:
+                        # New part of the link we want to display
+                        cable_ids.append(cable.id)
+                        edge_ids += 1
+                        edges.append(create_edge(edge_ids, cable, origin, termination_b, tmp_path, circuit))
 
-        if qs_device.id not in nodes_ids:
+                    if not valid_path:
-            if hide_unconnected == None or (hide_unconnected is True and device_has_connections is True): 
+                        valid_path = True
-                nodes_ids.append(qs_device.id)
 
-                dev_name = qs_device.name
+                    if termination_b.device.id not in nodes_devices and \
-                if dev_name is None:
+                      (termination_b.device.device_role.id in intermediate_dev_role_ids or termination_b.device.id in device_ids) :
-                    dev_name = "device name unknown"
+                        nodes_devices[termination_b.device.id] = termination_b.device
+                    # Reset for next iteration
+                    origin = None
+            # endfor (trace)
 
-                node_content = ""
+            if valid_path and path_start.device.id not in nodes_devices:
+                nodes_devices[path_start.device.id] = path_start.device
 
-                if qs_device.device_type is not None:
+    # endfor (link)
-                    node_content += "<tr><th>Type: </th><td>" + qs_device.device_type.model + "</td></tr>"
+    
-                if qs_device.device_role.name is not None:
+    for qs_device in queryset:
-                    node_content +=  "<tr><th>Role: </th><td>" + qs_device.device_role.name + "</td></tr>"
+        if qs_device.id not in nodes_devices and not hide_unconnected:
-                if qs_device.serial != "":
+            nodes_devices[qs_device.id] = qs_device
-                    node_content += "<tr><th>Serial: </th><td>" + qs_device.serial + "</td></tr>"
-                if qs_device.primary_ip is not None:
-                    node_content += "<tr><th>IP Address: </th><td>" + str(qs_device.primary_ip.address) + "</td></tr>"
-
-                dev_title = "<table> %s </table>" % (node_content)
-
-                node = {}
-                node["id"] = qs_device.id
-                node["name"] = dev_name
-                node["label"] = dev_name
-                node["title"] = dev_title
-                node["shape"] = 'image'
-                if qs_device.device_role.slug in settings.PLUGINS_CONFIG["netbox_topology_views"]["device_img"]:
-                    node["image"] = '../../static/netbox_topology_views/img/'  + qs_device.device_role.slug + ".png"
-                else:
-                    node["image"] = "../../static/netbox_topology_views/img/role-unknown.png"
-
-                if qs_device.device_role.color != "":
-                    node["color.border"] = "#" + qs_device.device_role.color
-
-                if "coordinates" in qs_device.custom_field_data:
-                    if qs_device.custom_field_data["coordinates"] is not None:
-                        if ';' in qs_device.custom_field_data["coordinates"]:
-                            cords =  qs_device.custom_field_data["coordinates"].split(";")
-                            node["x"] = int(cords[0])
-                            node["y"] = int(cords[1])
-                            node["physics"] = False
-                nodes.append(node)
 
     results = {}
-    results["nodes"] = nodes
+    results["nodes"] = [create_node(d) for d in nodes_devices.values()]
     results["edges"] = edges
     return results
 
-
 class TopologyHomeView(PermissionRequiredMixin, View):
-    permission_required = ('dcim.view_site', 'dcim.view_device')
+    permission_required = ("dcim.view_site", "dcim.view_device")
 
     """
     Show the home page
     """
     def get(self, request):
         self.filterset = DeviceFilterSet
-        self.queryset = Device.objects.all()
+        self.queryset = Device.objects.all().select_related("device_type", "device_role")
         self.queryset = self.filterset(request.GET, self.queryset).qs
         topo_data = None
 
         if request.GET:
-            hide_unconnected = None
+            hide_unconnected = False
-            if 'hide_unconnected' in request.GET:
+            if "hide_unconnected" in request.GET:
                 if request.GET["hide_unconnected"] == "on" :
                     hide_unconnected = True
 
-            if 'draw_init' in request.GET:
+            if "intermediate_dev_role_id" in request.GET:
-                if request.GET["draw_init"].lower() == 'true':
+                intermediate_dev_role_ids = list(map(int, request.GET.getlist("intermediate_dev_role_id")))
-                    topo_data = get_topology_data(self.queryset, hide_unconnected)
             else:
-                topo_data = get_topology_data(self.queryset, hide_unconnected)
+                intermediate_dev_role_ids = []
+
+            end2end_connections = False
+            if "end2end_connections" in request.GET:
+                if request.GET["end2end_connections"] == "on" :
+                    end2end_connections = True
+            
+            if "draw_init" in request.GET:
+                if request.GET["draw_init"].lower() == "true":
+                    topo_data = get_topology_data(self.queryset, hide_unconnected, intermediate_dev_role_ids, end2end_connections)
+            else:
+                topo_data = get_topology_data(self.queryset, hide_unconnected, intermediate_dev_role_ids, end2end_connections)
         else:
             preselected_device_roles = settings.PLUGINS_CONFIG["netbox_topology_views"]["preselected_device_roles"]
+            preselected_intermediate_dev_roles = settings.PLUGINS_CONFIG["netbox_topology_views"]["preselected_intermediate_dev_roles"]
             preselected_tags = settings.PLUGINS_CONFIG["netbox_topology_views"]["preselected_tags"]
 
-            q_device_role_id = DeviceRole.objects.filter(name__in=preselected_device_roles).values_list('id', flat=True)
+            q_device_role_id = DeviceRole.objects.filter(name__in=preselected_device_roles).values_list("id", flat=True)
-            q_tags = Tag.objects.filter(name__in=preselected_tags).values_list('name', flat=True)
+            q_intermediate_dev_role_id = DeviceRole.objects.filter(name__in=preselected_intermediate_dev_roles).values_list("id", flat=True)
+            q_tags = Tag.objects.filter(name__in=preselected_tags).values_list("name", flat=True)
 
             q = QueryDict(mutable=True)
-            q.setlist('device_role_id', list(q_device_role_id))
+            q.setlist("device_role_id", list(q_device_role_id))
-            q.setlist('tag', list(q_tags))
+            q.setlist("intermediate_dev_role_id", list(q_intermediate_dev_role_id))
-            q['draw_init'] = settings.PLUGINS_CONFIG["netbox_topology_views"]["draw_default_layout"]
+            q.setlist("tag", list(q_tags))
+            q["end2end_connections"] = settings.PLUGINS_CONFIG["netbox_topology_views"]["end2end_connections"]
+            q["draw_init"] = settings.PLUGINS_CONFIG["netbox_topology_views"]["draw_default_layout"]
             query_string = q.urlencode()
             return HttpResponseRedirect(request.path + "?" + query_string)
 
-        return render(request, 'netbox_topology_views/index.html' , {
+        return render(request, "netbox_topology_views/index.html" , {
-             'filter_form': DeviceFilterForm(request.GET, label_suffix=''),
+             "filter_form": DeviceFilterForm(request.GET, label_suffix=""),
-             'topology_data': json.dumps(topo_data)
+             "topology_data": json.dumps(topo_data)
             }
         )