llama_index: [Bug]: draw_all_possible_flow returns a blank html

Bug Description

I am trying out the example specified in https://docs.llamaindex.ai/en/stable/examples/workflow/rag/ page. Please find my code below

from llama_index.core.workflow import Event
from llama_index.core.schema import NodeWithScore


class RetrieverEvent(Event):
    """Result of running retrieval"""

    nodes: list[NodeWithScore]


class RerankEvent(Event):
    """Result of running reranking on retrieved nodes"""

    nodes: list[NodeWithScore]

from llama_index.core import SimpleDirectoryReader, VectorStoreIndex
from llama_index.core.response_synthesizers import CompactAndRefine
from llama_index.core.postprocessor.llm_rerank import LLMRerank
from llama_index.core.workflow import (
    Context,
    Workflow,
    StartEvent,
    StopEvent,
    step,
)

from llama_index.llms.groq import Groq
from llama_index.embeddings.huggingface import HuggingFaceEmbedding

class RAGWorkflow(Workflow):
    @step
    async def ingest(self, ctx: Context, ev: StartEvent) -> StopEvent | None:
        """Entry point to ingest a document, triggered by a StartEvent with `dirname`."""
        dirname = ev.get("dirname")
        if not dirname:
            return None

        documents = SimpleDirectoryReader(dirname).load_data()
        index = VectorStoreIndex.from_documents(
            documents=documents,
            embed_model=HuggingFaceEmbedding(model_name="BAAI/bge-small-en-v1.5"),
        )
        return StopEvent(result=index)

    @step
    async def retrieve(
        self, ctx: Context, ev: StartEvent
    ) -> RetrieverEvent | None:
        "Entry point for RAG, triggered by a StartEvent with `query`."
        query = ev.get("query")
        index = ev.get("index")

        if not query:
            return None

        print(f"Query the database with: {query}")

        # store the query in the global context
        await ctx.set("query", query)

        # get the index from the global context
        if index is None:
            print("Index is empty, load some documents before querying!")
            return None

        retriever = index.as_retriever(similarity_top_k=2)
        nodes = await retriever.aretrieve(query)
        print(f"Retrieved {len(nodes)} nodes.")
        return RetrieverEvent(nodes=nodes)

    @step
    async def rerank(self, ctx: Context, ev: RetrieverEvent) -> RerankEvent:
        # Rerank the nodes
        ranker = LLMRerank(
            choice_batch_size=5, top_n=3, llm=Groq(model="llama-3.1-70b-versatile")
        )
        print(await ctx.get("query", default=None), flush=True)
        new_nodes = ranker.postprocess_nodes(
            ev.nodes, query_str=await ctx.get("query", default=None)
        )
        print(f"Reranked nodes to {len(new_nodes)}")
        print(new_nodes)
        return RerankEvent(nodes=new_nodes)

    @step
    async def synthesize(self, ctx: Context, ev: RerankEvent) -> StopEvent:
        """Return a streaming response using reranked nodes."""
        llm=Groq(model="llama-3.1-70b-versatile")
        summarizer = CompactAndRefine(llm=llm, streaming=True, verbose=True)
        query = await ctx.get("query", default=None)

        response = await summarizer.asynthesize(query, nodes=ev.nodes)
        return StopEvent(result=response)

from llama_index.utils.workflow import (
    draw_all_possible_flows,
    draw_most_recent_execution,
)

# # Draw all
draw_all_possible_flows(RAGWorkflow, filename="multi_step_workflow.html")

# Draw an execution
w = RAGWorkflow()

# Ingest the documents
index = await w.run(dirname="Data")
result = await w.run(query="What is Fibromyalgia?", index=index)
async for chunk in result.async_response_gen():
    print(chunk, end="", flush=True)
draw_most_recent_execution(w, filename="rag_flow_recent.html")

Balnk html is returned

import IPython IPython.display.HTML(filename=‘/content/rag_flow_recent.html’)

Content of HTML file

<html>
    <head>
        <meta charset="utf-8">
        
            <script src="lib/bindings/utils.js"></script>
            <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/vis-network/9.1.2/dist/dist/vis-network.min.css" integrity="sha512-WgxfT5LWjfszlPHXRmBWHkV2eceiWTOBvrKCNbdgDYTHrT2AeLCGbF4sZlZw3UMN3WtL0tGUoIAKsu8mllg/XA==" crossorigin="anonymous" referrerpolicy="no-referrer" />
            <script src="https://cdnjs.cloudflare.com/ajax/libs/vis-network/9.1.2/dist/vis-network.min.js" integrity="sha512-LnvoEWDFrqGHlHmDD2101OrLcbsfkrzoSpvtSQtxK3RMnRV0eOkhhBN2dXHKRrUU8p2DGRTk35n4O8nWSVe1mQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
            
        
<center>
<h1></h1>
</center>

<!-- <link rel="stylesheet" href="../node_modules/vis/dist/vis.min.css" type="text/css" />
<script type="text/javascript" src="../node_modules/vis/dist/vis.js"> </script>-->
        <link
          href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta3/dist/css/bootstrap.min.css"
          rel="stylesheet"
          integrity="sha384-eOJMYsd53ii+scO/bJGFsiCZc+5NDVN2yr8+0RDqr0Ql0h+rP48ckxlpbzKgwra6"
          crossorigin="anonymous"
        />
        <script
          src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta3/dist/js/bootstrap.bundle.min.js"
          integrity="sha384-JEW9xMcG8R+pH31jmWH6WWP0WintQrMb4s7ZOdauHnUtxwoG2vI5DkLtS3qm9Ekf"
          crossorigin="anonymous"
        ></script>


        <center>
          <h1></h1>
        </center>
        <style type="text/css">

             #mynetwork {
                 width: 100%;
                 height: 750px;
                 background-color: #ffffff;
                 border: 1px solid lightgray;
                 position: relative;
                 float: left;
             }

             

             

             
        </style>
    </head>


    <body>
        <div class="card" style="width: 100%">
            
            
            <div id="mynetwork" class="card-body"></div>
        </div>

        
        

        <script type="text/javascript">

              // initialize global variables.
              var edges;
              var nodes;
              var allNodes;
              var allEdges;
              var nodeColors;
              var originalNodes;
              var network;
              var container;
              var options, data;
              var filter = {
                  item : '',
                  property : '',
                  value : []
              };

              

              

              // This method is responsible for drawing the graph, returns the drawn network
              function drawGraph() {
                  var container = document.getElementById('mynetwork');

                  

                  // parsing and collecting nodes and edges from the python
                  nodes = new vis.DataSet([{"color": "#FFA07A", "id": "StopEvent", "label": "StopEvent", "shape": "ellipse"}, {"color": "#ADD8E6", "id": "_done", "label": "_done", "shape": "box"}, {"color": "#ADD8E6", "id": "ingest", "label": "ingest", "shape": "box"}, {"color": "#E27AFF", "id": "StartEvent", "label": "StartEvent", "shape": "ellipse"}, {"color": "#ADD8E6", "id": "rerank", "label": "rerank", "shape": "box"}, {"color": "#90EE90", "id": "RetrieverEvent", "label": "RetrieverEvent", "shape": "ellipse"}, {"color": "#ADD8E6", "id": "retrieve", "label": "retrieve", "shape": "box"}, {"color": "#ADD8E6", "id": "synthesize", "label": "synthesize", "shape": "box"}, {"color": "#90EE90", "id": "RerankEvent", "label": "RerankEvent", "shape": "ellipse"}]);
                  edges = new vis.DataSet([{"arrows": "to", "from": "StopEvent", "to": "_done"}, {"arrows": "to", "from": "StopEvent", "to": "_done"}, {"arrows": "to", "from": "ingest", "to": "StopEvent"}, {"arrows": "to", "from": "StartEvent", "to": "ingest"}, {"arrows": "to", "from": "rerank", "to": "RerankEvent"}, {"arrows": "to", "from": "RetrieverEvent", "to": "rerank"}, {"arrows": "to", "from": "retrieve", "to": "RetrieverEvent"}, {"arrows": "to", "from": "StartEvent", "to": "retrieve"}, {"arrows": "to", "from": "synthesize", "to": "StopEvent"}, {"arrows": "to", "from": "RerankEvent", "to": "synthesize"}]);

                  nodeColors = {};
                  allNodes = nodes.get({ returnType: "Object" });
                  for (nodeId in allNodes) {
                    nodeColors[nodeId] = allNodes[nodeId].color;
                  }
                  allEdges = edges.get({ returnType: "Object" });
                  // adding nodes and edges to the graph
                  data = {nodes: nodes, edges: edges};

                  var options = {
    "configure": {
        "enabled": false
    },
    "edges": {
        "color": {
            "inherit": true
        },
        "smooth": {
            "enabled": true,
            "type": "dynamic"
        }
    },
    "interaction": {
        "dragNodes": true,
        "hideEdgesOnDrag": false,
        "hideNodesOnDrag": false
    },
    "physics": {
        "enabled": true,
        "stabilization": {
            "enabled": true,
            "fit": true,
            "iterations": 1000,
            "onlyDynamicEdges": false,
            "updateInterval": 50
        }
    }
};

                  


                  

                  network = new vis.Network(container, data, options);

                  

                  

                  


                  

                  return network;

              }
              drawGraph();
        </script>
    </body>
</html>
Please help me with some directives to visualize the workflow

Version

llama_index.core==0.11.8

Steps to Reproduce

draw_all_possible_flows(RAGWorkflow, filename=“multi_step_workflow.html”)

Relevant Logs/Tracbacks

No response

About this issue

  • Original URL
  • State: closed
  • Created 2 years ago
  • Reactions: 3
  • Comments: 27

Most upvoted comments

Your observations highlight a few key points regarding compatibility and functionality across different Python versions and environments. Here are some detailed insights and recommendations based on your observations:

1. Python 3.11.0 Compatibility

  • Observation: The workflow and visualization work correctly with Python 3.11.0.
  • Recommendation: Python 3.11.0 supports the | operator for type annotations, which simplifies type hinting. If possible, use Python 3.11.0 or later for development to leverage these features and avoid compatibility issues.

2. Python 3.9.12 Compatibility

  • Observation: The | operator does not work in Python 3.9.12, and you had to modify num_of_events in utils.py.
  • Recommendation:
    • Type Hinting: Use Union from the typing module instead of the | operator for type annotations in Python 3.9.12.
    • Validation Fix: Modifying num_of_events directly in utils.py is a workaround but not ideal. Instead, ensure that each step function has exactly one parameter annotated with the Event type to pass the validation.

3. Visualization in Google Colab

  • Observation: Visualization is not working in Google Colab even with notebook=True.
  • Recommendation:
    • Ensure Proper Installation: Make sure all required packages are installed in Colab.
      !pip install pyvis llama-index-core llama-index-llms-openai llama-index-embeddings-openai llama-index-readers-file llama-index-utils-workflow
      
    • Check HTML Rendering: Use the following approach to render the HTML content in Colab:
      from pyvis.network import Network
      from IPython.core.display import display, HTML
      
      def draw_all_possible_flows(
          workflow: Workflow,
          filename: str = "workflow_all_flows.html",
          notebook: bool = True,  # Set notebook to True
      ) -> None:
          """Draws all possible flows of the workflow."""
          net = Network(directed=True, height="750px", width="100%")
      
          # Add the nodes + edge for stop events
          net.add_node(
              StopEvent.__name__,
              label=StopEvent.__name__,
              color="#FFA07A",
              shape="ellipse",
          )
          net.add_node("_done", label="_done", color="#ADD8E6", shape="box")
          net.add_edge(StopEvent.__name__, "_done")
      
          # Add nodes from all steps
          steps = get_steps_from_class(workflow)
          if not steps:
              # If no steps are defined in the class, try to get them from the instance
              steps = get_steps_from_instance(workflow)
      
          step_config: Optional[StepConfig] = None
          for step_name, step_func in steps.items():
              step_config = getattr(step_func, "__step_config", None)
              if step_config is None:
                  continue
      
              net.add_node(
                  step_name, label=step_name, color="#ADD8E6", shape="box"
              )  # Light blue for steps
      
              for event_type in step_config.accepted_events:
                  net.add_node(
                      event_type.__name__,
                      label=event_type.__name__,
                      color="#90EE90" if event_type != StartEvent else "#E27AFF",
                      shape="ellipse",
                  )  # Light green for events
      
          # Add edges from all steps
          for step_name, step_func in steps.items():
              step_config = getattr(step_func, "__step_config", None)
      
              if step_config is None:
                  continue
      
              for return_type in step_config.return_types:
                  if return_type != type(None):
                      net.add_edge(step_name, return_type.__name__)
      
              for event_type in step_config.accepted_events:
                  net.add_edge(event_type.__name__, step_name)
      
          if notebook:
              net.show(filename, notebook=True)
              with open(filename, "r") as file:
                  display(HTML(file.read()))
          else:
              net.show(filename)
      
      # Example usage in Google Colab
      draw_all_possible_flows(
          RAGWorkflow, filename="multi_step_workflow.html", notebook=True
      )
      

Summary

  • Python 3.11.0: Works well with the | operator and the workflow visualization.
  • Python 3.9.12: Use Union for type annotations and ensure proper event parameter annotations to avoid modifying utils.py.
  • Google Colab: Ensure all dependencies are installed and use the provided method to render HTML content.

By following these recommendations, you should be able to achieve consistent functionality across different environments and Python versions.

To continue talking to Dosu, mention @dosu.

@dosu

did as instructructed but still encounter

AttributeError: 'NoneType' object has no attribute 'render'

To resolve the AttributeError: 'NoneType' object has no attribute 'render' when trying to visualize the RAGWorkflow using draw_all_possible_flows with notebook=True in Google Colab, you need to ensure that the pyvis library is properly configured to render within a Jupyter notebook environment.

Here is an example of how you can modify your code to work in Google Colab:

from pyvis.network import Network
from IPython.core.display import display, HTML

def draw_all_possible_flows(
    workflow: Workflow,
    filename: str = "workflow_all_flows.html",
    notebook: bool = True,  # Set notebook to True
) -> None:
    """Draws all possible flows of the workflow."""
    net = Network(directed=True, height="750px", width="100%")

    # Add the nodes + edge for stop events
    net.add_node(
        StopEvent.__name__,
        label=StopEvent.__name__,
        color="#FFA07A",
        shape="ellipse",
    )
    net.add_node("_done", label="_done", color="#ADD8E6", shape="box")
    net.add_edge(StopEvent.__name__, "_done")

    # Add nodes from all steps
    steps = get_steps_from_class(workflow)
    if not steps:
        # If no steps are defined in the class, try to get them from the instance
        steps = get_steps_from_instance(workflow)

    step_config: Optional[StepConfig] = None
    for step_name, step_func in steps.items():
        step_config = getattr(step_func, "__step_config", None)
        if step_config is None:
            continue

        net.add_node(
            step_name, label=step_name, color="#ADD8E6", shape="box"
        )  # Light blue for steps

        for event_type in step_config.accepted_events:
            net.add_node(
                event_type.__name__,
                label=event_type.__name__,
                color="#90EE90" if event_type != StartEvent else "#E27AFF",
                shape="ellipse",
            )  # Light green for events

    # Add edges from all steps
    for step_name, step_func in steps.items():
        step_config = getattr(step_func, "__step_config", None)

        if step_config is None:
            continue

        for return_type in step_config.return_types:
            if return_type != type(None):
                net.add_edge(step_name, return_type.__name__)

        for event_type in step_config.accepted_events:
            net.add_edge(event_type.__name__, step_name)

    if notebook:
        net.show(filename, notebook=True)
        with open(filename, "r") as file:
            display(HTML(file.read()))
    else:
        net.show(filename)

# Example usage in Google Colab
draw_all_possible_flows(
    RAGWorkflow, filename="multi_step_workflow.html", notebook=True
)

This code ensures that the visualization is properly rendered within the Google Colab notebook by using the notebook=True parameter in the net.show method and displaying the HTML content using IPython.display.HTML. Additionally, make sure to install the necessary dependencies as shown in the example:

!pip install pyvis llama-index-core llama-index-llms-openai llama-index-embeddings-openai llama-index-readers-file llama-index-utils-workflow

By following these steps, you should be able to visualize the RAGWorkflow correctly in Google Colab [1][2][3].

To continue talking to Dosu, mention @dosu.