feat : pass results to metadata - avoid sending too much data to LLM

sorbobotapp/app.py

@@ -160,10 +160,11 @@ with doc_column:
     if len(st.session_state.history) > 0:
         for doc in st.session_state.history[-1].documents:
             doc_content = json.loads(doc.page_content)
+            doc_metadata = doc.metadata
 
             expander = st.expander(doc_content["title"])
-            expander.markdown(f"**HalID** : https://hal.science/{doc_content['hal_id']}")
-            expander.markdown(doc_content["abstract"])
+            expander.markdown(f"**HalID** : https://hal.science/{doc_metadata['hal_id']}")
+            expander.markdown(doc_metadata["abstract"])
             expander.markdown(f"**Authors** : {doc_content['authors']}")
             expander.markdown(f"**Keywords** : {doc_content['keywords']}")
-            expander.markdown(f"**Distance** : {doc_content['distance']}")
+            expander.markdown(f"**Distance** : {doc_metadata['distance']}")

sorbobotapp/vector_store.py

@@ -229,16 +229,18 @@ class CustomVectorStore(VectorStore):
                 Document(
                     page_content=json.dumps(
                         {
-                            "abstract": result["abstract"][0],
-                            "id": result["id"],
                             "title": result["title"][0],
                             "authors": result["authors"],
-                            "doi": result["doi"],
-                            "hal_id": result["hal_id"],
                             "keywords": result["keywords"],
-                            "distance": result["distance"],
                         }
                     ),
+                    metadata={
+                        "id": result["id"],
+                        "doi": result["doi"],
+                        "hal_id": result["hal_id"],
+                        "distance": result["distance"],
+                        "abstract": result["abstract"],
+                    },
                 ),
                 result["distance"] if self.embedding_function is not None else None,
             )