Create create_database.py

create_database.py

@@ -0,0 +1,24 @@
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import Chroma
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+import json
+
+    def create_vector_database(input_path, persist_directory):
+        # Load preprocessed data
+        with open(input_path, "r") as f:
+            docs = json.load(f)
+
+        # Load an embedding model
+        embedding_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+
+        # Split text into smaller chunks
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)
+        all_texts = [chunk for doc in docs for chunk in text_splitter.split_text(doc["content"])]
+
+        # Create a ChromaDB vector store
+        vector_db = Chroma.from_texts(texts=all_texts, embedding=embedding_model, persist_directory=persist_directory)
+
+        print("Vector database created successfully!")
+
+    if __name__ == "__main__":
+        create_vector_database("preprocessed_data.json", "db")  # Change paths as needed