Update app.py

app.py

@@ -2,6 +2,7 @@ import base64
 import os
 
 import streamlit as st
+from langchain.embeddings.openai import OpenAIEmbeddings
 from langchain.chains import RetrievalQA
 from langchain.document_loaders import PDFMinerLoader
 from langchain.embeddings import SentenceTransformerEmbeddings
@@ -36,11 +37,12 @@ def data_ingestion():
                 loader = PDFMinerLoader(os.path.join(root, file))
 
     documents = loader.load()
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=500)
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=20)
     splits = text_splitter.split_documents(documents)
 
-    # create embeddings here
-    embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
+    # create embeddings of the chunked document
+    #embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
+    embeddings = OpenAIEmbeddings()
     vectordb = FAISS.from_documents(splits, embeddings)
     vectordb.save_local("faiss_index")