feat: add token limit with tiktoken

app.py

@@ -58,6 +58,7 @@ def initialize_session_state():
             verbose=True,
             memory=memory,
             return_source_documents=True,
+            max_tokens_limit=3700,
         )
 
 

requirements.txt

@@ -1,11 +1,12 @@
 gpt4all==1.0.12
 langchain==0.0.313
 openai==0.28.1
+pandas==2.1.1
+pgvector==0.2.3
+psycopg2-binary==2.9.9
+psycopg2==2.9.9
 streamlit==1.27.2
 streamlit-chat==0.1.1
 SQLAlchemy==2.0.22
 sqlite-vss==0.1.2
-pandas==2.1.1
-pgvector==0.2.3
-psycopg2-binary==2.9.9
-psycopg2==2.9.9
+tiktoken==0.5.1