Update preprocess.py

preprocess.py

@@ -1,24 +1,23 @@
 import os
 from bs4 import BeautifulSoup
 import html2text
+import json
 
-    def load_html_documents(folder_path):
-        documents = []
-        for file_name in os.listdir(folder_path):
-            if file_name.endswith(".html"):
-                file_path = os.path.join(folder_path, file_name)
-                with open(file_path, "r", encoding="latin-1") as file:
-                    soup = BeautifulSoup(file, "html.parser")
-                    text = html2text.html2text(soup.prettify())  # Convert HTML to readable text
-                    documents.append({"filename": file_name, "content": text})
-        return documents
+def load_html_documents(folder_path):
+    documents = []
+    for file_name in os.listdir(folder_path):
+        if file_name.endswith(".html"):
+            file_path = os.path.join(folder_path, file_name)
+            with open(file_path, "r", encoding="latin-1") as file:
+                soup = BeautifulSoup(file, "html.parser")
+                text = html2text.html2text(soup.prettify())
+                documents.append({"filename": file_name, "content": text})
+    return documents
 
-    def preprocess_and_save(folder_path, output_path):
-        # Load documents
-        docs = load_html_documents(folder_path)
+def preprocess_and_save(folder_path, output_path):
+    docs = load_html_documents(folder_path)
+    with open(output_path, "w") as f:
+        json.dump(docs, f)
 
-        # Save preprocessed documents (you can customize the saving format)
-        # For example, save as a JSON file
-        import json
-        with open(output_path, "w") as f:
-            json.dump(docs, f)
+if __name__ == "__main__":
+    preprocess_and_save("./documents", "preprocessed_data.json")  # Update path