Rename document_loader.py to preprocess.py

document_loader.py

@@ -1,20 +0,0 @@
-import os
-from bs4 import BeautifulSoup
-import html2text
-
-def load_html_documents(folder_path):
-    documents = []
-    for file_name in os.listdir(folder_path):
-        if file_name.endswith(".html"):
-            file_path = os.path.join(folder_path, file_name)
-            with open(file_path, "r", encoding="latin-1") as file:
-                soup = BeautifulSoup(file, "html.parser")
-                text = html2text.html2text(soup.prettify())  # Convert HTML to readable text
-                documents.append({"filename": file_name, "content": text})
-    return documents
-
-# Change "your_html_folder_path" to your actual folder path
-#docs = load_html_documents("documents")
-
-# Print a sample document
-#print(docs[0])

preprocess.py

@@ -0,0 +1,24 @@
+import os
+from bs4 import BeautifulSoup
+import html2text
+
+    def load_html_documents(folder_path):
+        documents = []
+        for file_name in os.listdir(folder_path):
+            if file_name.endswith(".html"):
+                file_path = os.path.join(folder_path, file_name)
+                with open(file_path, "r", encoding="latin-1") as file:
+                    soup = BeautifulSoup(file, "html.parser")
+                    text = html2text.html2text(soup.prettify())  # Convert HTML to readable text
+                    documents.append({"filename": file_name, "content": text})
+        return documents
+
+    def preprocess_and_save(folder_path, output_path):
+        # Load documents
+        docs = load_html_documents(folder_path)
+
+        # Save preprocessed documents (you can customize the saving format)
+        # For example, save as a JSON file
+        import json
+        with open(output_path, "w") as f:
+            json.dump(docs, f)