+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ⬆️ Back to Top +
diff --git a/__pycache__/main.cpython-312.pyc b/__pycache__/main.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3bb8858398bb9345ef8947017430d289ef170795 Binary files /dev/null and b/__pycache__/main.cpython-312.pyc differ diff --git a/backend/__init__.py b/backend/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..870e9568c7d9a67bb88f952133124804fb322e7b --- /dev/null +++ b/backend/__init__.py @@ -0,0 +1 @@ +from multi_agents import agents \ No newline at end of file diff --git a/backend/__pycache__/__init__.cpython-312.pyc b/backend/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c928be3f62fca1fa9dc21204233a3d01ad8f02a9 Binary files /dev/null and b/backend/__pycache__/__init__.cpython-312.pyc differ diff --git a/backend/__pycache__/utils.cpython-312.pyc b/backend/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4c732e3f3b1c06c50cc1664551fdb94d579813e3 Binary files /dev/null and b/backend/__pycache__/utils.cpython-312.pyc differ diff --git a/backend/chat/__init__.py b/backend/chat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..00c51a25e6c0bfc2d1bc6e1261f4cf21b69fc239 --- /dev/null +++ b/backend/chat/__init__.py @@ -0,0 +1 @@ +from .chat import ChatAgentWithMemory \ No newline at end of file diff --git a/backend/chat/__pycache__/__init__.cpython-312.pyc b/backend/chat/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..69516bef55195d81a8e84e40561aa23d50af16f7 Binary files /dev/null and b/backend/chat/__pycache__/__init__.cpython-312.pyc differ diff --git a/backend/chat/__pycache__/chat.cpython-312.pyc b/backend/chat/__pycache__/chat.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..59291f07703bbd5d27539748b8ede6c69af12c70 Binary files /dev/null and b/backend/chat/__pycache__/chat.cpython-312.pyc differ diff --git a/backend/chat/chat.py b/backend/chat/chat.py new file mode 100644 index 0000000000000000000000000000000000000000..cb7a53abedbee0d3978d9503a6630887ac91cda4 --- /dev/null +++ b/backend/chat/chat.py @@ -0,0 +1,106 @@ +from fastapi import WebSocket +import uuid + +from gpt_researcher.utils.llm import get_llm +from gpt_researcher.memory import Memory +from gpt_researcher.config.config import Config + +from langgraph.prebuilt import create_react_agent +from langgraph.checkpoint.memory import MemorySaver + +from langchain_community.vectorstores import InMemoryVectorStore +from langchain.text_splitter import RecursiveCharacterTextSplitter +from langchain.tools import Tool, tool + +class ChatAgentWithMemory: + def __init__( + self, + report: str, + config_path, + headers, + vector_store = None + ): + self.report = report + self.headers = headers + self.config = Config(config_path) + self.vector_store = vector_store + self.graph = self.create_agent() + + def create_agent(self): + """Create React Agent Graph""" + cfg = Config() + + # Retrieve LLM using get_llm with settings from config + provider = get_llm( + llm_provider=cfg.smart_llm_provider, + model=cfg.smart_llm_model, + temperature=0.35, + max_tokens=cfg.smart_token_limit, + **self.config.llm_kwargs + ).llm + + # If vector_store is not initialized, process documents and add to vector_store + if not self.vector_store: + documents = self._process_document(self.report) + self.chat_config = {"configurable": {"thread_id": str(uuid.uuid4())}} + self.embedding = Memory( + cfg.embedding_provider, + cfg.embedding_model, + **cfg.embedding_kwargs + ).get_embeddings() + self.vector_store = InMemoryVectorStore(self.embedding) + self.vector_store.add_texts(documents) + + # Create the React Agent Graph with the configured provider + graph = create_react_agent( + provider, + tools=[self.vector_store_tool(self.vector_store)], + checkpointer=MemorySaver() + ) + + return graph + + def vector_store_tool(self, vector_store) -> Tool: + """Create Vector Store Tool""" + @tool + def retrieve_info(query): + """ + Consult the report for relevant contexts whenever you don't know something + """ + retriever = vector_store.as_retriever(k = 4) + return retriever.invoke(query) + return retrieve_info + + def _process_document(self, report): + """Split Report into Chunks""" + text_splitter = RecursiveCharacterTextSplitter( + chunk_size=1024, + chunk_overlap=20, + length_function=len, + is_separator_regex=False, + ) + documents = text_splitter.split_text(report) + return documents + + async def chat(self, message, websocket): + """Chat with React Agent""" + message = f""" + You are GPT Researcher, a autonomous research agent created by an open source community at https://github.com/assafelovic/gpt-researcher, homepage: https://gptr.dev. + To learn more about GPT Researcher you can suggest to check out: https://docs.gptr.dev. + + This is a chat message between the user and you: GPT Researcher. + The chat is about a research reports that you created. Answer based on the given context and report. + You must include citations to your answer based on the report. + + Report: {self.report} + User Message: {message} + """ + inputs = {"messages": [("user", message)]} + response = await self.graph.ainvoke(inputs, config=self.chat_config) + ai_message = response["messages"][-1].content + if websocket is not None: + await websocket.send_json({"type": "chat", "content": ai_message}) + + def get_context(self): + """return the current context of the chat""" + return self.report diff --git a/backend/memory/__init__.py b/backend/memory/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/memory/draft.py b/backend/memory/draft.py new file mode 100644 index 0000000000000000000000000000000000000000..a5e6862146d56672857e837a91b31c6b0e23b962 --- /dev/null +++ b/backend/memory/draft.py @@ -0,0 +1,10 @@ +from typing import TypedDict, List, Annotated +import operator + + +class DraftState(TypedDict): + task: dict + topic: str + draft: dict + review: str + revision_notes: str \ No newline at end of file diff --git a/backend/memory/research.py b/backend/memory/research.py new file mode 100644 index 0000000000000000000000000000000000000000..337e1f17705e8eb44e99e3fa37a390fa2e36f423 --- /dev/null +++ b/backend/memory/research.py @@ -0,0 +1,20 @@ +from typing import TypedDict, List, Annotated +import operator + + +class ResearchState(TypedDict): + task: dict + initial_research: str + sections: List[str] + research_data: List[dict] + # Report layout + title: str + headers: dict + date: str + table_of_contents: str + introduction: str + conclusion: str + sources: List[str] + report: str + + diff --git a/backend/report_type/__init__.py b/backend/report_type/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6f1e0faef859d32df2a48fbf67c99bc7b66aa1db --- /dev/null +++ b/backend/report_type/__init__.py @@ -0,0 +1,7 @@ +from .basic_report.basic_report import BasicReport +from .detailed_report.detailed_report import DetailedReport + +__all__ = [ + "BasicReport", + "DetailedReport" +] \ No newline at end of file diff --git a/backend/report_type/__pycache__/__init__.cpython-312.pyc b/backend/report_type/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7c096c84c1dff1a9618079809c4243f29139634 Binary files /dev/null and b/backend/report_type/__pycache__/__init__.cpython-312.pyc differ diff --git a/backend/report_type/basic_report/__init__.py b/backend/report_type/basic_report/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/report_type/basic_report/__pycache__/__init__.cpython-312.pyc b/backend/report_type/basic_report/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..81e7cb3045a3f9ed56d2afcef731a0bdfdfbe4a4 Binary files /dev/null and b/backend/report_type/basic_report/__pycache__/__init__.cpython-312.pyc differ diff --git a/backend/report_type/basic_report/__pycache__/basic_report.cpython-312.pyc b/backend/report_type/basic_report/__pycache__/basic_report.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..80106abafb9f41bf84ec5d940b6dcaa0f23fa87d Binary files /dev/null and b/backend/report_type/basic_report/__pycache__/basic_report.cpython-312.pyc differ diff --git a/backend/report_type/basic_report/basic_report.py b/backend/report_type/basic_report/basic_report.py new file mode 100644 index 0000000000000000000000000000000000000000..fd29c732d0b6d887869984ad8da8fcf0493c5c1c --- /dev/null +++ b/backend/report_type/basic_report/basic_report.py @@ -0,0 +1,46 @@ +from fastapi import WebSocket +from typing import Any + +from gpt_researcher import GPTResearcher + + +class BasicReport: + def __init__( + self, + query: str, + report_type: str, + report_source: str, + source_urls, + document_urls, + tone: Any, + config_path: str, + websocket: WebSocket, + headers=None + ): + self.query = query + self.report_type = report_type + self.report_source = report_source + self.source_urls = source_urls + self.document_urls = document_urls + self.tone = tone + self.config_path = config_path + self.websocket = websocket + self.headers = headers or {} + + async def run(self): + # Initialize researcher + researcher = GPTResearcher( + query=self.query, + report_type=self.report_type, + report_source=self.report_source, + source_urls=self.source_urls, + document_urls=self.document_urls, + tone=self.tone, + config_path=self.config_path, + websocket=self.websocket, + headers=self.headers + ) + + await researcher.conduct_research() + report = await researcher.write_report() + return report diff --git a/backend/report_type/detailed_report/README.md b/backend/report_type/detailed_report/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1348d7a22f4df4304990e4b57e406ae16c16a660 --- /dev/null +++ b/backend/report_type/detailed_report/README.md @@ -0,0 +1,12 @@ +## Detailed Reports + +Introducing long and detailed reports, with a completely new architecture inspired by the latest [STORM](https://arxiv.org/abs/2402.14207) paper. + +In this method we do the following: + +1. Trigger Initial GPT Researcher report based on task +2. Generate subtopics from research summary +3. For each subtopic the headers of the subtopic report are extracted and accumulated +4. For each subtopic a report is generated making sure that any information about the headers accumulated until now are not re-generated. +5. An additional introduction section is written along with a table of contents constructed from the entire report. +6. The final report is constructed by appending these : Intro + Table of contents + Subsection reports \ No newline at end of file diff --git a/backend/report_type/detailed_report/__init__.py b/backend/report_type/detailed_report/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/report_type/detailed_report/__pycache__/__init__.cpython-312.pyc b/backend/report_type/detailed_report/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e75aba000b4992347fd191acd8f398d8007fdf2c Binary files /dev/null and b/backend/report_type/detailed_report/__pycache__/__init__.cpython-312.pyc differ diff --git a/backend/report_type/detailed_report/__pycache__/detailed_report.cpython-312.pyc b/backend/report_type/detailed_report/__pycache__/detailed_report.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7f74a5e6146f32fa3b8c1149e4b8705c6588c8a9 Binary files /dev/null and b/backend/report_type/detailed_report/__pycache__/detailed_report.cpython-312.pyc differ diff --git a/backend/report_type/detailed_report/detailed_report.py b/backend/report_type/detailed_report/detailed_report.py new file mode 100644 index 0000000000000000000000000000000000000000..290f2215b8e8f808501ff1fba84a7e94b31e1b30 --- /dev/null +++ b/backend/report_type/detailed_report/detailed_report.py @@ -0,0 +1,139 @@ +import asyncio +from typing import List, Dict, Set, Optional, Any +from fastapi import WebSocket + +from gpt_researcher import GPTResearcher + + +class DetailedReport: + def __init__( + self, + query: str, + report_type: str, + report_source: str, + source_urls: List[str] = [], + document_urls: List[str] = [], + config_path: str = None, + tone: Any = "", + websocket: WebSocket = None, + subtopics: List[Dict] = [], + headers: Optional[Dict] = None + ): + self.query = query + self.report_type = report_type + self.report_source = report_source + self.source_urls = source_urls + self.document_urls = document_urls + self.config_path = config_path + self.tone = tone + self.websocket = websocket + self.subtopics = subtopics + self.headers = headers or {} + + self.gpt_researcher = GPTResearcher( + query=self.query, + report_type="research_report", + report_source=self.report_source, + source_urls=self.source_urls, + document_urls=self.document_urls, + config_path=self.config_path, + tone=self.tone, + websocket=self.websocket, + headers=self.headers + ) + self.existing_headers: List[Dict] = [] + self.global_context: List[str] = [] + self.global_written_sections: List[str] = [] + self.global_urls: Set[str] = set( + self.source_urls) if self.source_urls else set() + + async def run(self) -> str: + await self._initial_research() + subtopics = await self._get_all_subtopics() + report_introduction = await self.gpt_researcher.write_introduction() + _, report_body = await self._generate_subtopic_reports(subtopics) + self.gpt_researcher.visited_urls.update(self.global_urls) + report = await self._construct_detailed_report(report_introduction, report_body) + return report + + async def _initial_research(self) -> None: + await self.gpt_researcher.conduct_research() + self.global_context = self.gpt_researcher.context + self.global_urls = self.gpt_researcher.visited_urls + + async def _get_all_subtopics(self) -> List[Dict]: + subtopics_data = await self.gpt_researcher.get_subtopics() + + all_subtopics = [] + if subtopics_data and subtopics_data.subtopics: + for subtopic in subtopics_data.subtopics: + all_subtopics.append({"task": subtopic.task}) + else: + print(f"Unexpected subtopics data format: {subtopics_data}") + + return all_subtopics + + async def _generate_subtopic_reports(self, subtopics: List[Dict]) -> tuple: + subtopic_reports = [] + subtopics_report_body = "" + + for subtopic in subtopics: + result = await self._get_subtopic_report(subtopic) + if result["report"]: + subtopic_reports.append(result) + subtopics_report_body += f"\n\n\n{result['report']}" + + return subtopic_reports, subtopics_report_body + + async def _get_subtopic_report(self, subtopic: Dict) -> Dict[str, str]: + current_subtopic_task = subtopic.get("task") + subtopic_assistant = GPTResearcher( + query=current_subtopic_task, + report_type="subtopic_report", + report_source=self.report_source, + websocket=self.websocket, + headers=self.headers, + parent_query=self.query, + subtopics=self.subtopics, + visited_urls=self.global_urls, + agent=self.gpt_researcher.agent, + role=self.gpt_researcher.role, + tone=self.tone, + ) + + subtopic_assistant.context = list(set(self.global_context)) + await subtopic_assistant.conduct_research() + + draft_section_titles = await subtopic_assistant.get_draft_section_titles(current_subtopic_task) + + if not isinstance(draft_section_titles, str): + draft_section_titles = str(draft_section_titles) + + parse_draft_section_titles = self.gpt_researcher.extract_headers(draft_section_titles) + parse_draft_section_titles_text = [header.get( + "text", "") for header in parse_draft_section_titles] + + relevant_contents = await subtopic_assistant.get_similar_written_contents_by_draft_section_titles( + current_subtopic_task, parse_draft_section_titles_text, self.global_written_sections + ) + + subtopic_report = await subtopic_assistant.write_report(self.existing_headers, relevant_contents) + + self.global_written_sections.extend(self.gpt_researcher.extract_sections(subtopic_report)) + self.global_context = list(set(subtopic_assistant.context)) + self.global_urls.update(subtopic_assistant.visited_urls) + + self.existing_headers.append({ + "subtopic task": current_subtopic_task, + "headers": self.gpt_researcher.extract_headers(subtopic_report), + }) + + return {"topic": subtopic, "report": subtopic_report} + + async def _construct_detailed_report(self, introduction: str, report_body: str) -> str: + toc = self.gpt_researcher.table_of_contents(report_body) + conclusion = await self.gpt_researcher.write_report_conclusion(report_body) + conclusion_with_references = self.gpt_researcher.add_references( + conclusion, self.gpt_researcher.visited_urls) + report = f"{introduction}\n\n{toc}\n\n{report_body}\n\n{conclusion_with_references}" + return report diff --git a/backend/server/__init__.py b/backend/server/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/backend/server/__pycache__/__init__.cpython-312.pyc b/backend/server/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..672366d9480e7ab9c2d808a48498f8447e52ca8a Binary files /dev/null and b/backend/server/__pycache__/__init__.cpython-312.pyc differ diff --git a/backend/server/__pycache__/server.cpython-312.pyc b/backend/server/__pycache__/server.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..02aa29ab278e5ae5828332acfe20b437c6b99ee2 Binary files /dev/null and b/backend/server/__pycache__/server.cpython-312.pyc differ diff --git a/backend/server/__pycache__/server_utils.cpython-312.pyc b/backend/server/__pycache__/server_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f78d761e038a1f84d00da4e8f6bd331d102f716f Binary files /dev/null and b/backend/server/__pycache__/server_utils.cpython-312.pyc differ diff --git a/backend/server/__pycache__/websocket_manager.cpython-312.pyc b/backend/server/__pycache__/websocket_manager.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..74fd54ea92a6cf3adc771d8fd96af3127c740704 Binary files /dev/null and b/backend/server/__pycache__/websocket_manager.cpython-312.pyc differ diff --git a/backend/server/app.py b/backend/server/app.py new file mode 100644 index 0000000000000000000000000000000000000000..e15d43a8cbd0c4a4400525c291f5a8dca96e6ab0 --- /dev/null +++ b/backend/server/app.py @@ -0,0 +1,16 @@ +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +import logging + +logger = logging.getLogger(__name__) + +app = FastAPI() + +# Add CORS middleware +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], # In production, replace with your frontend domain + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) \ No newline at end of file diff --git a/backend/server/logging_config.py b/backend/server/logging_config.py new file mode 100644 index 0000000000000000000000000000000000000000..c3821273bcfe71721a707e2de56dd6e5b62d8c99 --- /dev/null +++ b/backend/server/logging_config.py @@ -0,0 +1,83 @@ +import logging +import json +import os +from datetime import datetime +from pathlib import Path + +class JSONResearchHandler: + def __init__(self, json_file): + self.json_file = json_file + self.research_data = { + "timestamp": datetime.now().isoformat(), + "events": [], + "content": { + "query": "", + "sources": [], + "context": [], + "report": "", + "costs": 0.0 + } + } + + def log_event(self, event_type: str, data: dict): + self.research_data["events"].append({ + "timestamp": datetime.now().isoformat(), + "type": event_type, + "data": data + }) + self._save_json() + + def update_content(self, key: str, value): + self.research_data["content"][key] = value + self._save_json() + + def _save_json(self): + with open(self.json_file, 'w') as f: + json.dump(self.research_data, f, indent=2) + +def setup_research_logging(): + # Create logs directory if it doesn't exist + logs_dir = Path("logs") + logs_dir.mkdir(exist_ok=True) + + # Generate timestamp for log files + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + # Create log file paths + log_file = logs_dir / f"research_{timestamp}.log" + json_file = logs_dir / f"research_{timestamp}.json" + + # Configure file handler for research logs + file_handler = logging.FileHandler(log_file) + file_handler.setLevel(logging.INFO) + file_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) + + # Get research logger and configure it + research_logger = logging.getLogger('research') + research_logger.setLevel(logging.INFO) + + # Remove any existing handlers to avoid duplicates + research_logger.handlers.clear() + + # Add file handler + research_logger.addHandler(file_handler) + + # Add stream handler for console output + console_handler = logging.StreamHandler() + console_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) + research_logger.addHandler(console_handler) + + # Prevent propagation to root logger to avoid duplicate logs + research_logger.propagate = False + + # Create JSON handler + json_handler = JSONResearchHandler(json_file) + + return str(log_file), str(json_file), research_logger, json_handler + +# Create a function to get the logger and JSON handler +def get_research_logger(): + return logging.getLogger('research') + +def get_json_handler(): + return getattr(logging.getLogger('research'), 'json_handler', None) \ No newline at end of file diff --git a/backend/server/server.py b/backend/server/server.py new file mode 100644 index 0000000000000000000000000000000000000000..6d2f3ee297771eba6e696dca81cb88c41ccd669d --- /dev/null +++ b/backend/server/server.py @@ -0,0 +1,134 @@ +import json +import os +from typing import Dict, List + +from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect, File, UploadFile, Header +from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates +from pydantic import BaseModel + +from backend.server.websocket_manager import WebSocketManager +from backend.server.server_utils import ( + get_config_dict, + update_environment_variables, handle_file_upload, handle_file_deletion, + execute_multi_agents, handle_websocket_communication +) + + +from gpt_researcher.utils.logging_config import setup_research_logging + +import logging + +# Get logger instance +logger = logging.getLogger(__name__) + +# Don't override parent logger settings +logger.propagate = True + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + handlers=[ + logging.StreamHandler() # Only log to console + ] +) + +# Models + + +class ResearchRequest(BaseModel): + task: str + report_type: str + agent: str + + +class ConfigRequest(BaseModel): + ANTHROPIC_API_KEY: str + TAVILY_API_KEY: str + LANGCHAIN_TRACING_V2: str + LANGCHAIN_API_KEY: str + OPENAI_API_KEY: str + DOC_PATH: str + RETRIEVER: str + GOOGLE_API_KEY: str = '' + GOOGLE_CX_KEY: str = '' + BING_API_KEY: str = '' + SEARCHAPI_API_KEY: str = '' + SERPAPI_API_KEY: str = '' + SERPER_API_KEY: str = '' + SEARX_URL: str = '' + XAI_API_KEY: str + DEEPSEEK_API_KEY: str + + +# App initialization +app = FastAPI() + +# Static files and templates +app.mount("/site", StaticFiles(directory="./frontend"), name="site") +app.mount("/static", StaticFiles(directory="./frontend/static"), name="static") +templates = Jinja2Templates(directory="./frontend") + +# WebSocket manager +manager = WebSocketManager() + +# Middleware +app.add_middleware( + CORSMiddleware, + allow_origins=["http://localhost:3000"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# Constants +DOC_PATH = os.getenv("DOC_PATH", "./my-docs") + +# Startup event + + +@app.on_event("startup") +def startup_event(): + os.makedirs("outputs", exist_ok=True) + app.mount("/outputs", StaticFiles(directory="outputs"), name="outputs") + os.makedirs(DOC_PATH, exist_ok=True) + + +# Routes + + +@app.get("/") +async def read_root(request: Request): + return templates.TemplateResponse("index.html", {"request": request, "report": None}) + + +@app.get("/files/") +async def list_files(): + files = os.listdir(DOC_PATH) + print(f"Files in {DOC_PATH}: {files}") + return {"files": files} + + +@app.post("/api/multi_agents") +async def run_multi_agents(): + return await execute_multi_agents(manager) + + +@app.post("/upload/") +async def upload_file(file: UploadFile = File(...)): + return await handle_file_upload(file, DOC_PATH) + + +@app.delete("/files/{filename}") +async def delete_file(filename: str): + return await handle_file_deletion(filename, DOC_PATH) + + +@app.websocket("/ws") +async def websocket_endpoint(websocket: WebSocket): + await manager.connect(websocket) + try: + await handle_websocket_communication(websocket, manager) + except WebSocketDisconnect: + await manager.disconnect(websocket) diff --git a/backend/server/server_utils.py b/backend/server/server_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b06eceb18ff794bc376ebd4533b98c64786b9e84 --- /dev/null +++ b/backend/server/server_utils.py @@ -0,0 +1,259 @@ +import json +import os +import re +import time +import shutil +from typing import Dict, List, Any +from fastapi.responses import JSONResponse, FileResponse +from gpt_researcher.document.document import DocumentLoader +from backend.utils import write_md_to_pdf, write_md_to_word, write_text_to_md +from pathlib import Path +from datetime import datetime +from fastapi import HTTPException +import logging + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + +class CustomLogsHandler: + """Custom handler to capture streaming logs from the research process""" + def __init__(self, websocket, task: str): + self.logs = [] + self.websocket = websocket + sanitized_filename = sanitize_filename(f"task_{int(time.time())}_{task}") + self.log_file = os.path.join("outputs", f"{sanitized_filename}.json") + self.timestamp = datetime.now().isoformat() + # Initialize log file with metadata + os.makedirs("outputs", exist_ok=True) + with open(self.log_file, 'w') as f: + json.dump({ + "timestamp": self.timestamp, + "events": [], + "content": { + "query": "", + "sources": [], + "context": [], + "report": "", + "costs": 0.0 + } + }, f, indent=2) + + async def send_json(self, data: Dict[str, Any]) -> None: + """Store log data and send to websocket""" + # Send to websocket for real-time display + if self.websocket: + await self.websocket.send_json(data) + + # Read current log file + with open(self.log_file, 'r') as f: + log_data = json.load(f) + + # Update appropriate section based on data type + if data.get('type') == 'logs': + log_data['events'].append({ + "timestamp": datetime.now().isoformat(), + "type": "event", + "data": data + }) + else: + # Update content section for other types of data + log_data['content'].update(data) + + # Save updated log file + with open(self.log_file, 'w') as f: + json.dump(log_data, f, indent=2) + logger.debug(f"Log entry written to: {self.log_file}") + + +class Researcher: + def __init__(self, query: str, report_type: str = "research_report"): + self.query = query + self.report_type = report_type + # Generate unique ID for this research task + self.research_id = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{hash(query)}" + # Initialize logs handler with research ID + self.logs_handler = CustomLogsHandler(self.research_id) + self.researcher = GPTResearcher( + query=query, + report_type=report_type, + websocket=self.logs_handler + ) + + async def research(self) -> dict: + """Conduct research and return paths to generated files""" + await self.researcher.conduct_research() + report = await self.researcher.write_report() + + # Generate the files + sanitized_filename = sanitize_filename(f"task_{int(time.time())}_{self.query}") + file_paths = await generate_report_files(report, sanitized_filename) + + # Get the JSON log path that was created by CustomLogsHandler + json_relative_path = os.path.relpath(self.logs_handler.log_file) + + return { + "output": { + **file_paths, # Include PDF, DOCX, and MD paths + "json": json_relative_path + } + } + +def sanitize_filename(filename: str) -> str: + # Split into components + prefix, timestamp, *task_parts = filename.split('_') + task = '_'.join(task_parts) + + # Calculate max length for task portion + # 255 - len("outputs/") - len("task_") - len(timestamp) - len("_.json") - safety_margin + max_task_length = 255 - 8 - 5 - 10 - 6 - 10 # ~216 chars for task + + # Truncate task if needed + truncated_task = task[:max_task_length] if len(task) > max_task_length else task + + # Reassemble and clean the filename + sanitized = f"{prefix}_{timestamp}_{truncated_task}" + return re.sub(r"[^\w\s-]", "", sanitized).strip() + + +async def handle_start_command(websocket, data: str, manager): + json_data = json.loads(data[6:]) + task, report_type, source_urls, document_urls, tone, headers, report_source = extract_command_data( + json_data) + + if not task or not report_type: + print("Error: Missing task or report_type") + return + + # Create logs handler with websocket and task + logs_handler = CustomLogsHandler(websocket, task) + # Initialize log content with query + await logs_handler.send_json({ + "query": task, + "sources": [], + "context": [], + "report": "" + }) + + sanitized_filename = sanitize_filename(f"task_{int(time.time())}_{task}") + + report = await manager.start_streaming( + task, + report_type, + report_source, + source_urls, + document_urls, + tone, + websocket, + headers + ) + report = str(report) + file_paths = await generate_report_files(report, sanitized_filename) + # Add JSON log path to file_paths + file_paths["json"] = os.path.relpath(logs_handler.log_file) + await send_file_paths(websocket, file_paths) + + +async def handle_human_feedback(data: str): + feedback_data = json.loads(data[14:]) # Remove "human_feedback" prefix + print(f"Received human feedback: {feedback_data}") + # TODO: Add logic to forward the feedback to the appropriate agent or update the research state + +async def handle_chat(websocket, data: str, manager): + json_data = json.loads(data[4:]) + print(f"Received chat message: {json_data.get('message')}") + await manager.chat(json_data.get("message"), websocket) + +async def generate_report_files(report: str, filename: str) -> Dict[str, str]: + pdf_path = await write_md_to_pdf(report, filename) + docx_path = await write_md_to_word(report, filename) + md_path = await write_text_to_md(report, filename) + return {"pdf": pdf_path, "docx": docx_path, "md": md_path} + + +async def send_file_paths(websocket, file_paths: Dict[str, str]): + await websocket.send_json({"type": "path", "output": file_paths}) + + +def get_config_dict( + langchain_api_key: str, openai_api_key: str, tavily_api_key: str, + google_api_key: str, google_cx_key: str, bing_api_key: str, + searchapi_api_key: str, serpapi_api_key: str, serper_api_key: str, searx_url: str +) -> Dict[str, str]: + return { + "LANGCHAIN_API_KEY": langchain_api_key or os.getenv("LANGCHAIN_API_KEY", ""), + "OPENAI_API_KEY": openai_api_key or os.getenv("OPENAI_API_KEY", ""), + "TAVILY_API_KEY": tavily_api_key or os.getenv("TAVILY_API_KEY", ""), + "GOOGLE_API_KEY": google_api_key or os.getenv("GOOGLE_API_KEY", ""), + "GOOGLE_CX_KEY": google_cx_key or os.getenv("GOOGLE_CX_KEY", ""), + "BING_API_KEY": bing_api_key or os.getenv("BING_API_KEY", ""), + "SEARCHAPI_API_KEY": searchapi_api_key or os.getenv("SEARCHAPI_API_KEY", ""), + "SERPAPI_API_KEY": serpapi_api_key or os.getenv("SERPAPI_API_KEY", ""), + "SERPER_API_KEY": serper_api_key or os.getenv("SERPER_API_KEY", ""), + "SEARX_URL": searx_url or os.getenv("SEARX_URL", ""), + "LANGCHAIN_TRACING_V2": os.getenv("LANGCHAIN_TRACING_V2", "true"), + "DOC_PATH": os.getenv("DOC_PATH", "./my-docs"), + "RETRIEVER": os.getenv("RETRIEVER", ""), + "EMBEDDING_MODEL": os.getenv("OPENAI_EMBEDDING_MODEL", "") + } + + +def update_environment_variables(config: Dict[str, str]): + for key, value in config.items(): + os.environ[key] = value + + +async def handle_file_upload(file, DOC_PATH: str) -> Dict[str, str]: + file_path = os.path.join(DOC_PATH, os.path.basename(file.filename)) + with open(file_path, "wb") as buffer: + shutil.copyfileobj(file.file, buffer) + print(f"File uploaded to {file_path}") + + document_loader = DocumentLoader(DOC_PATH) + await document_loader.load() + + return {"filename": file.filename, "path": file_path} + + +async def handle_file_deletion(filename: str, DOC_PATH: str) -> JSONResponse: + file_path = os.path.join(DOC_PATH, os.path.basename(filename)) + if os.path.exists(file_path): + os.remove(file_path) + print(f"File deleted: {file_path}") + return JSONResponse(content={"message": "File deleted successfully"}) + else: + print(f"File not found: {file_path}") + return JSONResponse(status_code=404, content={"message": "File not found"}) + + +async def execute_multi_agents(manager) -> Any: + websocket = manager.active_connections[0] if manager.active_connections else None + if websocket: + report = await run_research_task("Is AI in a hype cycle?", websocket, stream_output) + return {"report": report} + else: + return JSONResponse(status_code=400, content={"message": "No active WebSocket connection"}) + + +async def handle_websocket_communication(websocket, manager): + while True: + data = await websocket.receive_text() + if data.startswith("start"): + await handle_start_command(websocket, data, manager) + elif data.startswith("human_feedback"): + await handle_human_feedback(data) + elif data.startswith("chat"): + await handle_chat(websocket, data, manager) + else: + print("Error: Unknown command or not enough parameters provided.") + + +def extract_command_data(json_data: Dict) -> tuple: + return ( + json_data.get("task"), + json_data.get("report_type"), + json_data.get("source_urls"), + json_data.get("document_urls"), + json_data.get("tone"), + json_data.get("headers", {}), + json_data.get("report_source") + ) diff --git a/backend/server/websocket_manager.py b/backend/server/websocket_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..8a9b4c6e4844173aab30785297b8bf5b3db2a048 --- /dev/null +++ b/backend/server/websocket_manager.py @@ -0,0 +1,125 @@ +import asyncio +import datetime +from typing import Dict, List + +from fastapi import WebSocket + +from backend.report_type import BasicReport, DetailedReport +from backend.chat import ChatAgentWithMemory + +from gpt_researcher.utils.enum import ReportType, Tone +from multi_agents.main import run_research_task +from gpt_researcher.actions import stream_output # Import stream_output +from backend.server.server_utils import CustomLogsHandler + + +class WebSocketManager: + """Manage websockets""" + + def __init__(self): + """Initialize the WebSocketManager class.""" + self.active_connections: List[WebSocket] = [] + self.sender_tasks: Dict[WebSocket, asyncio.Task] = {} + self.message_queues: Dict[WebSocket, asyncio.Queue] = {} + self.chat_agent = None + + async def start_sender(self, websocket: WebSocket): + """Start the sender task.""" + queue = self.message_queues.get(websocket) + if not queue: + return + + while True: + message = await queue.get() + if websocket in self.active_connections: + try: + if message == "ping": + await websocket.send_text("pong") + else: + await websocket.send_text(message) + except: + break + else: + break + + async def connect(self, websocket: WebSocket): + """Connect a websocket.""" + await websocket.accept() + self.active_connections.append(websocket) + self.message_queues[websocket] = asyncio.Queue() + self.sender_tasks[websocket] = asyncio.create_task( + self.start_sender(websocket)) + + async def disconnect(self, websocket: WebSocket): + """Disconnect a websocket.""" + if websocket in self.active_connections: + self.active_connections.remove(websocket) + self.sender_tasks[websocket].cancel() + await self.message_queues[websocket].put(None) + del self.sender_tasks[websocket] + del self.message_queues[websocket] + + async def start_streaming(self, task, report_type, report_source, source_urls, document_urls, tone, websocket, headers=None): + """Start streaming the output.""" + tone = Tone[tone] + # add customized JSON config file path here + config_path = "default" + report = await run_agent(task, report_type, report_source, source_urls, document_urls, tone, websocket, headers = headers, config_path = config_path) + #Create new Chat Agent whenever a new report is written + self.chat_agent = ChatAgentWithMemory(report, config_path, headers) + return report + + async def chat(self, message, websocket): + """Chat with the agent based message diff""" + if self.chat_agent: + await self.chat_agent.chat(message, websocket) + else: + await websocket.send_json({"type": "chat", "content": "Knowledge empty, please run the research first to obtain knowledge"}) + +async def run_agent(task, report_type, report_source, source_urls, document_urls, tone: Tone, websocket, headers=None, config_path=""): + """Run the agent.""" + start_time = datetime.datetime.now() + + # Create logs handler for this research task + logs_handler = CustomLogsHandler(websocket, task) + + # Initialize researcher based on report type + if report_type == "multi_agents": + report = await run_research_task( + query=task, + websocket=logs_handler, # Use logs_handler instead of raw websocket + stream_output=stream_output, + tone=tone, + headers=headers + ) + report = report.get("report", "") + + elif report_type == ReportType.DetailedReport.value: + researcher = DetailedReport( + query=task, + report_type=report_type, + report_source=report_source, + source_urls=source_urls, + document_urls=document_urls, + tone=tone, + config_path=config_path, + websocket=logs_handler, # Use logs_handler instead of raw websocket + headers=headers + ) + report = await researcher.run() + + else: + researcher = BasicReport( + query=task, + report_type=report_type, + report_source=report_source, + source_urls=source_urls, + document_urls=document_urls, + tone=tone, + config_path=config_path, + websocket=logs_handler, # Use logs_handler instead of raw websocket + headers=headers + ) + report = await researcher.run() + + return report diff --git a/backend/utils.py b/backend/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5870747fdcdeb25b83937581f8a4a77e3d972b6a --- /dev/null +++ b/backend/utils.py @@ -0,0 +1,92 @@ +import aiofiles +import urllib +import mistune + +async def write_to_file(filename: str, text: str) -> None: + """Asynchronously write text to a file in UTF-8 encoding. + + Args: + filename (str): The filename to write to. + text (str): The text to write. + """ + # Ensure text is a string + if not isinstance(text, str): + text = str(text) + + # Convert text to UTF-8, replacing any problematic characters + text_utf8 = text.encode('utf-8', errors='replace').decode('utf-8') + + async with aiofiles.open(filename, "w", encoding='utf-8') as file: + await file.write(text_utf8) + +async def write_text_to_md(text: str, filename: str = "") -> str: + """Writes text to a Markdown file and returns the file path. + + Args: + text (str): Text to write to the Markdown file. + + Returns: + str: The file path of the generated Markdown file. + """ + file_path = f"outputs/{filename[:60]}.md" + await write_to_file(file_path, text) + return urllib.parse.quote(file_path) + +async def write_md_to_pdf(text: str, filename: str = "") -> str: + """Converts Markdown text to a PDF file and returns the file path. + + Args: + text (str): Markdown text to convert. + + Returns: + str: The encoded file path of the generated PDF. + """ + file_path = f"outputs/{filename[:60]}.pdf" + + try: + from md2pdf.core import md2pdf + md2pdf(file_path, + md_content=text, + # md_file_path=f"{file_path}.md", + css_file_path="./frontend/pdf_styles.css", + base_url=None) + print(f"Report written to {file_path}") + except Exception as e: + print(f"Error in converting Markdown to PDF: {e}") + return "" + + encoded_file_path = urllib.parse.quote(file_path) + return encoded_file_path + +async def write_md_to_word(text: str, filename: str = "") -> str: + """Converts Markdown text to a DOCX file and returns the file path. + + Args: + text (str): Markdown text to convert. + + Returns: + str: The encoded file path of the generated DOCX. + """ + file_path = f"outputs/{filename[:60]}.docx" + + try: + from docx import Document + from htmldocx import HtmlToDocx + # Convert report markdown to HTML + html = mistune.html(text) + # Create a document object + doc = Document() + # Convert the html generated from the report to document format + HtmlToDocx().add_html_to_document(html, doc) + + # Saving the docx document to file_path + doc.save(file_path) + + print(f"Report written to {file_path}") + + encoded_file_path = urllib.parse.quote(file_path) + return encoded_file_path + + except Exception as e: + print(f"Error in converting Markdown to DOCX: {e}") + return "" \ No newline at end of file diff --git a/citation.cff b/citation.cff new file mode 100644 index 0000000000000000000000000000000000000000..f87c4afe9a3b837df9173109532ba5f5820814e8 --- /dev/null +++ b/citation.cff @@ -0,0 +1,10 @@ +cff-version: 1.0.0 +message: "If you use this software, please cite it as below." +authors: + - family-names: Elovic + given-names: Assaf +title: gpt-researcher +version: 0.5.4 +date-released: 2023-07-23 +repository-code: https://github.com/assafelovic/gpt-researcher +url: https://gptr.dev \ No newline at end of file diff --git a/cli.py b/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..8543e38ebabd953b59fbcb1533152451832b85d0 --- /dev/null +++ b/cli.py @@ -0,0 +1,139 @@ +""" +Provides a command line interface for the GPTResearcher class. + +Usage: + +```shell +python cli.py "
+
+
+{description}
+{siteConfig.tagline}
+
+ Say Hello to GPT Researcher, your AI mate for rapid insights and comprehensive research.
+ GPT Researcher takes care of everything from accurate source gathering and organization of research results to generation of customized reports with citations.
+
An agent tailored specifically to your task + will be generated to provide the most precise and relevant research results.
+ +{questionForHuman}
+