Upload folder using huggingface_hub

src/document_processor.py

@@ -88,7 +88,6 @@ class ChunkMetadata:
 
 @dataclass
 class DocumentChunk:
-    """Represents a chunk of processed document content."""
     content: str
     metadata: ChunkMetadata
     embedding: Optional[List[float]] = None

src/vector_store.py

@@ -13,21 +13,9 @@ load_dotenv()
 
 
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from src.document_processor import DocumentChunk, ChunkMetadata
 try:
     from logger.custom_logger import CustomLoggerTracker
-    from src.config import Config
-    
-    from src.ingestion_pipeline import DocumentIngestionPipeline
-    from src.rag_engine import RAGEngine
-    from src.metadata_manager import MetadataManager
-    from src.document_processor import ( ChunkMetadata, 
-                                        DocumentProcessorFactory,
-                                        DocumentType,
-                                        DocumentChunk,
-                                        DocumentProcessor)
-    from src.pdf_processor import PDFProcessor
-    from src.excel_processor import ExcelProcessor
-    from src.image_processor import ImageProcessor
     custom_log = CustomLoggerTracker()
     logger = custom_log.get_logger("vector_store")
 
@@ -446,12 +434,4 @@ if __name__=="__main__":
     if health:
         logger.info("Vector store is healthy and ready.")
     else:
-        logger.error("Vector store is not accessible.")
-    
-    # Register processors
-    DocumentProcessorFactory.register_processor(DocumentType.PDF, PDFProcessor)
-    DocumentProcessorFactory.register_processor(DocumentType.EXCEL, ExcelProcessor)
-    DocumentProcessorFactory.register_processor(DocumentType.IMAGE, ImageProcessor)
-            
-    file_path = "/Users/ahmedmostafa/Downloads/eval_Korean_qa/data/documents/4.외주입고현황.xlsx"
-    document_processor = DocumentProcessor(file_path)
+        logger.error("Vector store is not accessible.")