refactor: Transform document parsing into pluggable processor architecture
Refactors PR #190's hardcoded Unstructured.io integration into a flexible, extensible plugin system supporting multiple text extraction engines. - **`DocumentProcessor` ABC**: Abstract interface for all processors - **`ProcessorRegistry`**: Central registry for discovery and routing - **`ProcessingResult`**: Standardized output format across processors - **`UnstructuredProcessor`**: Refactored from `UnstructuredClient` - **`TesseractProcessor`**: Local OCR for images (lightweight alternative) - **`CustomHTTPProcessor`**: Generic wrapper for custom HTTP APIs - New `get_document_processor_config()` returns structured config - Supports enabling/disabling individual processors - Per-processor configuration via environment variables - **Breaking Change**: `ENABLE_UNSTRUCTURED_PARSING` replaced with: - `ENABLE_DOCUMENT_PROCESSING=true/false` (master switch) - `ENABLE_UNSTRUCTURED=true/false` (per-processor) - `ENABLE_TESSERACT=true/false` - `ENABLE_CUSTOM_PROCESSOR=true/false` - `parse_document()` now uses `ProcessorRegistry` - Auto-selects appropriate processor based on MIME type - Processor priority system (Unstructured=10, Tesseract=5, Custom=1) - `initialize_document_processors()` registers processors at startup - Integrated into both BasicAuth and OAuth lifespans - Graceful degradation if processors fail to initialize ```env ENABLE_DOCUMENT_PROCESSING=false ENABLE_UNSTRUCTURED=false UNSTRUCTURED_API_URL=http://unstructured:8000 UNSTRUCTURED_STRATEGY=auto # auto|fast|hi_res UNSTRUCTURED_LANGUAGES=eng,deu ENABLE_TESSERACT=false TESSERACT_LANG=eng ENABLE_CUSTOM_PROCESSOR=false CUSTOM_PROCESSOR_URL=http://localhost:9000/process CUSTOM_PROCESSOR_TYPES=application/pdf,image/jpeg ``` - **Removed**: `tests/test_unstructured_config.py` (legacy tests) - **Added**: `tests/unit/test_document_processor_config.py` - 7 unit tests for new config system - Tests individual and multi-processor configurations - **Added**: - `nextcloud_mcp_server/document_processors/__init__.py` - `nextcloud_mcp_server/document_processors/base.py` - `nextcloud_mcp_server/document_processors/registry.py` - `nextcloud_mcp_server/document_processors/unstructured.py` - `nextcloud_mcp_server/document_processors/tesseract.py` - `nextcloud_mcp_server/document_processors/custom_http.py` - `tests/unit/test_document_processor_config.py` - **Modified**: - `nextcloud_mcp_server/config.py` - New plugin config system - `nextcloud_mcp_server/app.py` - Processor initialization - `nextcloud_mcp_server/utils/document_parser.py` - Uses registry - `nextcloud_mcp_server/server/webdav.py` - Import updates - `env.sample` - New configuration format - `docker-compose.yml` - (profile changes from previous work) - **Removed**: - `nextcloud_mcp_server/client/unstructured_client.py` - Replaced by UnstructuredProcessor - `tests/test_unstructured_config.py` - Replaced with new tests ✅ **Extensible**: Add processors without modifying core code ✅ **Testable**: Mock processors for unit tests ✅ **Configurable**: Enable only needed processors ✅ **Flexible**: Choose fast (Tesseract) vs accurate (Unstructured) ✅ **Opt-in**: Disabled by default, no mandatory dependencies Users upgrading from PR #190 need to update environment variables: ```bash ENABLE_UNSTRUCTURED_PARSING=true ENABLE_DOCUMENT_PROCESSING=true ENABLE_UNSTRUCTURED=true ``` 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
@@ -1,172 +0,0 @@
|
||||
"""Unit tests for Unstructured API configuration."""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from nextcloud_mcp_server.client.unstructured_client import UnstructuredClient
|
||||
from nextcloud_mcp_server.config import (
|
||||
get_unstructured_languages,
|
||||
get_unstructured_strategy,
|
||||
)
|
||||
|
||||
|
||||
class TestUnstructuredStrategy:
|
||||
"""Test strategy configuration."""
|
||||
|
||||
def test_strategy_default(self):
|
||||
"""Test that strategy defaults to 'auto'."""
|
||||
os.environ.pop("UNSTRUCTURED_STRATEGY", None)
|
||||
assert get_unstructured_strategy() == "auto"
|
||||
|
||||
def test_strategy_custom_auto(self):
|
||||
"""Test custom strategy 'auto'."""
|
||||
os.environ["UNSTRUCTURED_STRATEGY"] = "auto"
|
||||
try:
|
||||
assert get_unstructured_strategy() == "auto"
|
||||
finally:
|
||||
os.environ.pop("UNSTRUCTURED_STRATEGY", None)
|
||||
|
||||
def test_strategy_custom_fast(self):
|
||||
"""Test custom strategy 'fast'."""
|
||||
os.environ["UNSTRUCTURED_STRATEGY"] = "fast"
|
||||
try:
|
||||
assert get_unstructured_strategy() == "fast"
|
||||
finally:
|
||||
os.environ.pop("UNSTRUCTURED_STRATEGY", None)
|
||||
|
||||
def test_strategy_custom_hi_res(self):
|
||||
"""Test custom strategy 'hi_res'."""
|
||||
os.environ["UNSTRUCTURED_STRATEGY"] = "hi_res"
|
||||
try:
|
||||
assert get_unstructured_strategy() == "hi_res"
|
||||
finally:
|
||||
os.environ.pop("UNSTRUCTURED_STRATEGY", None)
|
||||
|
||||
def test_strategy_invalid_fallback(self, caplog):
|
||||
"""Test that invalid strategy falls back to 'hi_res'."""
|
||||
import logging
|
||||
|
||||
os.environ["UNSTRUCTURED_STRATEGY"] = "invalid_strategy"
|
||||
try:
|
||||
# Ensure logging is captured at WARNING level
|
||||
with caplog.at_level(logging.WARNING):
|
||||
strategy = get_unstructured_strategy()
|
||||
assert strategy == "hi_res"
|
||||
assert "Invalid UNSTRUCTURED_STRATEGY" in caplog.text
|
||||
finally:
|
||||
os.environ.pop("UNSTRUCTURED_STRATEGY", None)
|
||||
|
||||
def test_strategy_case_insensitive(self):
|
||||
"""Test that strategy is case-insensitive."""
|
||||
os.environ["UNSTRUCTURED_STRATEGY"] = "HI_RES"
|
||||
try:
|
||||
assert get_unstructured_strategy() == "hi_res"
|
||||
finally:
|
||||
os.environ.pop("UNSTRUCTURED_STRATEGY", None)
|
||||
|
||||
|
||||
class TestUnstructuredLanguages:
|
||||
"""Test language configuration."""
|
||||
|
||||
def test_languages_default(self):
|
||||
"""Test that languages default to English and German."""
|
||||
os.environ.pop("UNSTRUCTURED_LANGUAGES", None)
|
||||
assert get_unstructured_languages() == ["eng", "deu"]
|
||||
|
||||
def test_languages_single(self):
|
||||
"""Test single language configuration."""
|
||||
os.environ["UNSTRUCTURED_LANGUAGES"] = "eng"
|
||||
try:
|
||||
assert get_unstructured_languages() == ["eng"]
|
||||
finally:
|
||||
os.environ.pop("UNSTRUCTURED_LANGUAGES", None)
|
||||
|
||||
def test_languages_multiple(self):
|
||||
"""Test multiple languages configuration."""
|
||||
os.environ["UNSTRUCTURED_LANGUAGES"] = "eng,fra,spa"
|
||||
try:
|
||||
assert get_unstructured_languages() == ["eng", "fra", "spa"]
|
||||
finally:
|
||||
os.environ.pop("UNSTRUCTURED_LANGUAGES", None)
|
||||
|
||||
def test_languages_whitespace_trimming(self):
|
||||
"""Test that whitespace is trimmed from language codes."""
|
||||
os.environ["UNSTRUCTURED_LANGUAGES"] = "eng, deu , fra "
|
||||
try:
|
||||
assert get_unstructured_languages() == ["eng", "deu", "fra"]
|
||||
finally:
|
||||
os.environ.pop("UNSTRUCTURED_LANGUAGES", None)
|
||||
|
||||
def test_languages_empty_fallback(self, caplog):
|
||||
"""Test that empty languages string falls back to default."""
|
||||
import logging
|
||||
|
||||
os.environ["UNSTRUCTURED_LANGUAGES"] = ""
|
||||
try:
|
||||
with caplog.at_level(logging.WARNING):
|
||||
languages = get_unstructured_languages()
|
||||
assert languages == ["eng", "deu"]
|
||||
assert "No languages specified" in caplog.text
|
||||
finally:
|
||||
os.environ.pop("UNSTRUCTURED_LANGUAGES", None)
|
||||
|
||||
def test_languages_only_whitespace_fallback(self, caplog):
|
||||
"""Test that whitespace-only string falls back to default."""
|
||||
import logging
|
||||
|
||||
os.environ["UNSTRUCTURED_LANGUAGES"] = " , , "
|
||||
try:
|
||||
with caplog.at_level(logging.WARNING):
|
||||
languages = get_unstructured_languages()
|
||||
assert languages == ["eng", "deu"]
|
||||
assert "No languages specified" in caplog.text
|
||||
finally:
|
||||
os.environ.pop("UNSTRUCTURED_LANGUAGES", None)
|
||||
|
||||
|
||||
class TestUnstructuredClientConfiguration:
|
||||
"""Test that UnstructuredClient respects configuration."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_uses_default_strategy(self):
|
||||
"""Test that client uses default strategy from environment."""
|
||||
os.environ.pop("UNSTRUCTURED_STRATEGY", None)
|
||||
os.environ["UNSTRUCTURED_API_URL"] = "http://test:8000"
|
||||
|
||||
try:
|
||||
_client = UnstructuredClient()
|
||||
# The partition_document method should use get_unstructured_strategy() when strategy is None
|
||||
# We can't test the actual call without a running API, but we can verify the config is read
|
||||
assert get_unstructured_strategy() == "auto"
|
||||
finally:
|
||||
os.environ.pop("UNSTRUCTURED_API_URL", None)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_uses_default_languages(self):
|
||||
"""Test that client uses default languages from environment."""
|
||||
os.environ.pop("UNSTRUCTURED_LANGUAGES", None)
|
||||
os.environ["UNSTRUCTURED_API_URL"] = "http://test:8000"
|
||||
|
||||
try:
|
||||
_client = UnstructuredClient()
|
||||
# The partition_document method should use get_unstructured_languages() when languages is None
|
||||
assert get_unstructured_languages() == ["eng", "deu"]
|
||||
finally:
|
||||
os.environ.pop("UNSTRUCTURED_API_URL", None)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_uses_custom_configuration(self):
|
||||
"""Test that client uses custom configuration from environment."""
|
||||
os.environ["UNSTRUCTURED_STRATEGY"] = "hi_res"
|
||||
os.environ["UNSTRUCTURED_LANGUAGES"] = "eng,fra,spa"
|
||||
os.environ["UNSTRUCTURED_API_URL"] = "http://test:8000"
|
||||
|
||||
try:
|
||||
_client = UnstructuredClient()
|
||||
assert get_unstructured_strategy() == "hi_res"
|
||||
assert get_unstructured_languages() == ["eng", "fra", "spa"]
|
||||
finally:
|
||||
os.environ.pop("UNSTRUCTURED_STRATEGY", None)
|
||||
os.environ.pop("UNSTRUCTURED_LANGUAGES", None)
|
||||
os.environ.pop("UNSTRUCTURED_API_URL", None)
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Unit tests for document processor configuration."""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class TestDocumentProcessorConfig:
|
||||
"""Test document processor configuration system."""
|
||||
|
||||
def test_config_disabled_by_default(self):
|
||||
"""Test that document processing is disabled by default."""
|
||||
from nextcloud_mcp_server.config import get_document_processor_config
|
||||
|
||||
os.environ.pop("ENABLE_DOCUMENT_PROCESSING", None)
|
||||
config = get_document_processor_config()
|
||||
assert config["enabled"] is False
|
||||
|
||||
def test_config_enabled(self):
|
||||
"""Test enabling document processing."""
|
||||
from nextcloud_mcp_server.config import get_document_processor_config
|
||||
|
||||
os.environ["ENABLE_DOCUMENT_PROCESSING"] = "true"
|
||||
try:
|
||||
config = get_document_processor_config()
|
||||
assert config["enabled"] is True
|
||||
finally:
|
||||
os.environ.pop("ENABLE_DOCUMENT_PROCESSING", None)
|
||||
|
||||
def test_unstructured_processor_config(self):
|
||||
"""Test Unstructured processor configuration."""
|
||||
from nextcloud_mcp_server.config import get_document_processor_config
|
||||
|
||||
os.environ["ENABLE_UNSTRUCTURED"] = "true"
|
||||
os.environ["UNSTRUCTURED_API_URL"] = "http://test:8000"
|
||||
os.environ["UNSTRUCTURED_STRATEGY"] = "hi_res"
|
||||
os.environ["UNSTRUCTURED_LANGUAGES"] = "eng,fra"
|
||||
os.environ["UNSTRUCTURED_TIMEOUT"] = "60"
|
||||
|
||||
try:
|
||||
config = get_document_processor_config()
|
||||
assert "unstructured" in config["processors"]
|
||||
unst_config = config["processors"]["unstructured"]
|
||||
assert unst_config["api_url"] == "http://test:8000"
|
||||
assert unst_config["strategy"] == "hi_res"
|
||||
assert unst_config["languages"] == ["eng", "fra"]
|
||||
assert unst_config["timeout"] == 60
|
||||
finally:
|
||||
os.environ.pop("ENABLE_UNSTRUCTURED", None)
|
||||
os.environ.pop("UNSTRUCTURED_API_URL", None)
|
||||
os.environ.pop("UNSTRUCTURED_STRATEGY", None)
|
||||
os.environ.pop("UNSTRUCTURED_LANGUAGES", None)
|
||||
os.environ.pop("UNSTRUCTURED_TIMEOUT", None)
|
||||
|
||||
def test_tesseract_processor_config(self):
|
||||
"""Test Tesseract processor configuration."""
|
||||
from nextcloud_mcp_server.config import get_document_processor_config
|
||||
|
||||
os.environ["ENABLE_TESSERACT"] = "true"
|
||||
os.environ["TESSERACT_LANG"] = "eng+deu"
|
||||
os.environ["TESSERACT_CMD"] = "/usr/local/bin/tesseract"
|
||||
|
||||
try:
|
||||
config = get_document_processor_config()
|
||||
assert "tesseract" in config["processors"]
|
||||
tess_config = config["processors"]["tesseract"]
|
||||
assert tess_config["lang"] == "eng+deu"
|
||||
assert tess_config["tesseract_cmd"] == "/usr/local/bin/tesseract"
|
||||
finally:
|
||||
os.environ.pop("ENABLE_TESSERACT", None)
|
||||
os.environ.pop("TESSERACT_LANG", None)
|
||||
os.environ.pop("TESSERACT_CMD", None)
|
||||
|
||||
def test_custom_processor_config(self):
|
||||
"""Test custom processor configuration."""
|
||||
from nextcloud_mcp_server.config import get_document_processor_config
|
||||
|
||||
os.environ["ENABLE_CUSTOM_PROCESSOR"] = "true"
|
||||
os.environ["CUSTOM_PROCESSOR_NAME"] = "my_ocr"
|
||||
os.environ["CUSTOM_PROCESSOR_URL"] = "http://localhost:9000/process"
|
||||
os.environ["CUSTOM_PROCESSOR_API_KEY"] = "secret"
|
||||
os.environ["CUSTOM_PROCESSOR_TIMEOUT"] = "30"
|
||||
os.environ["CUSTOM_PROCESSOR_TYPES"] = "application/pdf,image/jpeg"
|
||||
|
||||
try:
|
||||
config = get_document_processor_config()
|
||||
assert "custom" in config["processors"]
|
||||
custom_config = config["processors"]["custom"]
|
||||
assert custom_config["name"] == "my_ocr"
|
||||
assert custom_config["api_url"] == "http://localhost:9000/process"
|
||||
assert custom_config["api_key"] == "secret"
|
||||
assert custom_config["timeout"] == 30
|
||||
assert "application/pdf" in custom_config["supported_types"]
|
||||
assert "image/jpeg" in custom_config["supported_types"]
|
||||
finally:
|
||||
os.environ.pop("ENABLE_CUSTOM_PROCESSOR", None)
|
||||
os.environ.pop("CUSTOM_PROCESSOR_NAME", None)
|
||||
os.environ.pop("CUSTOM_PROCESSOR_URL", None)
|
||||
os.environ.pop("CUSTOM_PROCESSOR_API_KEY", None)
|
||||
os.environ.pop("CUSTOM_PROCESSOR_TIMEOUT", None)
|
||||
os.environ.pop("CUSTOM_PROCESSOR_TYPES", None)
|
||||
|
||||
def test_multiple_processors(self):
|
||||
"""Test configuration with multiple processors enabled."""
|
||||
from nextcloud_mcp_server.config import get_document_processor_config
|
||||
|
||||
os.environ["ENABLE_DOCUMENT_PROCESSING"] = "true"
|
||||
os.environ["ENABLE_UNSTRUCTURED"] = "true"
|
||||
os.environ["ENABLE_TESSERACT"] = "true"
|
||||
|
||||
try:
|
||||
config = get_document_processor_config()
|
||||
assert config["enabled"] is True
|
||||
assert "unstructured" in config["processors"]
|
||||
assert "tesseract" in config["processors"]
|
||||
finally:
|
||||
os.environ.pop("ENABLE_DOCUMENT_PROCESSING", None)
|
||||
os.environ.pop("ENABLE_UNSTRUCTURED", None)
|
||||
os.environ.pop("ENABLE_TESSERACT", None)
|
||||
|
||||
def test_default_processor_selection(self):
|
||||
"""Test default processor configuration."""
|
||||
from nextcloud_mcp_server.config import get_document_processor_config
|
||||
|
||||
os.environ.pop("DOCUMENT_PROCESSOR", None)
|
||||
config = get_document_processor_config()
|
||||
assert config["default_processor"] == "unstructured"
|
||||
|
||||
os.environ["DOCUMENT_PROCESSOR"] = "tesseract"
|
||||
try:
|
||||
config = get_document_processor_config()
|
||||
assert config["default_processor"] == "tesseract"
|
||||
finally:
|
||||
os.environ.pop("DOCUMENT_PROCESSOR", None)
|
||||
Reference in New Issue
Block a user