fix: Align PDF text extraction between indexing and context expansion

This commit fixes two critical issues with PDF processing:

1. **Text extraction mismatch (context expansion bug)**:
   - Indexing used pymupdf4llm.to_markdown() producing markdown text
   - Context expansion used page.get_text() producing plain text
   - Different text formats caused character offset misalignment
   - Search would find correct chunk, but expansion showed wrong section
   - Fixed by making context.py use pymupdf4llm.to_markdown() consistently

2. **Diagnostic logging for page number assignment**:
   - Added logging to verify page_boundaries exist in metadata
   - Added logging to verify assign_page_numbers() assigns values
   - Helps diagnose why page numbers show as null in search results

3. **mime_type storage bug**:
   - Fixed incorrect field reference in processor.py:405
   - Was using file_metadata.get("content_type", "")
   - Should use content_type from WebDAV response

Changes:
- nextcloud_mcp_server/search/context.py: Use pymupdf4llm.to_markdown()
  for PDF text extraction to match indexing method
- nextcloud_mcp_server/vector/processor.py: Add diagnostic logging for
  page boundaries and assignment, fix mime_type storage
- tests/unit/client/test_webdav.py: Fix import sorting

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2025-11-20 13:57:50 +01:00
parent f1a5fac1b9
commit d67aa6ae5c
3 changed files with 188 additions and 7 deletions
+50 -1
View File
@@ -288,6 +288,30 @@ async def _index_document(
file_metadata = result.metadata
title = file_metadata.get("title") or file_path.split("/")[-1]
etag = "" # WebDAV read_file doesn't return etag
# Diagnostic: Log page boundary information if available
if "page_boundaries" in file_metadata:
page_boundaries = file_metadata["page_boundaries"]
logger.info(
f"Page boundaries for {file_path}: "
f"{len(page_boundaries)} pages, text length: {len(content)}"
)
# Log first 3 page boundaries for debugging
for boundary in page_boundaries[:3]:
logger.debug(
f" Page {boundary['page']}: "
f"offsets [{boundary['start_offset']}:{boundary['end_offset']}]"
)
# Verify last boundary matches text length
if page_boundaries:
last_boundary = page_boundaries[-1]
if last_boundary["end_offset"] != len(content):
logger.warning(
f"Text length mismatch: content={len(content)}, "
f"last_boundary_end={last_boundary['end_offset']}"
)
else:
logger.debug(f"No page_boundaries in metadata for {file_path}")
except Exception as e:
logger.error(f"Failed to process file {file_path}: {e}")
raise
@@ -305,6 +329,31 @@ async def _index_document(
if doc_task.doc_type == "file" and "page_boundaries" in file_metadata:
assign_page_numbers(chunks, file_metadata["page_boundaries"])
# Diagnostic: Verify page number assignment
assigned_count = sum(1 for c in chunks if c.page_number is not None)
logger.info(
f"Assigned page numbers to {assigned_count}/{len(chunks)} chunks "
f"for {file_path}"
)
# Log first 3 chunks to see their page assignments
for i, chunk in enumerate(chunks[:3]):
logger.debug(
f" Chunk {i}: page={chunk.page_number}, "
f"offsets=[{chunk.start_offset}:{chunk.end_offset}]"
)
# Warning if NO page numbers were assigned
if assigned_count == 0:
logger.warning(
f"NO page numbers assigned! "
f"Text length: {len(content)}, "
f"Chunks: {len(chunks)}, "
f"Chunk offset range: [{chunks[0].start_offset}:{chunks[-1].end_offset}], "
f"Page boundaries: {len(file_metadata['page_boundaries'])} pages, "
f"First boundary: {file_metadata['page_boundaries'][0] if file_metadata['page_boundaries'] else 'None'}"
)
# Extract chunk texts for embedding
chunk_texts = [chunk.text for chunk in chunks]
@@ -353,7 +402,7 @@ async def _index_document(
**(
{
"file_path": file_path, # Store file path for retrieval
"mime_type": file_metadata.get("content_type", ""),
"mime_type": content_type, # From WebDAV response
"file_size": file_metadata.get("file_size"),
"page_number": chunk.page_number,
"page_count": file_metadata.get("page_count"),