brandon/nextcloud-mcp-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Centralize PDF processing and generate separate images per chunk

Browse Source 
Previously, pymupdf4llm.to_markdown() was called twice - once in
PyMuPDFProcessor during indexing and again in PDFHighlighter during
visualization. Different image path lengths caused different character
offsets, leading to highlighted pages not matching their chunks.

Also fixed issue where all chunks on the same page showed all highlights
instead of just their own highlight. Now restores original page contents
between chunks using xref stream caching.

Changes:
- Add PDFHighlighter class requiring pre-computed page_boundaries and
  full_text from document processor (no fallback extraction)
- Pass pre-computed data from processor to highlighter
- Extract page-relative portion of chunk text for cross-page chunks
- Add bounding box highlighting using text anchor search
- Run highlight generation in parallel with embedding/BM25
- Cache and restore page contents to isolate highlights per chunk

Results: Highlighting success rate improved from 51% to 95% (121/128).

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
2025-11-22 02:42:28 +01:00
parent a62a007c87
commit fffe483c02
8 changed files with 1211 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nextcloud_mcp_server/auth/static/vector-viz.css
+27
View File
@@ -190,3 +190,30 @@
color: var(--color-text-maxcontrast);
font-style: italic;
}
/* PDF highlighted image styles */
.chunk-image-container {
margin-bottom: 16px;
border: 1px solid var(--color-border);
border-radius: var(--border-radius);
overflow: hidden;
background: #fff;
}
.chunk-image-header {
background: var(--color-background-dark);
padding: 8px 12px;
font-size: 12px;
font-weight: 500;
color: var(--color-text-maxcontrast);
border-bottom: 1px solid var(--color-border);
font-family: var(--font-face);
}
.chunk-highlighted-image {
display: block;
max-width: 100%;
height: auto;
cursor: zoom-in;
}
.chunk-highlighted-image:hover {
opacity: 0.95;
}
nextcloud_mcp_server/auth/templates/vector_viz.html
+14
View File
@@ -147,6 +147,20 @@
</template>
<template x-if="!chunkLoading[`${result.doc_type}_${result.id}_${result.chunk_start_offset || 0}`]">
<div>
<!-- Highlighted page image for PDFs -->
<template x-if="expandedChunks[`${result.doc_type}_${result.id}_${result.chunk_start_offset || 0}`]?.highlighted_page_image">
<div class="chunk-image-container">
<div class="chunk-image-header">
<span>Page <span x-text="expandedChunks[`${result.doc_type}_${result.id}_${result.chunk_start_offset || 0}`]?.page_number"></span></span>
</div>
<img
:src="'data:image/png;base64,' + expandedChunks[`${result.doc_type}_${result.id}_${result.chunk_start_offset || 0}`]?.highlighted_page_image"
:alt="'Page ' + expandedChunks[`${result.doc_type}_${result.id}_${result.chunk_start_offset || 0}`]?.page_number"
class="chunk-highlighted-image"
/>
</div>
</template>
<!-- Text context -->
<template x-if="expandedChunks[`${result.doc_type}_${result.id}_${result.chunk_start_offset || 0}`]?.has_more_before">
<span class="chunk-ellipsis">...</span>
</template>
nextcloud_mcp_server/auth/viz_routes.py
+65 -10
View File
@@ -564,17 +564,72 @@ async def chunk_context_endpoint(request: Request) -> JSONResponse:
f"after_len={len(chunk_context.after_context)}"
)
# For PDF files, also fetch the highlighted page image from Qdrant
highlighted_page_image = None
page_number = None
if doc_type == "file":
try:
from qdrant_client.models import FieldCondition, Filter, MatchValue
settings = get_settings()
qdrant_client = await get_qdrant_client()
username = request.user.display_name
# Query for this specific chunk's highlighted image
points_response = await qdrant_client.scroll(
collection_name=settings.get_collection_name(),
scroll_filter=Filter(
must=[
get_placeholder_filter(),
FieldCondition(
key="doc_id", match=MatchValue(value=doc_id_int)
),
FieldCondition(
key="user_id", match=MatchValue(value=username)
),
FieldCondition(
key="chunk_start_offset", match=MatchValue(value=start)
),
FieldCondition(
key="chunk_end_offset", match=MatchValue(value=end)
),
]
),
limit=1,
with_vectors=False,
with_payload=["highlighted_page_image", "page_number"],
)
points = points_response[0]
if points and points[0].payload:
highlighted_page_image = points[0].payload.get(
"highlighted_page_image"
)
page_number = points[0].payload.get("page_number")
if highlighted_page_image:
logger.info(
f"Found highlighted image for chunk: "
f"page={page_number}, image_size={len(highlighted_page_image)}"
)
except Exception as e:
logger.warning(f"Failed to fetch highlighted image: {e}")
# Return response compatible with frontend expectations
return JSONResponse(
{
"success": True,
"chunk_text": chunk_context.chunk_text,
"before_context": chunk_context.before_context,
"after_context": chunk_context.after_context,
"has_more_before": chunk_context.has_before_truncation,
"has_more_after": chunk_context.has_after_truncation,
}
)
response_data: dict = {
"success": True,
"chunk_text": chunk_context.chunk_text,
"before_context": chunk_context.before_context,
"after_context": chunk_context.after_context,
"has_more_before": chunk_context.has_before_truncation,
"has_more_after": chunk_context.has_after_truncation,
}
# Add image data if available
if highlighted_page_image:
response_data["highlighted_page_image"] = highlighted_page_image
response_data["page_number"] = page_number
return JSONResponse(response_data)
except ValueError as e:
logger.error(f"Invalid parameter format: {e}")
nextcloud_mcp_server/document_processors/pymupdf.py
+2 -2
View File
@@ -102,7 +102,7 @@ class PyMuPDFProcessor(DocumentProcessor):
await progress_callback(0, 100, "Processing PDF in background thread")
# Run CPU-bound PDF processing in thread pool to avoid blocking event loop
result = await anyio.to_thread.run_sync(
result = await anyio.to_thread.run_sync( # type: ignore[attr-defined]
self._process_sync,
content,
filename,
@@ -240,7 +240,7 @@ class PyMuPDFProcessor(DocumentProcessor):
# Basic document info
metadata["page_count"] = doc.page_count
metadata["format"] = "PDF 1." + str(
doc.pdf_version() if hasattr(doc, "pdf_version") else "?"
doc.pdf_version() if hasattr(doc, "pdf_version") else "?" # type: ignore[call-non-callable]
)
if filename:
nextcloud_mcp_server/embedding/bm25_provider.py
+1 -1
View File
@@ -66,7 +66,7 @@ class BM25SparseEmbeddingProvider:
import anyio
# Run CPU-bound BM25 encoding in thread pool to avoid blocking event loop
sparse_embeddings = await anyio.to_thread.run_sync(
sparse_embeddings = await anyio.to_thread.run_sync( # type: ignore[attr-defined]
lambda: list(self.model.embed(texts))
)
nextcloud_mcp_server/search/pdf_highlighter.py
+824
View File
@@ -0,0 +1,824 @@
"""PDF chunk highlighting utilities for vector visualization.
This module provides utilities to generate highlighted page images showing
matched chunks and their context from semantic search results.
The highlighting uses character offsets to precisely locate chunks within
PDF documents, ensuring accurate highlighting even when text formatting
varies between indexing and rendering.
"""
import logging
import re
from typing import Optional
import pymupdf
import pymupdf4llm
logger = logging.getLogger(__name__)
class PDFHighlighter:
"""Generate highlighted page images from PDF chunks."""
# Color definitions (RGB, 0-1 range)
COLORS = {
"yellow": [1, 1, 0],
"red": [1, 0, 0],
"green": [0, 1, 0],
"blue": [0, 0, 1],
"orange": [1, 0.5, 0],
"pink": [1, 0, 1],
"gray": [0.7, 0.7, 0.7],
"light_blue": [0.7, 0.9, 1.0],
"light_green": [0.7, 1.0, 0.7],
}
@staticmethod
def strip_markdown(text: str) -> str:
"""Remove markdown formatting to improve search accuracy.
Args:
text: Text with potential markdown formatting
Returns:
Plain text with markdown removed
"""
# Remove bold/italic markers
text = re.sub(r"\*\*(.+?)\*\*", r"\1", text)
text = re.sub(r"\*(.+?)\*", r"\1", text)
text = re.sub(r"__(.+?)__", r"\1", text)
text = re.sub(r"_(.+?)_", r"\1", text)
# Remove headers
text = re.sub(r"^#+\s+", "", text, flags=re.MULTILINE)
# Remove inline code
text = re.sub(r"`(.+?)`", r"\1", text)
return text.strip()
@staticmethod
def extract_pdf_text_with_boundaries(
pdf_doc: pymupdf.Document,
) -> tuple[str, list[dict]]:
"""Extract full document text with page boundary tracking.
Uses pymupdf4llm.to_markdown() for consistency with indexing.
IMPORTANT: Must use write_images=True to match PyMuPDFProcessor behavior!
Even though we don't need the images, we need the image references in the
markdown text to maintain consistent character offsets with indexing.
Args:
pdf_doc: Open PyMuPDF document
Returns:
Tuple of (full_text, page_boundaries) where page_boundaries is a list of:
{"page": 1, "start_offset": 0, "end_offset": 1234}
"""
import tempfile
from pathlib import Path
page_boundaries = []
text_parts = []
current_offset = 0
# Use temp directory for image output (images are discarded after extraction)
temp_dir = Path(tempfile.mkdtemp(prefix="pdf_highlight_"))
for page_idx in range(pdf_doc.page_count):
page_md = pymupdf4llm.to_markdown(
pdf_doc,
pages=[page_idx],
write_images=True, # Must match indexing! Otherwise offsets misalign
image_path=temp_dir,
page_chunks=False,
)
page_boundaries.append(
{
"page": page_idx + 1, # 1-indexed
"start_offset": current_offset,
"end_offset": current_offset + len(page_md),
}
)
text_parts.append(page_md)
current_offset += len(page_md)
full_text = "".join(text_parts)
# Clean up temp directory and extracted images
import shutil
try:
shutil.rmtree(temp_dir)
except Exception as e:
logger.warning(f"Failed to clean up temp directory {temp_dir}: {e}")
return full_text, page_boundaries
@staticmethod
def find_chunk_page(
chunk_start_offset: int,
chunk_end_offset: int,
page_boundaries: list[dict],
) -> Optional[dict]:
"""Find which page contains the most of a given chunk.
Args:
chunk_start_offset: Chunk start position in full document
chunk_end_offset: Chunk end position in full document
page_boundaries: Page boundary list from extract_pdf_text_with_boundaries()
Returns:
Dict with keys: page_num, overlap_chars, page_relative_start, page_relative_end
or None if chunk not found on any page
"""
chunk_pages = []
for boundary in page_boundaries:
page_start = boundary["start_offset"]
page_end = boundary["end_offset"]
# Check if chunk overlaps with this page
if chunk_start_offset < page_end and chunk_end_offset > page_start:
overlap_start = max(chunk_start_offset, page_start)
overlap_end = min(chunk_end_offset, page_end)
overlap_chars = overlap_end - overlap_start
chunk_pages.append(
{
"page_num": boundary["page"],
"overlap_chars": overlap_chars,
"page_relative_start": overlap_start - page_start,
"page_relative_end": overlap_end - page_start,
}
)
if not chunk_pages:
return None
# Return page with maximum overlap
return max(chunk_pages, key=lambda p: p["overlap_chars"])
@staticmethod
def highlight_chunk_by_word_positions(
page: pymupdf.Page,
chunk_text: str,
color: str = "yellow",
search_region: tuple[float, float, float, float] | None = None,
) -> int:
"""Highlight chunk using word-position matching.
This method matches words from the chunk to their positions on the PDF page,
avoiding text search mismatches between markdown-formatted text and raw PDF text.
Args:
page: PyMuPDF page object
chunk_text: Text to highlight (may contain markdown)
color: Color name from COLORS dict
search_region: Optional (x0, y0, x1, y1) bounding box to constrain search.
If provided, only words within this region are considered.
Returns:
Number of highlight rectangles added
"""
# Tokenize chunk into words (alphanumeric only, lowercase)
chunk_words = re.findall(
r"\w+", PDFHighlighter.strip_markdown(chunk_text).lower()
)
if not chunk_words:
logger.warning("No words found in chunk text")
return 0
# Get all words from page with positions
# Format: (x0, y0, x1, y1, "word", block_no, line_no, word_no)
try:
page_words = page.get_text("words")
except Exception as e:
logger.error(f"Failed to extract words from page: {e}")
return 0
if not page_words:
logger.warning("No words found on page")
return 0
# Filter words by search region if provided
if search_region:
rx0, ry0, rx1, ry1 = search_region
# Allow some tolerance (10 points) for words near region boundary
tolerance = 10
page_words = [
w
for w in page_words
if (
w[0] >= rx0 - tolerance
and w[2] <= rx1 + tolerance
and w[1] >= ry0 - tolerance
and w[3] <= ry1 + tolerance
)
]
logger.debug(
f"Filtered to {len(page_words)} words in region "
f"({rx0:.0f}, {ry0:.0f}, {rx1:.0f}, {ry1:.0f})"
)
if not page_words:
logger.warning("No words found in search region")
return 0
# Find matching word sequence - use FIRST match, not longest
# This ensures we highlight the actual chunk location, not similar text elsewhere
matches = []
# Build a simple word-to-positions index for the first few chunk words
# to find candidate starting positions
first_chunk_word = chunk_words[0] if chunk_words else ""
candidate_starts = []
for i, pw in enumerate(page_words):
page_word = pw[4].lower()
# Check if this could be the start of the chunk
if (
first_chunk_word == page_word
or first_chunk_word in page_word
or page_word in first_chunk_word
):
candidate_starts.append(i)
# Try each candidate start position and take the FIRST good match
for start_pos in candidate_starts:
current_matches = []
chunk_idx = 0
skip_count = 0
max_skips = 3 # Allow some formatting differences
for page_idx in range(start_pos, len(page_words)):
if chunk_idx >= len(chunk_words):
break
page_word = page_words[page_idx][4].lower()
chunk_word = chunk_words[chunk_idx]
# Check for match (allow partial matches for flexibility)
if (
chunk_word == page_word
or chunk_word in page_word
or page_word in chunk_word
):
current_matches.append(page_words[page_idx])
chunk_idx += 1
skip_count = 0
elif skip_count < max_skips:
# Allow skipping some words (formatting, punctuation)
skip_count += 1
continue
else:
break
# Accept if we matched at least 50% of chunk words
if len(current_matches) >= len(chunk_words) * 0.5:
matches = current_matches
logger.debug(
f"Found match at position {start_pos}: "
f"{len(matches)}/{len(chunk_words)} words"
)
break # Take FIRST match, not best/longest
if not matches:
logger.debug(f"No word matches found (chunk has {len(chunk_words)} words)")
return 0
logger.debug(
f"Matched {len(matches)} words out of {len(chunk_words)} chunk words"
)
# Build rectangles from matched words
rects = [pymupdf.Rect(w[0], w[1], w[2], w[3]) for w in matches]
# Check if matches are contiguous (not scattered across the page)
# Scattered matches indicate false positives from common words
if len(rects) > 1:
# Sort by vertical position then horizontal
sorted_matches = sorted(matches, key=lambda w: (round(w[1]), w[0]))
# Check for large vertical gaps (more than ~2 lines apart)
# A typical line height is 12-20 points
max_line_gap = 50 # Points - allows for ~2-3 lines gap
prev_y = sorted_matches[0][1]
large_gaps = 0
for match in sorted_matches[1:]:
y_gap = match[1] - prev_y
if y_gap > max_line_gap:
large_gaps += 1
prev_y = match[1]
# If matches are scattered (many large gaps), reject this match
# A chunk should be mostly contiguous text
if large_gaps > len(matches) * 0.3: # More than 30% have gaps
logger.debug(
f"Rejecting scattered matches: {large_gaps} large gaps "
f"out of {len(matches)} matches"
)
return 0
# Merge adjacent rectangles on the same line for cleaner highlighting
merged_rects = []
sorted_rects = sorted(rects, key=lambda r: (round(r.y0), r.x0))
current_rect = None
for rect in sorted_rects:
if current_rect is None:
current_rect = rect
elif abs(rect.y0 - current_rect.y0) < 5: # Same line (within 5 points)
current_rect = current_rect | rect # Union
else:
merged_rects.append(current_rect)
current_rect = rect
if current_rect:
merged_rects.append(current_rect)
# Add highlights
rgb = PDFHighlighter.COLORS.get(color, PDFHighlighter.COLORS["yellow"])
for rect in merged_rects:
highlight = page.add_highlight_annot(rect)
highlight.set_colors({"stroke": rgb})
highlight.set_info(
content="Chunk from semantic search",
title="PDF Highlighter (word-position)",
)
highlight.update()
return len(merged_rects)
@staticmethod
def find_unique_phrase(
text: str, min_len: int = 30, max_len: int = 80
) -> str | None:
"""Find a relatively unique phrase from text for location search.
Looks for phrases that are likely to be unique on the page:
- Prefers phrases with numbers or special terms
- Avoids very common words
Args:
text: Source text to extract phrase from
min_len: Minimum phrase length
max_len: Maximum phrase length
Returns:
A phrase likely to be unique, or None if not found
"""
clean_text = PDFHighlighter.strip_markdown(text).strip()
if not clean_text:
return None
# Try first sentence (often unique due to context)
sentences = re.split(r"[.!?]\s+", clean_text)
for sentence in sentences:
sentence = sentence.strip()
if min_len <= len(sentence) <= max_len:
return sentence
elif len(sentence) > max_len:
return sentence[:max_len]
# Fallback: first N chars
if len(clean_text) >= min_len:
return clean_text[:max_len]
return clean_text if clean_text else None
@staticmethod
def highlight_chunk_on_page(
page: pymupdf.Page,
chunk_text: str,
color: str = "yellow",
page_relative_start: int | None = None,
page_relative_end: int | None = None,
page_text_length: int | None = None,
) -> int:
"""Add bounding box highlight to a PDF page for the given chunk text.
Uses text search to find the chunk's location on the page, then draws
a bounding box around that region. Falls back to character offset estimation
if text search fails.
Args:
page: PyMuPDF page object
chunk_text: Text to highlight (may contain markdown)
color: Color name from COLORS dict
page_relative_start: Character offset where chunk starts on page (optional)
page_relative_end: Character offset where chunk ends on page (optional)
page_text_length: Total character length of page text (optional)
Returns:
Number of highlights added (1 for bounding box, 0 if failed)
"""
page_rect = page.rect
rgb = PDFHighlighter.COLORS.get(color, PDFHighlighter.COLORS["yellow"])
# Strip markdown for searching
search_text = PDFHighlighter.strip_markdown(chunk_text)
# Try to find chunk location using text search
# Search for progressively shorter phrases until we find a match
anchor_rect = None
search_phrases = []
# Build search phrases from chunk text
sentences = re.split(r"[.!?]\s+", search_text)
for sentence in sentences[:3]: # Try first 3 sentences
sentence = sentence.strip()
if len(sentence) >= 20:
search_phrases.append(sentence[:80])
if len(sentence) >= 40:
search_phrases.append(sentence[:40])
# Also try first N characters
if len(search_text) >= 30:
search_phrases.append(search_text[:60])
search_phrases.append(search_text[:30])
for phrase in search_phrases:
if not phrase:
continue
rects = page.search_for(phrase.strip())
if rects:
anchor_rect = rects[0] # Use first match
logger.debug(f"Found chunk anchor using phrase: '{phrase[:30]}...'")
break
if not anchor_rect:
page_num = page.number + 1 if page.number is not None else "unknown"
logger.warning(f"Could not find chunk text on page {page_num}")
return 0
# Calculate chunk height based on character count
# Estimate ~15 chars per line, ~12pt line height
chunk_chars = len(search_text)
estimated_lines = max(1, chunk_chars / 60) # ~60 chars per line typical
estimated_height = estimated_lines * 14 # ~14pt per line
# Build bounding box starting from anchor
chunk_rect = pymupdf.Rect(
page_rect.x0 + 30, # Left margin
anchor_rect.y0 - 5, # Start slightly above anchor
page_rect.x1 - 30, # Right margin
min(
anchor_rect.y0 + estimated_height + 10, page_rect.y1 - 30
), # Estimated bottom
)
# Draw a visible rectangle around the chunk region
shape = page.new_shape()
shape.draw_rect(chunk_rect)
shape.finish(
color=rgb, # Border color
fill=None, # No fill (transparent)
width=2.5, # Border width
dashes="[4 2]", # Dashed line
)
shape.commit()
# Add semi-transparent fill for visibility
fill_shape = page.new_shape()
fill_shape.draw_rect(chunk_rect)
fill_shape.finish(
color=None, # No border
fill=[1, 1, 0.7], # Light yellow fill
fill_opacity=0.15, # Very transparent
)
fill_shape.commit()
logger.debug(
f"Added bounding box at y={chunk_rect.y0:.0f}-{chunk_rect.y1:.0f} "
f"(estimated {estimated_lines:.1f} lines)"
)
return 1
@staticmethod
def highlight_chunk(
pdf_bytes: bytes,
chunk_start_offset: int,
chunk_end_offset: int,
stored_page_number: Optional[int] = None,
color: str = "yellow",
zoom: float = 2.0,
) -> Optional[tuple[bytes, int, int]]:
"""Generate PNG image of PDF page with highlighted chunk.
This is the main entry point for highlighting. It:
1. Extracts document text with page boundaries
2. Finds which page contains the chunk
3. Extracts chunk text using character offsets
4. Highlights the chunk on the page
5. Renders page to PNG
Args:
pdf_bytes: PDF file bytes
chunk_start_offset: Chunk start position (document-level)
chunk_end_offset: Chunk end position (document-level)
stored_page_number: Page number from metadata (optional, for validation)
color: Highlight color name
zoom: Rendering zoom factor (2.0 = 144 DPI)
Returns:
Tuple of (png_bytes, page_number, highlight_count) or None if failed
"""
import tempfile
from pathlib import Path
temp_pdf_path = None
try:
# Write PDF to temp file with consistent name "pdf.pdf"
# This ensures image references match indexing (e.g., pdf-0001.png)
# Different temp filenames would cause different markdown text lengths!
temp_dir = Path(tempfile.mkdtemp(prefix="pdf_highlight_"))
temp_pdf_path = temp_dir / "pdf.pdf"
temp_pdf_path.write_bytes(pdf_bytes)
# Open PDF from temp file
doc = pymupdf.open(temp_pdf_path)
# Extract text with page boundaries
full_text, page_boundaries = (
PDFHighlighter.extract_pdf_text_with_boundaries(doc)
)
# Find which page contains the chunk
chunk_page_info = PDFHighlighter.find_chunk_page(
chunk_start_offset, chunk_end_offset, page_boundaries
)
if not chunk_page_info:
logger.error("Chunk not found on any page")
doc.close()
return None
page_num = chunk_page_info["page_num"]
# Log if page differs from stored metadata
if stored_page_number and stored_page_number != page_num:
logger.info(
f"Chunk primarily on page {page_num}, metadata says {stored_page_number}"
)
# Extract page text
page_boundary = page_boundaries[page_num - 1]
page_start = page_boundary["start_offset"]
page_end = page_boundary["end_offset"]
page_text = full_text[page_start:page_end]
# Extract chunk text using page-relative offsets
page_relative_start = chunk_page_info["page_relative_start"]
page_relative_end = chunk_page_info["page_relative_end"]
chunk_text = page_text[page_relative_start:page_relative_end]
# Calculate page text length for region estimation
page_text_length = page_end - page_start
logger.debug(
f"Extracted {len(chunk_text)} chars on page {page_num} "
f"(offsets {page_relative_start}-{page_relative_end} of {page_text_length})"
)
# Get page and add highlights
page = doc[page_num - 1]
highlight_count = PDFHighlighter.highlight_chunk_on_page(
page,
chunk_text,
color,
page_relative_start=page_relative_start,
page_relative_end=page_relative_end,
page_text_length=page_text_length,
)
if highlight_count == 0:
logger.warning("No highlights added")
doc.close()
return None
# Render page to PNG
mat = pymupdf.Matrix(zoom, zoom)
pix = page.get_pixmap(matrix=mat, alpha=False)
png_bytes = pix.tobytes("png")
doc.close()
logger.info(
f"Generated {len(png_bytes):,} byte image with {highlight_count} highlights"
)
return (png_bytes, page_num, highlight_count)
except Exception as e:
logger.error(f"Error highlighting chunk: {e}", exc_info=True)
return None
finally:
# Clean up temp directory and PDF file
if temp_pdf_path and temp_pdf_path.parent.exists():
try:
import shutil
shutil.rmtree(temp_pdf_path.parent)
except Exception as e:
logger.warning(
f"Failed to delete temp directory {temp_pdf_path.parent}: {e}"
)
@staticmethod
def highlight_chunks_batch(
pdf_bytes: bytes,
chunks: list[tuple[int, int, int, int | None, str]],
page_boundaries: list[dict],
full_text: str,
color: str = "yellow",
zoom: float = 2.0,
) -> dict[int, tuple[bytes, int, int]]:
"""Generate highlighted images for multiple chunks.
Opens PDF once for rendering, uses pre-computed page boundaries from the
document processor. This ensures consistent character offsets between
chunking and highlighting.
Args:
pdf_bytes: PDF file bytes
chunks: List of (chunk_index, start_offset, end_offset, stored_page_number, chunk_text)
The chunk_index is used as the key in the returned dict.
chunk_text is the actual text content of the chunk.
page_boundaries: Pre-computed page boundaries from document processor.
Each entry: {"page": 1, "start_offset": 0, "end_offset": 1234}
full_text: Full document text for extracting page-relative portions.
color: Highlight color name
zoom: Rendering zoom factor (2.0 = 144 DPI)
Returns:
Dict mapping chunk_index to (png_bytes, page_number, highlight_count)
Chunks that fail to highlight are omitted from the result.
"""
import shutil
import tempfile
from collections import defaultdict
from pathlib import Path
results: dict[int, tuple[bytes, int, int]] = {}
if not chunks:
return results
temp_pdf_path = None
try:
# Write PDF to temp file
temp_dir = Path(tempfile.mkdtemp(prefix="pdf_highlight_batch_"))
temp_pdf_path = temp_dir / "pdf.pdf"
temp_pdf_path.write_bytes(pdf_bytes)
# Open PDF once (only for rendering, not text extraction)
doc = pymupdf.open(temp_pdf_path)
logger.debug(
f"Batch highlighting: {len(chunks)} chunks, "
f"{len(page_boundaries)} pages"
)
# Group chunks by their target page for efficient rendering
# We'll render each page only once with all its highlights
chunks_by_page: dict[int, list[tuple[int, dict, str]]] = defaultdict(list)
for chunk_tuple in chunks:
# Unpack chunk tuple - chunk_text is now passed directly
chunk_index, start_offset, end_offset, stored_page_num, chunk_text = (
chunk_tuple
)
# Find which page contains this chunk
chunk_page_info = PDFHighlighter.find_chunk_page(
start_offset, end_offset, page_boundaries
)
if not chunk_page_info:
logger.warning(f"Chunk {chunk_index}: not found on any page")
continue
page_num = chunk_page_info["page_num"]
# Log if page differs from stored metadata
if stored_page_num and stored_page_num != page_num:
logger.debug(
f"Chunk {chunk_index}: found on page {page_num}, "
f"metadata says {stored_page_num}"
)
# Extract page-relative portion of chunk text
# This is critical for cross-page chunks where the start
# of the chunk might be on a different page
page_boundary = page_boundaries[page_num - 1]
page_start = page_boundary["start_offset"]
page_end = page_boundary["end_offset"]
page_text_length = page_end - page_start
# Calculate what portion of the chunk appears on this page
chunk_start_on_page = max(start_offset, page_start)
chunk_end_on_page = min(end_offset, page_end)
# Extract just the text that appears on this page
page_relative_text = full_text[chunk_start_on_page:chunk_end_on_page]
chunks_by_page[page_num].append(
(chunk_index, chunk_page_info, page_relative_text, page_text_length)
)
logger.debug(
f"Chunks distributed across {len(chunks_by_page)} unique pages"
)
# Process each chunk, rendering with only its own highlights
# Store original page contents to restore between chunks
page_contents_cache: dict[int, list[bytes]] = {}
for page_num, page_chunks in chunks_by_page.items():
page = doc[page_num - 1]
# Cache original page contents (before any highlights added)
# xref is the PDF object reference for each content stream
if page_num not in page_contents_cache:
page_contents_cache[page_num] = []
xrefs = page.get_contents()
for xref in xrefs:
page_contents_cache[page_num].append(doc.xref_stream(xref))
for (
chunk_index,
chunk_page_info,
chunk_text,
page_text_length,
) in page_chunks:
try:
# Restore original page contents to remove previous highlights
# Highlights are drawn shapes, not annotations, so we must
# restore the content stream to clear them
xrefs = page.get_contents()
for i, xref in enumerate(xrefs):
if i < len(page_contents_cache[page_num]):
doc.update_stream(
xref, page_contents_cache[page_num][i]
)
# Add highlights for this chunk with region constraint
page_relative_start = chunk_page_info["page_relative_start"]
page_relative_end = chunk_page_info["page_relative_end"]
highlight_count = PDFHighlighter.highlight_chunk_on_page(
page,
chunk_text,
color,
page_relative_start=page_relative_start,
page_relative_end=page_relative_end,
page_text_length=page_text_length,
)
if highlight_count == 0:
logger.warning(f"Chunk {chunk_index}: no highlights added")
continue
# Render page to PNG
mat = pymupdf.Matrix(zoom, zoom)
pix = page.get_pixmap(matrix=mat, alpha=False)
png_bytes = pix.tobytes("png")
results[chunk_index] = (png_bytes, page_num, highlight_count)
logger.debug(
f"Chunk {chunk_index}: {len(png_bytes):,} bytes, "
f"page {page_num}, {highlight_count} highlights"
)
except Exception as e:
logger.error(f"Chunk {chunk_index}: error - {e}")
continue
doc.close()
logger.info(
f"Batch highlighted {len(results)}/{len(chunks)} chunks successfully"
)
return results
except Exception as e:
logger.error(f"Error in batch highlighting: {e}", exc_info=True)
return results
finally:
# Clean up temp directory
if temp_pdf_path and temp_pdf_path.parent.exists():
try:
shutil.rmtree(temp_pdf_path.parent)
except Exception as e:
logger.warning(f"Failed to clean up temp dir: {e}")
nextcloud_mcp_server/vector/document_chunker.py
+1 -1
View File
@@ -75,7 +75,7 @@ class DocumentChunker:
return [ChunkWithPosition(text="", start_offset=0, end_offset=0)]
# Run CPU-bound text splitting in thread pool to avoid blocking event loop
docs = await anyio.to_thread.run_sync(
docs = await anyio.to_thread.run_sync( # type: ignore[attr-defined]
self.splitter.create_documents,
[content],
)
nextcloud_mcp_server/vector/processor.py
+277 -99
View File
@@ -256,115 +256,265 @@ async def _index_document(
settings = get_settings()
# Fetch document content
if doc_task.doc_type == "note":
document = await nc_client.notes.get_note(int(doc_task.doc_id))
content = f"{document['title']}\n\n{document['content']}"
title = document["title"]
etag = document.get("etag", "")
file_metadata = {} # No file-specific metadata for notes
file_path = None # Notes don't have file paths
elif doc_task.doc_type == "file":
# For files, doc_id is now the numeric file ID, file_path comes from DocumentTask
if not doc_task.file_path:
raise ValueError(
f"File path required for file indexing but not provided (file_id={doc_task.doc_id})"
)
file_path = doc_task.file_path
# Read file content via WebDAV
content_bytes, content_type = await nc_client.webdav.read_file(file_path)
# Use document processor registry to extract text
from nextcloud_mcp_server.document_processors import get_registry
registry = get_registry()
try:
result = await registry.process(
content=content_bytes,
content_type=content_type,
filename=file_path,
)
content = result.text
file_metadata = result.metadata
title = file_metadata.get("title") or file_path.split("/")[-1]
etag = "" # WebDAV read_file doesn't return etag
# Diagnostic: Log page boundary information if available
if "page_boundaries" in file_metadata:
page_boundaries = file_metadata["page_boundaries"]
logger.info(
f"Page boundaries for {file_path}: "
f"{len(page_boundaries)} pages, text length: {len(content)}"
with trace_operation(
"vector_sync.fetch_content",
attributes={
"vector_sync.doc_type": doc_task.doc_type,
"vector_sync.doc_id": doc_task.doc_id,
},
):
if doc_task.doc_type == "note":
document = await nc_client.notes.get_note(int(doc_task.doc_id))
content = f"{document['title']}\n\n{document['content']}"
title = document["title"]
etag = document.get("etag", "")
file_metadata = {} # No file-specific metadata for notes
file_path = None # Notes don't have file paths
content_bytes = None # Notes don't have binary content
content_type = None
elif doc_task.doc_type == "file":
# For files, doc_id is now the numeric file ID, file_path comes from DocumentTask
if not doc_task.file_path:
raise ValueError(
f"File path required for file indexing but not provided (file_id={doc_task.doc_id})"
)
# Log first 3 page boundaries for debugging
for boundary in page_boundaries[:3]:
logger.debug(
f" Page {boundary['page']}: "
f"offsets [{boundary['start_offset']}:{boundary['end_offset']}]"
file_path = doc_task.file_path
# Read file content via WebDAV
content_bytes, content_type = await nc_client.webdav.read_file(file_path)
else:
raise ValueError(f"Unsupported doc_type: {doc_task.doc_type}")
# Process file content (text extraction)
if doc_task.doc_type == "file":
# Type narrowing: content_bytes and content_type are set for files
assert content_bytes is not None
assert content_type is not None
assert file_path is not None
with trace_operation(
"vector_sync.document_process",
attributes={
"vector_sync.content_type": content_type,
"vector_sync.file_size": len(content_bytes),
},
):
# Use document processor registry to extract text
from nextcloud_mcp_server.document_processors import get_registry
registry = get_registry()
try:
result = await registry.process(
content=content_bytes,
content_type=content_type,
filename=file_path,
)
content = result.text
file_metadata = result.metadata
title = file_metadata.get("title") or file_path.split("/")[-1]
etag = "" # WebDAV read_file doesn't return etag
# Diagnostic: Log page boundary information if available
if "page_boundaries" in file_metadata:
page_boundaries = file_metadata["page_boundaries"]
logger.info(
f"Page boundaries for {file_path}: "
f"{len(page_boundaries)} pages, text length: {len(content)}"
)
# Verify last boundary matches text length
if page_boundaries:
last_boundary = page_boundaries[-1]
if last_boundary["end_offset"] != len(content):
logger.warning(
f"Text length mismatch: content={len(content)}, "
f"last_boundary_end={last_boundary['end_offset']}"
# Log first 3 page boundaries for debugging
for boundary in page_boundaries[:3]:
logger.debug(
f" Page {boundary['page']}: "
f"offsets [{boundary['start_offset']}:{boundary['end_offset']}]"
)
else:
logger.debug(f"No page_boundaries in metadata for {file_path}")
except Exception as e:
logger.error(f"Failed to process file {file_path}: {e}")
raise
else:
raise ValueError(f"Unsupported doc_type: {doc_task.doc_type}")
# Verify last boundary matches text length
if page_boundaries:
last_boundary = page_boundaries[-1]
if last_boundary["end_offset"] != len(content):
logger.warning(
f"Text length mismatch: content={len(content)}, "
f"last_boundary_end={last_boundary['end_offset']}"
)
else:
logger.debug(f"No page_boundaries in metadata for {file_path}")
except Exception as e:
logger.error(f"Failed to process file {file_path}: {e}")
raise
# Tokenize and chunk (using configured chunk size and overlap)
chunker = DocumentChunker(
chunk_size=settings.document_chunk_size,
overlap=settings.document_chunk_overlap,
)
chunks = await chunker.chunk_text(content)
with trace_operation(
"vector_sync.chunk_text",
attributes={
"vector_sync.input_chars": len(content),
"vector_sync.chunk_size": settings.document_chunk_size,
"vector_sync.overlap": settings.document_chunk_overlap,
},
):
chunker = DocumentChunker(
chunk_size=settings.document_chunk_size,
overlap=settings.document_chunk_overlap,
)
chunks = await chunker.chunk_text(content)
# Assign page numbers to chunks if page boundaries are available (PDFs)
if doc_task.doc_type == "file" and "page_boundaries" in file_metadata:
assign_page_numbers(chunks, file_metadata["page_boundaries"])
with trace_operation(
"vector_sync.assign_page_numbers",
attributes={
"vector_sync.chunk_count": len(chunks),
"vector_sync.page_count": len(file_metadata["page_boundaries"]),
},
):
assign_page_numbers(chunks, file_metadata["page_boundaries"])
# Diagnostic: Verify page number assignment
assigned_count = sum(1 for c in chunks if c.page_number is not None)
logger.info(
f"Assigned page numbers to {assigned_count}/{len(chunks)} chunks "
f"for {file_path}"
)
# Log first 3 chunks to see their page assignments
for i, chunk in enumerate(chunks[:3]):
logger.debug(
f" Chunk {i}: page={chunk.page_number}, "
f"offsets=[{chunk.start_offset}:{chunk.end_offset}]"
# Diagnostic: Verify page number assignment
assigned_count = sum(1 for c in chunks if c.page_number is not None)
logger.info(
f"Assigned page numbers to {assigned_count}/{len(chunks)} chunks "
f"for {file_path}"
)
# Warning if NO page numbers were assigned
if assigned_count == 0:
logger.warning(
f"NO page numbers assigned! "
f"Text length: {len(content)}, "
f"Chunks: {len(chunks)}, "
f"Chunk offset range: [{chunks[0].start_offset}:{chunks[-1].end_offset}], "
f"Page boundaries: {len(file_metadata['page_boundaries'])} pages, "
f"First boundary: {file_metadata['page_boundaries'][0] if file_metadata['page_boundaries'] else 'None'}"
)
# Log first 3 chunks to see their page assignments
for i, chunk in enumerate(chunks[:3]):
logger.debug(
f" Chunk {i}: page={chunk.page_number}, "
f"offsets=[{chunk.start_offset}:{chunk.end_offset}]"
)
# Warning if NO page numbers were assigned
if assigned_count == 0:
logger.warning(
f"NO page numbers assigned! "
f"Text length: {len(content)}, "
f"Chunks: {len(chunks)}, "
f"Chunk offset range: [{chunks[0].start_offset}:{chunks[-1].end_offset}], "
f"Page boundaries: {len(file_metadata['page_boundaries'])} pages, "
f"First boundary: {file_metadata['page_boundaries'][0] if file_metadata['page_boundaries'] else 'None'}"
)
# Extract chunk texts for embedding
chunk_texts = [chunk.text for chunk in chunks]
# Generate dense embeddings (I/O bound - external API call)
embedding_service = get_embedding_service()
dense_embeddings = await embedding_service.embed_batch(chunk_texts)
# Initialize results containers
dense_embeddings: list = []
sparse_embeddings: list = []
chunk_images: dict[int, dict] = {}
# Generate sparse embeddings (BM25 for keyword matching)
bm25_service = get_bm25_service()
sparse_embeddings = await bm25_service.encode_batch(chunk_texts)
# Determine if we need PDF highlighting
is_pdf = doc_task.doc_type == "file" and content_type == "application/pdf"
# Define async tasks for parallel execution
async def generate_dense_embeddings():
"""Generate dense embeddings (I/O bound - external API call)."""
nonlocal dense_embeddings
with trace_operation(
"vector_sync.embed_dense",
attributes={
"vector_sync.chunk_count": len(chunk_texts),
"vector_sync.total_chars": sum(len(t) for t in chunk_texts),
},
):
embedding_service = get_embedding_service()
dense_embeddings = await embedding_service.embed_batch(chunk_texts)
async def generate_sparse_embeddings():
"""Generate sparse embeddings (BM25 for keyword matching)."""
nonlocal sparse_embeddings
with trace_operation(
"vector_sync.embed_sparse",
attributes={
"vector_sync.chunk_count": len(chunk_texts),
},
):
bm25_service = get_bm25_service()
sparse_embeddings = await bm25_service.encode_batch(chunk_texts)
async def generate_highlights():
"""Generate highlighted page images for PDF chunks (CPU-bound)."""
nonlocal chunk_images
if not is_pdf:
return
# Type narrowing: content_bytes is set for PDF files
assert content_bytes is not None
with trace_operation(
"vector_sync.generate_highlights",
attributes={
"vector_sync.chunk_count": len(chunks),
"vector_sync.pdf_size": len(content_bytes),
},
):
import base64
from nextcloud_mcp_server.search.pdf_highlighter import PDFHighlighter
# Build chunk data for batch processing
# Format: (chunk_index, start_offset, end_offset, page_number, chunk_text)
chunk_data: list[tuple[int, int, int, int | None, str]] = [
(i, chunk.start_offset, chunk.end_offset, chunk.page_number, chunk.text)
for i, chunk in enumerate(chunks)
if chunk.page_number is not None
]
# Get pre-computed page boundaries from document processor
page_boundaries = file_metadata.get("page_boundaries")
if not page_boundaries:
logger.warning("No page boundaries available, skipping highlighting")
return
logger.info(
f"Batch generating highlighted page images for {len(chunk_data)} PDF chunks"
)
# Run CPU-bound highlighting in thread pool
# Pass pre-computed page boundaries and full text to avoid re-processing the PDF
batch_results = await anyio.to_thread.run_sync( # type: ignore[attr-defined]
lambda: PDFHighlighter.highlight_chunks_batch(
pdf_bytes=content_bytes,
chunks=chunk_data,
page_boundaries=page_boundaries,
full_text=content,
color="yellow",
zoom=2.0,
)
)
# Convert results to storage format
for chunk_index, (
png_bytes,
actual_page_num,
highlight_count,
) in batch_results.items():
image_base64 = base64.b64encode(png_bytes).decode("utf-8")
chunk_images[chunk_index] = {
"image": image_base64,
"page": actual_page_num,
"highlights": highlight_count,
"size": len(png_bytes),
}
logger.info(
f"Generated {len(chunk_images)}/{len(chunks)} highlighted page images "
f"(avg {sum(img['size'] for img in chunk_images.values()) // max(len(chunk_images), 1):,} bytes)"
)
# Run all embedding/highlighting operations in parallel
# - Dense embeddings: I/O bound (API call)
# - Sparse embeddings: CPU bound (local BM25)
# - Highlighting: CPU bound (PyMuPDF rendering, runs in thread pool)
with trace_operation(
"vector_sync.parallel_processing",
attributes={
"vector_sync.is_pdf": is_pdf,
"vector_sync.chunk_count": len(chunks),
},
):
async with anyio.create_task_group() as tg:
tg.start_soon(generate_dense_embeddings)
tg.start_soon(generate_sparse_embeddings)
tg.start_soon(generate_highlights)
# Prepare Qdrant points
indexed_at = int(time.time())
@@ -416,6 +566,16 @@ async def _index_document(
if doc_task.doc_type == "file"
else {}
),
# Highlighted page image (PDF only)
**(
{
"highlighted_page_image": chunk_images[i]["image"],
"highlighted_page_number": chunk_images[i]["page"],
"highlight_count": chunk_images[i]["highlights"],
}
if i in chunk_images
else {}
),
},
)
)
@@ -434,12 +594,30 @@ async def _index_document(
f"Failed to delete placeholder for {doc_task.doc_type}_{doc_task.doc_id}: {e}"
)
# Upsert to Qdrant
await qdrant_client.upsert(
collection_name=settings.get_collection_name(),
points=points,
wait=True,
)
# Upsert to Qdrant in batches to avoid timeout with large payloads
# Each batch is limited to avoid WriteTimeout when sending large image payloads
BATCH_SIZE = 10 # ~2MB per batch with images
with trace_operation(
"vector_sync.qdrant_upsert",
attributes={
"vector_sync.point_count": len(points),
"vector_sync.collection": settings.get_collection_name(),
"vector_sync.images_count": len(chunk_images),
"vector_sync.batch_size": BATCH_SIZE,
},
):
for batch_start in range(0, len(points), BATCH_SIZE):
batch_end = min(batch_start + BATCH_SIZE, len(points))
batch = points[batch_start:batch_end]
await qdrant_client.upsert(
collection_name=settings.get_collection_name(),
points=batch,
wait=True,
)
if batch_end < len(points):
logger.debug(
f"Upserted batch {batch_start // BATCH_SIZE + 1}/{(len(points) + BATCH_SIZE - 1) // BATCH_SIZE}"
)
logger.info(
f"Indexed {doc_task.doc_type}_{doc_task.doc_id} for {doc_task.user_id} "