137d1d6c75
This commit addresses critical performance issues with vector visualization search (reducing time from 40s to ~2s) and improves result visualization through better visual encoding. ## Performance Fixes ### 1. Fix blocking sleep in retry decorator (base.py:51) - Changed `time.sleep(5)` to `await anyio.sleep(5)` in @retry_on_429 - Prevents entire event loop from freezing during rate limit retries - Impact: Reduced search time from 22s to 16s initially ### 2. Add concurrency limiting for verification (verification.py:77-93) - Added `anyio.Semaphore(20)` to limit concurrent HTTP requests - Prevents connection pool exhaustion (RequestError) from 90+ simultaneous requests - Fixes false filtering (was filtering 77/90 results incorrectly) - Note: Semaphore still in code but verification removed from viz endpoint ### 3. Remove unnecessary verification from viz endpoint (viz_routes.py:483-486) - Visualization only needs Qdrant metadata (title, excerpt), not full content - Verification only required for sampling (LLM needs full note content) - Impact: Reduced search time from 43.7s to ~2s (final fix) ### 4. Restore streaming scanner pattern (scanner.py) - Process notes one-at-a-time using async generator - Avoids loading all notes into memory ## Visualization Improvements ### 5. Result-relative score normalization (viz_routes.py:489-504) - Normalize scores within result set: best=1.0, worst=0.0 - Removes arbitrary RRF normalization (theoretical max didn't make sense) - Makes visual encoding meaningful regardless of algorithm scores ### 6. Power scaling for marker sizes (userinfo_routes.py:743) - Changed from linear `8 + (score * 12)` to power `6 + (score² * 14)` - Creates dramatic visual contrast: 0.0→6px, 0.5→9.5px, 1.0→20px - Combined with opacity (0.2-1.0) for clear visual hierarchy ### 7. Multi-channel visual encoding (userinfo_routes.py:740-745) - Size: Exponentially scaled with score² - Opacity: Linear 0.2-1.0 (keeps all points visible) - Color: Viridis gradient (blue→yellow) - Effect: Top results are large/bright/opaque, context results small/dim/transparent ## Result - Search time: 40s → ~2s (20x faster) - Visual contrast: Subtle → dramatic (clear result hierarchy) - No arbitrary cutoffs: All results visible, best naturally highlighted 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
159 lines
5.3 KiB
Python
159 lines
5.3 KiB
Python
"""Base client for Nextcloud operations with shared authentication."""
|
|
|
|
import logging
|
|
import time
|
|
from abc import ABC
|
|
from functools import wraps
|
|
|
|
import anyio
|
|
from httpx import AsyncClient, HTTPStatusError, RequestError, codes
|
|
|
|
from nextcloud_mcp_server.observability.metrics import (
|
|
record_nextcloud_api_call,
|
|
record_nextcloud_api_retry,
|
|
)
|
|
from nextcloud_mcp_server.observability.tracing import trace_nextcloud_api_call
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def retry_on_429(func):
|
|
"""This decorator handles the 429 response from REST APIs
|
|
|
|
The `func` is assumed to be a method that is similar to `httpx.Client.get`,
|
|
and returns an `httpx.Response` object. In the case of `Too Many Requests` HTTP
|
|
response, the function will wait for a couple of seconds and retry the request.
|
|
"""
|
|
|
|
MAX_RETRIES = 5
|
|
|
|
@wraps(func)
|
|
async def wrapper(*args, **kwargs):
|
|
retries = 0
|
|
|
|
while retries < MAX_RETRIES:
|
|
try:
|
|
# Make GET API call
|
|
retries += 1
|
|
response = await func(*args, **kwargs)
|
|
break
|
|
|
|
except HTTPStatusError as e:
|
|
# If we get a '429 Client Error: Too Many Requests'
|
|
# error we wait a couple of seconds and do a retry
|
|
if e.response.status_code == codes.TOO_MANY_REQUESTS:
|
|
logger.warning(
|
|
f"429 Client Error: Too Many Requests, Number of attempts: {retries}"
|
|
)
|
|
# Record retry metric (extract app name from args if available)
|
|
if len(args) > 0 and hasattr(args[0], "app_name"):
|
|
record_nextcloud_api_retry(app=args[0].app_name, reason="429")
|
|
await anyio.sleep(5)
|
|
elif e.response.status_code == 404:
|
|
# 404 errors are often expected (e.g., checking if attachments exist)
|
|
# Log as debug instead of warning
|
|
logger.debug(
|
|
f"HTTPStatusError {e.response.status_code}: {e}, Number of attempts: {retries}"
|
|
)
|
|
raise
|
|
else:
|
|
logger.warning(
|
|
f"HTTPStatusError {e.response.status_code}: {e}, Number of attempts: {retries}"
|
|
)
|
|
raise
|
|
except RequestError as e:
|
|
logger.warning(
|
|
f"RequestError {e.request.url}: {e}, Number of attempts: {retries}"
|
|
)
|
|
raise
|
|
|
|
# If for loop ends without break statement
|
|
else:
|
|
logger.warning("All API call retries failed")
|
|
raise RuntimeError(
|
|
f"Maximum number of retries ({MAX_RETRIES}) exceeded without success"
|
|
)
|
|
|
|
return response
|
|
|
|
return wrapper
|
|
|
|
|
|
class BaseNextcloudClient(ABC):
|
|
"""Base class for all Nextcloud app clients."""
|
|
|
|
# Subclasses should set this to identify the app for metrics/tracing
|
|
app_name: str = "unknown"
|
|
|
|
def __init__(self, http_client: AsyncClient, username: str):
|
|
"""Initialize with shared HTTP client and username.
|
|
|
|
Args:
|
|
http_client: Authenticated AsyncClient instance
|
|
username: Nextcloud username for WebDAV operations
|
|
"""
|
|
self._client = http_client
|
|
self.username = username
|
|
|
|
def _get_webdav_base_path(self) -> str:
|
|
"""Helper to get the base WebDAV path for the authenticated user."""
|
|
return f"/remote.php/dav/files/{self.username}"
|
|
|
|
@retry_on_429
|
|
async def _make_request(self, method: str, url: str, **kwargs):
|
|
"""Common request wrapper with logging, tracing, and error handling.
|
|
|
|
Args:
|
|
method: HTTP method
|
|
url: Request URL
|
|
**kwargs: Additional request parameters
|
|
|
|
Returns:
|
|
Response object
|
|
"""
|
|
logger.debug(f"Making {method} request to {url}")
|
|
|
|
# Start timer for metrics
|
|
start_time = time.time()
|
|
status_code = 0
|
|
|
|
try:
|
|
# Wrap request in trace span
|
|
with trace_nextcloud_api_call(
|
|
app=self.app_name,
|
|
method=method,
|
|
path=url,
|
|
):
|
|
response = await self._client.request(method, url, **kwargs)
|
|
status_code = response.status_code
|
|
response.raise_for_status()
|
|
|
|
# Record successful API call metrics
|
|
duration = time.time() - start_time
|
|
record_nextcloud_api_call(
|
|
app=self.app_name,
|
|
method=method,
|
|
status_code=status_code,
|
|
duration=duration,
|
|
)
|
|
|
|
return response
|
|
|
|
except (HTTPStatusError, RequestError) as e:
|
|
# Record error metrics
|
|
if isinstance(e, HTTPStatusError):
|
|
status_code = e.response.status_code
|
|
else:
|
|
status_code = 0 # Connection error, no status code
|
|
|
|
duration = time.time() - start_time
|
|
record_nextcloud_api_call(
|
|
app=self.app_name,
|
|
method=method,
|
|
status_code=status_code,
|
|
duration=duration,
|
|
)
|
|
|
|
# Re-raise the exception
|
|
raise
|