"""Semantic deduplication for retrieval chunks and generated answer blocks."""

from __future__ import annotations

import difflib
import re


def _normalize(value: str) -> str:
    text = str(value or "").strip().lower()
    text = re.sub(r"[^\w\sа-яё0-9]", " ", text, flags=re.IGNORECASE)
    return re.sub(r"\s+", " ", text).strip()


def similarity(left: str, right: str) -> float:
    a = _normalize(left)
    b = _normalize(right)
    if not a or not b:
        return 0.0
    if a == b:
        return 1.0
    if a in b or b in a:
        shorter = min(len(a), len(b))
        longer = max(len(a), len(b))
        return shorter / max(longer, 1)
    return difflib.SequenceMatcher(None, a, b).ratio()


def deduplicate_chunks(chunks: list[str], *, threshold: float = 0.82) -> list[str]:
    kept: list[str] = []
    for chunk in chunks:
        text = str(chunk or "").strip()
        if not text:
            continue
        duplicate = False
        for prior in kept:
            if similarity(text, prior) >= threshold:
                duplicate = True
                break
        if duplicate:
            continue
        kept.append(text)
    return kept


def deduplicate_blocks(text: str, *, threshold: float = 0.82) -> str:
    value = str(text or "").strip()
    if not value:
        return ""

    blocks = [part.strip() for part in re.split(r"\n{2,}", value) if part.strip()]
    if len(blocks) < 2:
        return value

    kept = deduplicate_chunks(blocks, threshold=threshold)
    return "\n\n".join(kept).strip()
