Query Processing & Relevance.

# scoring/bm25_explained.py

"""
BM25 scoring explained with examples.

Understanding these components helps you tune relevance.
"""

import math
from dataclasses import dataclass
from typing import List, Dict


@dataclass
class BM25Params:
    """BM25 tuning parameters."""
    k1: float = 1.2   # Term frequency saturation
    b: float = 0.75   # Length normalization


class BM25Explainer:
    """
    Explains BM25 scoring step by step.
    """
    
    def __init__(self, params: BM25Params = None):
        self.params = params or BM25Params()
    
    def explain_idf(
        self,
        term: str,
        doc_freq: int,
        total_docs: int
    ) -> dict:
        """
        Explain Inverse Document Frequency.
        
        IDF measures how rare/common a term is across all documents.
        Rare terms get higher scores (more discriminating).
        
        Formula: log(1 + (N - n + 0.5) / (n + 0.5))
        Where N = total docs, n = docs containing term
        """
        
        idf = math.log(1 + (total_docs - doc_freq + 0.5) / (doc_freq + 0.5))
        
        return {
            "term": term,
            "docs_containing_term": doc_freq,
            "total_docs": total_docs,
            "idf_score": round(idf, 4),
            "interpretation": self._interpret_idf(idf, doc_freq, total_docs)
        }
    
    def _interpret_idf(self, idf: float, doc_freq: int, total_docs: int) -> str:
        pct = (doc_freq / total_docs) * 100
        
        if pct > 50:
            return f"Very common term ({pct:.1f}% of docs). Low discriminating power."
        elif pct >= 10:
            return f"Common term ({pct:.1f}% of docs). Moderate discriminating power."
        elif pct > 1:
            return f"Uncommon term ({pct:.1f}% of docs). Good discriminating power."
        else:
            return f"Rare term ({pct:.2f}% of docs). High discriminating power."
    
    def explain_tf_component(
        self,
        term_freq: int,
        doc_length: int,
        avg_doc_length: float
    ) -> dict:
        """
        Explain term frequency component.
        
        More occurrences = higher score, but with saturation.
        Longer documents are normalized.
        """
        
        k1 = self.params.k1
        b = self.params.b
        
        # Length normalization factor
        length_norm = 1 - b + b * (doc_length / avg_doc_length)
        
        # TF saturation
        tf_component = (term_freq * (k1 + 1)) / (term_freq + k1 * length_norm)
        
        return {
            "term_frequency": term_freq,
            "doc_length": doc_length,
            "avg_doc_length": avg_doc_length,
            "length_normalization": round(length_norm, 4),
            "tf_component": round(tf_component, 4),
            "interpretation": self._interpret_tf(term_freq, tf_component)
        }
    
    def _interpret_tf(self, term_freq: int, tf_component: float) -> str:
        if term_freq == 0:
            return "Term not in document. No contribution."
        elif term_freq == 1:
            return f"Single occurrence. Base contribution: {tf_component:.2f}"
        elif term_freq < 5:
            return f"{term_freq} occurrences. Moderate boost: {tf_component:.2f}"
        else:
            return f"{term_freq} occurrences. Saturating (diminishing returns): {tf_component:.2f}"
    
    def score_document(
        self,
        query_terms: List[str],
        document: Dict[str, int],  # term -> frequency
        doc_length: int,
        corpus_stats: dict
    ) -> dict:
        """
        Calculate full BM25 score with explanation.
        """
        
        total_docs = corpus_stats["total_docs"]
        avg_length = corpus_stats["avg_length"]
        doc_freqs = corpus_stats["doc_freqs"]  # term -> docs containing it
        
        term_scores = []
        total_score = 0.0
        
        for term in query_terms:
            term_freq = document.get(term, 0)
            doc_freq = doc_freqs.get(term, 0)
            
            # Calculate IDF
            if doc_freq == 0:
                idf = 0
            else:
                idf = math.log(1 + (total_docs - doc_freq + 0.5) / (doc_freq + 0.5))
            
            # Calculate TF component
            if term_freq == 0:
                tf_component = 0
            else:
                length_norm = 1 - self.params.b + self.params.b * (doc_length / avg_length)
                tf_component = (term_freq * (self.params.k1 + 1)) / (term_freq + self.params.k1 * length_norm)
            
            # Term score
            term_score = idf * tf_component
            total_score += term_score
            
            term_scores.append({
                "term": term,
                "idf": round(idf, 4),
                "tf_component": round(tf_component, 4),
                "term_score": round(term_score, 4)
            })
        
        return {
            "total_score": round(total_score, 4),
            "term_breakdown": term_scores
        }


# =============================================================================
# Example: Understanding Scoring
# =============================================================================

def demonstrate_scoring():
    """Show how BM25 scoring works in practice."""
    
    explainer = BM25Explainer()
    
    # Corpus statistics
    corpus = {
        "total_docs": 1_000_000,
        "avg_length": 50,  # words
        "doc_freqs": {
            "running": 50_000,      # 5% of docs
            "shoes": 100_000,       # 10% of docs
            "nike": 20_000,         # 2% of docs
            "professional": 30_000, # 3% of docs
            "the": 900_000,         # 90% of docs (stop word)
        }
    }
    
    print("=== IDF COMPARISON ===\n")
    
    for term, doc_freq in corpus["doc_freqs"].items():
        result = explainer.explain_idf(term, doc_freq, corpus["total_docs"])
        print(f"Term: '{term}'")
        print(f"  IDF: {result['idf_score']}")
        print(f"  {result['interpretation']}\n")
    
    print("\n=== DOCUMENT SCORING ===\n")
    
    # Score two documents for query "running shoes"
    query = ["running", "shoes"]
    
    doc1 = {
        "content": {"running": 3, "shoes": 2, "nike": 1},
        "length": 45,
        "title": "Nike Running Shoes Review"
    }
    
    doc2 = {
        "content": {"running": 1, "shoes": 1, "professional": 2, "the": 5},
        "length": 150,
        "title": "Professional Running Shoes Guide: The Complete Resource"
    }
    
    for i, doc in enumerate([doc1, doc2], 1):
        result = explainer.score_document(
            query,
            doc["content"],
            doc["length"],
            corpus
        )
        
        print(f"Document {i}: '{doc['title']}'")
        print(f"  Total Score: {result['total_score']}")
        print(f"  Breakdown:")
        for term_score in result["term_breakdown"]:
            print(f"    '{term_score['term']}': IDF={term_score['idf']} × TF={term_score['tf_component']} = {term_score['term_score']}")
        print()


# Output:
# === IDF COMPARISON ===
# 
# Term: 'running'
#   IDF: 2.9957
#   Uncommon term (5.0% of docs). Good discriminating power.
# 
# Term: 'shoes'
#   IDF: 2.3026
#   Common term (10.0% of docs). Moderate discriminating power.
# 
# Term: 'nike'
#   IDF: 3.912
#   Uncommon term (2.0% of docs). Good discriminating power.
# 
# Term: 'professional'
#   IDF: 3.5065
#   Uncommon term (3.0% of docs). Good discriminating power.
# 
# Term: 'the'
#   IDF: 0.1054
#   Very common term (90.0% of docs). Low discriminating power.
#
# === DOCUMENT SCORING ===
#
# Document 1: 'Nike Running Shoes Review'
#   Total Score: 8.0683
#   Breakdown:
#     'running': IDF=2.9957 × TF=1.6058 = 4.8107
#     'shoes': IDF=2.3026 × TF=1.4148 = 3.2577
#
# Document 2: 'Professional Running Shoes Guide: The Complete Resource'
#   Total Score: 2.9141
#   Breakdown:
#     'running': IDF=2.9957 × TF=0.55 = 1.6476
#     'shoes': IDF=2.3026 × TF=0.55 = 1.2664

# queries/query_vs_filter.py

"""
Proper use of query vs filter context.
"""


class SearchQueryBuilder:
    """
    Builds Elasticsearch queries with proper query/filter separation.
    """
    
    def build_search_query(
        self,
        search_text: str,
        filters: dict
    ) -> dict:
        """
        Build query with proper context separation.
        
        Query context: Full-text search (scored)
        Filter context: Exact matches, ranges (not scored, cached)
        """
        
        must_queries = []
        filter_clauses = []
        
        # Full-text search goes in QUERY context (scored)
        if search_text:
            must_queries.append({
                "multi_match": {
                    "query": search_text,
                    "fields": ["name^3", "brand^2", "description"],
                    "type": "best_fields"
                }
            })
        
        # Filters go in FILTER context (not scored, cacheable)
        if filters.get("category"):
            filter_clauses.append({
                "term": {"category": filters["category"]}
            })
        
        if filters.get("brand"):
            brands = filters["brand"]
            if isinstance(brands, list):
                filter_clauses.append({
                    "terms": {"brand.keyword": brands}
                })
            else:
                filter_clauses.append({
                    "term": {"brand.keyword": brands}
                })
        
        if filters.get("price_min") or filters.get("price_max"):
            price_range = {}
            if filters.get("price_min"):
                price_range["gte"] = filters["price_min"]
            if filters.get("price_max"):
                price_range["lte"] = filters["price_max"]
            filter_clauses.append({
                "range": {"price": price_range}
            })
        
        if filters.get("in_stock") is not None:
            filter_clauses.append({
                "term": {"in_stock": filters["in_stock"]}
            })
        
        if filters.get("rating_min"):
            filter_clauses.append({
                "range": {"rating": {"gte": filters["rating_min"]}}
            })
        
        # Build the bool query
        bool_query = {}
        
        if must_queries:
            bool_query["must"] = must_queries
        else:
            # No search text - match all, rely on filters
            bool_query["must"] = [{"match_all": {}}]
        
        if filter_clauses:
            bool_query["filter"] = filter_clauses
        
        return {"bool": bool_query}
    
    def explain_query(self, query: dict) -> str:
        """Explain what the query does."""
        
        explanation = []
        bool_query = query.get("bool", {})
        
        # Query context
        must = bool_query.get("must", [])
        for clause in must:
            if "multi_match" in clause:
                explanation.append(
                    f"SCORED: Search for '{clause['multi_match']['query']}' "
                    f"in fields {clause['multi_match']['fields']}"
                )
            elif "match_all" in clause:
                explanation.append("SCORED: Match all documents (no search text)")
        
        # Filter context
        filters = bool_query.get("filter", [])
        for clause in filters:
            if "term" in clause:
                field, value = list(clause["term"].items())[0]
                explanation.append(f"FILTER: {field} = {value}")
            elif "terms" in clause:
                field, values = list(clause["terms"].items())[0]
                explanation.append(f"FILTER: {field} in {values}")
            elif "range" in clause:
                field, bounds = list(clause["range"].items())[0]
                explanation.append(f"FILTER: {field} range {bounds}")
        
        return "\n".join(explanation)

# queries/boosting.py

"""
Boosting strategies to control relevance.
"""


class BoostingQueryBuilder:
    """
    Builds queries with various boosting strategies.
    """
    
    def field_boosting(self, search_text: str) -> dict:
        """
        Boost certain fields over others.
        
        Match in product name is more relevant than description.
        """
        
        return {
            "multi_match": {
                "query": search_text,
                "fields": [
                    "name^3",         # 3x boost for name matches
                    "brand^2",        # 2x boost for brand matches
                    "category^1.5",   # 1.5x boost for category
                    "description",    # Default 1x
                    "tags^1.2"        # Slight boost for tags
                ],
                "type": "best_fields",
                "tie_breaker": 0.3    # Some credit for matches in multiple fields
            }
        }
    
    def phrase_boosting(self, search_text: str) -> dict:
        """
        Boost exact phrase matches.
        
        "running shoes" as a phrase scores higher than
        documents with "running" and "shoes" separately.
        """
        
        return {
            "bool": {
                "must": [
                    {
                        "multi_match": {
                            "query": search_text,
                            "fields": ["name^3", "description"],
                            "type": "best_fields"
                        }
                    }
                ],
                "should": [
                    {
                        "match_phrase": {
                            "name": {
                                "query": search_text,
                                "boost": 2.0,  # 2x boost for exact phrase
                                "slop": 1      # Allow 1 word between
                            }
                        }
                    },
                    {
                        "match_phrase": {
                            "description": {
                                "query": search_text,
                                "boost": 1.5,
                                "slop": 2
                            }
                        }
                    }
                ]
            }
        }
    
    def negative_boosting(self, search_text: str) -> dict:
        """
        Demote certain documents.
        
        Out of stock items pushed down, not removed.
        """
        
        return {
            "boosting": {
                "positive": {
                    "multi_match": {
                        "query": search_text,
                        "fields": ["name^3", "description"]
                    }
                },
                "negative": {
                    "term": {"in_stock": False}
                },
                "negative_boost": 0.2  # Reduce score to 20%
            }
        }
    
    def conditional_boosting(self, search_text: str) -> dict:
        """
        Boost based on business rules.
        
        Sponsored products, featured items, etc.
        """
        
        return {
            "bool": {
                "must": [
                    {
                        "multi_match": {
                            "query": search_text,
                            "fields": ["name^3", "description"]
                        }
                    }
                ],
                "should": [
                    # Boost featured products
                    {
                        "term": {
                            "is_featured": {
                                "value": True,
                                "boost": 5.0
                            }
                        }
                    },
                    # Boost new products
                    {
                        "range": {
                            "created_at": {
                                "gte": "now-30d",
                                "boost": 2.0
                            }
                        }
                    },
                    # Boost highly rated products
                    {
                        "range": {
                            "rating": {
                                "gte": 4.5,
                                "boost": 1.5
                            }
                        }
                    },
                    # Boost products with many reviews
                    {
                        "range": {
                            "review_count": {
                                "gte": 100,
                                "boost": 1.3
                            }
                        }
                    }
                ]
            }
        }

# queries/function_score.py

"""
Function score queries for complex ranking.

When simple boosting isn't enough, function_score lets you
combine text relevance with arbitrary scoring functions.
"""


class FunctionScoreBuilder:
    """
    Builds function_score queries for custom ranking.
    """
    
    def popularity_boosting(self, search_text: str) -> dict:
        """
        Combine text relevance with popularity score.
        
        Final score = text_score * popularity_factor
        """
        
        return {
            "function_score": {
                "query": {
                    "multi_match": {
                        "query": search_text,
                        "fields": ["name^3", "description"]
                    }
                },
                "functions": [
                    {
                        # Boost by popularity score (stored field)
                        "field_value_factor": {
                            "field": "popularity_score",
                            "factor": 1.2,
                            "modifier": "sqrt",  # Dampen extreme values
                            "missing": 1         # Default if field missing
                        }
                    }
                ],
                "score_mode": "multiply",  # Multiply all function results
                "boost_mode": "multiply"   # Multiply with query score
            }
        }
    
    def recency_decay(self, search_text: str) -> dict:
        """
        Newer products score higher, with decay.
        
        Uses Gaussian decay: full score for recent,
        gradually decreasing for older products.
        """
        
        return {
            "function_score": {
                "query": {
                    "multi_match": {
                        "query": search_text,
                        "fields": ["name^3", "description"]
                    }
                },
                "functions": [
                    {
                        "gauss": {
                            "created_at": {
                                "origin": "now",     # Center point
                                "scale": "30d",      # 50% score at offset + scale (37d)
                                "offset": "7d",      # Full score for first 7 days
                                "decay": 0.5         # Score at scale distance
                            }
                        }
                    }
                ],
                "boost_mode": "multiply"
            }
        }
    
    def multi_factor_ranking(self, search_text: str) -> dict:
        """
        Combine multiple ranking factors.
        
        Real-world ranking often combines:
        - Text relevance
        - Popularity (sales, views)
        - Recency
        - Rating quality
        - Inventory status
        """
        
        return {
            "function_score": {
                "query": {
                    "multi_match": {
                        "query": search_text,
                        "fields": ["name^3", "brand^2", "description"]
                    }
                },
                "functions": [
                    # Factor 1: Popularity score
                    {
                        "field_value_factor": {
                            "field": "popularity_score",
                            "modifier": "log1p",  # log10(1 + x) to dampen
                            "missing": 0
                        },
                        "weight": 2  # This factor counts 2x
                    },
                    
                    # Factor 2: Rating quality
                    {
                        "script_score": {
                            "script": {
                                "source": """
                                    // Combine rating with review count
                                    // High rating with few reviews < Medium rating with many reviews
                                    double rating = doc['rating'].value;
                                    double reviews = doc['review_count'].value;
                                    double confidence = Math.min(1.0, reviews / 50.0);
                                    return rating * confidence;
                                """
                            }
                        },
                        "weight": 1.5
                    },
                    
                    # Factor 3: Recency decay
                    {
                        "gauss": {
                            "created_at": {
                                "origin": "now",
                                "scale": "90d",
                                "decay": 0.5
                            }
                        },
                        "weight": 1
                    },
                    
                    # Factor 4: In-stock boost
                    {
                        "filter": {"term": {"in_stock": True}},
                        "weight": 1.5
                    },
                    
                    # Factor 5: Featured boost
                    {
                        "filter": {"term": {"is_featured": True}},
                        "weight": 3
                    },
                    
                    # Factor 6: Price competitiveness (lower = better)
                    {
                        "script_score": {
                            "script": {
                                "source": """
                                    // Inverse of price, normalized
                                    double price = doc['price'].value;
                                    if (price <= 0) return 1.0;
                                    return 100.0 / (price + 100.0);
                                """
                            }
                        },
                        "weight": 0.5
                    }
                ],
                "score_mode": "sum",       # Sum all function scores
                "boost_mode": "multiply",  # Multiply sum with query score
                "max_boost": 10            # Cap the boost
            }
        }
    
    def personalized_ranking(
        self,
        search_text: str,
        user_preferences: dict
    ) -> dict:
        """
        Personalized ranking based on user behavior.
        
        Boost brands/categories user has purchased from before.
        """
        
        functions = [
            # Base popularity
            {
                "field_value_factor": {
                    "field": "popularity_score",
                    "modifier": "sqrt",
                    "missing": 1
                },
                "weight": 1
            }
        ]
        
        # Boost user's preferred brands
        if user_preferences.get("preferred_brands"):
            functions.append({
                "filter": {
                    "terms": {
                        "brand.keyword": user_preferences["preferred_brands"]
                    }
                },
                "weight": 2.0
            })
        
        # Boost user's preferred categories
        if user_preferences.get("preferred_categories"):
            functions.append({
                "filter": {
                    "terms": {
                        "category": user_preferences["preferred_categories"]
                    }
                },
                "weight": 1.5
            })
        
        # Boost user's price range
        if user_preferences.get("price_range"):
            price_range = user_preferences["price_range"]
            functions.append({
                "filter": {
                    "range": {
                        "price": {
                            "gte": price_range.get("min", 0),
                            "lte": price_range.get("max", 99999)
                        }
                    }
                },
                "weight": 1.3
            })
        
        return {
            "function_score": {
                "query": {
                    "multi_match": {
                        "query": search_text,
                        "fields": ["name^3", "brand^2", "description"]
                    }
                },
                "functions": functions,
                "score_mode": "sum",
                "boost_mode": "multiply"
            }
        }

# queries/multi_match.py

"""
Multi-match query types and when to use each.
"""


class MultiMatchExplainer:
    """
    Explains different multi_match types.
    """
    
    def best_fields(self, search_text: str) -> dict:
        """
        best_fields (default)
        
        Finds documents where ANY field matches.
        Final score = highest scoring field.
        
        Best for: Searching where match in one field is enough.
        Example: Product search (name OR description)
        """
        
        return {
            "multi_match": {
                "query": search_text,
                "fields": ["name^3", "description"],
                "type": "best_fields",
                "tie_breaker": 0.3  # Give some credit to other fields
            }
        }
        
        # Example:
        # Query: "laptop"
        # Doc A: name="Laptop Stand", description="Holds your computer"
        # Doc B: name="Computer Desk", description="Perfect for your laptop"
        # 
        # best_fields:
        # Doc A score = max(name_score, desc_score) = name_score (higher)
        # Doc B score = max(name_score, desc_score) = desc_score
        # 
        # With tie_breaker=0.3:
        # Doc A score = name_score + 0.3 * desc_score
    
    def most_fields(self, search_text: str) -> dict:
        """
        most_fields
        
        Finds documents matching in MULTIPLE fields.
        Final score = sum of all field scores.
        
        Best for: When matching across fields indicates better match.
        Example: Documents with same text in multiple languages.
        """
        
        return {
            "multi_match": {
                "query": search_text,
                "fields": [
                    "name", 
                    "name.stemmed",    # Stemmed version
                    "name.shingles"    # Word combinations
                ],
                "type": "most_fields"
            }
        }
        
        # Example:
        # Query: "running shoes"
        # Fields: name, name.stemmed, name.shingles
        # 
        # most_fields:
        # Score = name_score + stemmed_score + shingles_score
        # Doc matching in all 3 analyses scores highest
    
    def cross_fields(self, search_text: str) -> dict:
        """
        cross_fields
        
        Treats multiple fields as ONE big field.
        Query terms can match across different fields.
        
        Best for: Person names, addresses split across fields.
        Example: first_name="John", last_name="Smith", query="John Smith"
        """
        
        return {
            "multi_match": {
                "query": search_text,
                "fields": ["first_name", "last_name"],
                "type": "cross_fields",
                "operator": "and"  # All terms must match (across fields)
            }
        }
        
        # Example:
        # Query: "John Smith"
        # Doc: first_name="John", last_name="Smith"
        # 
        # best_fields: Would need "John" AND "Smith" in same field - NO MATCH
        # cross_fields: "John" in first_name, "Smith" in last_name - MATCH!
    
    def phrase(self, search_text: str) -> dict:
        """
        phrase
        
        Searches for exact phrase in each field.
        
        Best for: When word order matters.
        Example: with slop 0 (default), "New York" does not match "York New";
        a transposition needs slop 2, so the slop: 2 below would match it
        """
        
        return {
            "multi_match": {
                "query": search_text,
                "fields": ["name^3", "description"],
                "type": "phrase",
                "slop": 2  # Allow up to 2 words between
            }
        }
    
    def phrase_prefix(self, search_text: str) -> dict:
        """
        phrase_prefix
        
        Like phrase, but last term is treated as prefix.
        
        Best for: Autocomplete / search-as-you-type.
        Example: "quick bro" matches "quick brown fox"
        """
        
        return {
            "multi_match": {
                "query": search_text,
                "fields": ["name^3", "description"],
                "type": "phrase_prefix",
                "max_expansions": 50  # Limit prefix expansions
            }
        }
    
    def bool_prefix(self, search_text: str) -> dict:
        """
        bool_prefix
        
        Each term is a term query except the last, which is a prefix;
        combined with OR by default (set operator "and" to require all).
        
        Best for: Autocomplete where order doesn't matter.
        Example: "quick bro" matches "brown quick", "quick brown"
        """
        
        return {
            "multi_match": {
                "query": search_text,
                "fields": ["name^3", "description"],
                "type": "bool_prefix"
            }
        }

# queries/no_results.py

"""
Strategies for handling no results or poor results.

Nothing frustrates users more than "No results found"
when they know the product exists.
"""

from typing import List, Optional, Tuple
from dataclasses import dataclass


@dataclass
class SearchAttempt:
    """Record of a search attempt."""
    query: dict
    strategy: str
    result_count: int
    top_score: Optional[float]


class ResilientSearcher:
    """
    Implements fallback strategies for better results.
    """
    
    def __init__(self, es_client, index: str):
        self.es = es_client
        self.index = index
    
    async def search_with_fallback(
        self,
        search_text: str,
        filters: dict,
        min_results: int = 5
    ) -> Tuple[dict, List[SearchAttempt]]:
        """
        Search with progressive relaxation.
        
        Strategy:
        1. Try exact query with all filters
        2. Relax filters progressively
        3. Try fuzzy matching
        4. Try partial terms
        5. Suggest alternatives
        """
        
        attempts = []
        
        # Strategy 1: Exact query
        query = self._build_exact_query(search_text, filters)
        result = await self._execute_search(query)
        attempts.append(SearchAttempt(
            query=query,
            strategy="exact",
            result_count=result["hits"]["total"]["value"],
            top_score=self._get_top_score(result)
        ))
        
        if result["hits"]["total"]["value"] >= min_results:
            return result, attempts
        
        # Strategy 2: Relax non-essential filters
        if filters:
            relaxed_filters = self._relax_filters(filters)
            query = self._build_exact_query(search_text, relaxed_filters)
            result = await self._execute_search(query)
            attempts.append(SearchAttempt(
                query=query,
                strategy="relaxed_filters",
                result_count=result["hits"]["total"]["value"],
                top_score=self._get_top_score(result)
            ))
            
            if result["hits"]["total"]["value"] >= min_results:
                return result, attempts
        
        # Strategy 3: Fuzzy matching (typo tolerance)
        query = self._build_fuzzy_query(search_text, filters)
        result = await self._execute_search(query)
        attempts.append(SearchAttempt(
            query=query,
            strategy="fuzzy",
            result_count=result["hits"]["total"]["value"],
            top_score=self._get_top_score(result)
        ))
        
        if result["hits"]["total"]["value"] >= min_results:
            return result, attempts
        
        # Strategy 4: OR instead of AND
        query = self._build_any_term_query(search_text, filters)
        result = await self._execute_search(query)
        attempts.append(SearchAttempt(
            query=query,
            strategy="any_term",
            result_count=result["hits"]["total"]["value"],
            top_score=self._get_top_score(result)
        ))
        
        if result["hits"]["total"]["value"] >= min_results:
            return result, attempts
        
        # Strategy 5: No filters at all
        query = self._build_fuzzy_query(search_text, {})
        result = await self._execute_search(query)
        attempts.append(SearchAttempt(
            query=query,
            strategy="no_filters_fuzzy",
            result_count=result["hits"]["total"]["value"],
            top_score=self._get_top_score(result)
        ))
        
        return result, attempts
    
    def _build_exact_query(self, search_text: str, filters: dict) -> dict:
        """Build exact match query."""
        
        return {
            "query": {
                "bool": {
                    "must": [
                        {
                            "multi_match": {
                                "query": search_text,
                                "fields": ["name^3", "brand^2", "description"],
                                "type": "best_fields",
                                "operator": "and"  # All terms required
                            }
                        }
                    ],
                    "filter": self._build_filters(filters)
                }
            }
        }
    
    def _build_fuzzy_query(self, search_text: str, filters: dict) -> dict:
        """Build query with typo tolerance."""
        
        return {
            "query": {
                "bool": {
                    "must": [
                        {
                            "multi_match": {
                                "query": search_text,
                                "fields": ["name^3", "brand^2", "description"],
                                "type": "best_fields",
                                "fuzziness": "AUTO",  # 1 edit for 3-5 chars, 2 for 6+
                                "prefix_length": 2,    # Don't fuzzy first 2 chars
                                "operator": "and"
                            }
                        }
                    ],
                    "filter": self._build_filters(filters)
                }
            }
        }
    
    def _build_any_term_query(self, search_text: str, filters: dict) -> dict:
        """Build query where ANY term can match."""
        
        return {
            "query": {
                "bool": {
                    "must": [
                        {
                            "multi_match": {
                                "query": search_text,
                                "fields": ["name^3", "brand^2", "description"],
                                "type": "best_fields",
                                "operator": "or",      # Any term matches
                                "minimum_should_match": "50%"  # At least half
                            }
                        }
                    ],
                    "filter": self._build_filters(filters)
                }
            }
        }
    
    def _relax_filters(self, filters: dict) -> dict:
        """
        Relax filters by removing least important ones.
        
        Priority (keep first):
        1. category (high intent signal)
        2. in_stock (critical for purchase)
        3. brand (moderate intent)
        4. price_range (remove first)
        5. rating (remove first)
        """
        
        relaxed = dict(filters)
        
        # Remove in order of least importance
        for key in ["rating_min", "price_max", "price_min"]:
            if key in relaxed:
                del relaxed[key]
        
        return relaxed
    
    def _build_filters(self, filters: dict) -> list:
        """Build filter clauses."""
        
        clauses = []
        
        if filters.get("category"):
            clauses.append({"term": {"category": filters["category"]}})
        
        if filters.get("brand"):
            clauses.append({"term": {"brand.keyword": filters["brand"]}})
        
        if filters.get("in_stock") is not None:
            clauses.append({"term": {"in_stock": filters["in_stock"]}})
        
        if filters.get("price_min") or filters.get("price_max"):
            price_range = {}
            if filters.get("price_min"):
                price_range["gte"] = filters["price_min"]
            if filters.get("price_max"):
                price_range["lte"] = filters["price_max"]
            clauses.append({"range": {"price": price_range}})
        
        return clauses
    
    async def _execute_search(self, query: dict) -> dict:
        """Execute search and return results."""
        
        return await self.es.search(index=self.index, body=query, size=20)
    
    def _get_top_score(self, result: dict) -> Optional[float]:
        """Get score of top result."""
        
        hits = result.get("hits", {}).get("hits", [])
        if hits:
            return hits[0].get("_score")
        return None


# =============================================================================
# "Did You Mean" Suggestions
# =============================================================================

class SpellingSuggester:
    """
    Provides spelling suggestions for queries.
    """
    
    def __init__(self, es_client, index: str):
        self.es = es_client
        self.index = index
    
    async def get_suggestions(self, search_text: str) -> List[str]:
        """
        Get spelling suggestions using Elasticsearch suggester.
        """
        
        response = await self.es.search(
            index=self.index,
            body={
                "suggest": {
                    "text": search_text,
                    "simple_phrase": {
                        "phrase": {
                            "field": "name.trigram",
                            "size": 3,
                            "gram_size": 3,
                            "direct_generator": [{
                                "field": "name.trigram",
                                "suggest_mode": "popular"
                            }],
                            "highlight": {
                                "pre_tag": "<em>",
                                "post_tag": "</em>"
                            }
                        }
                    }
                }
            }
        )
        
        suggestions = []
        for suggestion in response.get("suggest", {}).get("simple_phrase", []):
            for option in suggestion.get("options", []):
                suggestions.append(option["text"])
        
        return suggestions

# monitoring/search_quality.py

"""
Metrics for measuring search quality.

You can't improve what you don't measure.
"""

from dataclasses import dataclass
from datetime import datetime
from typing import List, Optional


@dataclass
class SearchEvent:
    """A single search interaction."""
    
    query: str
    result_count: int
    clicked_position: Optional[int]  # Position of clicked result (1-indexed)
    clicked_product_id: Optional[str]
    time_to_click_ms: Optional[int]
    converted: bool  # Did user add to cart / purchase
    timestamp: datetime


class SearchQualityMetrics:
    """
    Calculates search quality metrics.
    """
    
    def calculate_metrics(self, events: List[SearchEvent]) -> dict:
        """
        Calculate key search quality metrics.
        """
        
        total_searches = len(events)
        
        if total_searches == 0:
            return {"error": "No events to analyze"}
        
        # Zero Result Rate
        zero_results = sum(1 for e in events if e.result_count == 0)
        zero_result_rate = zero_results / total_searches
        
        # Click-Through Rate (CTR)
        clicks = sum(1 for e in events if e.clicked_position is not None)
        ctr = clicks / total_searches
        
        # Mean Reciprocal Rank (MRR)
        # Measures how high the clicked result ranked
        reciprocal_ranks = []
        for e in events:
            if e.clicked_position:
                reciprocal_ranks.append(1.0 / e.clicked_position)
        mrr = sum(reciprocal_ranks) / len(reciprocal_ranks) if reciprocal_ranks else 0
        
        # Clicks in Top 3
        top3_clicks = sum(
            1 for e in events 
            if e.clicked_position and e.clicked_position <= 3
        )
        top3_rate = top3_clicks / clicks if clicks > 0 else 0
        
        # Conversion Rate (searches that led to purchase)
        conversions = sum(1 for e in events if e.converted)
        conversion_rate = conversions / total_searches
        
        # Average Time to Click
        click_times = [e.time_to_click_ms for e in events if e.time_to_click_ms]
        avg_time_to_click = sum(click_times) / len(click_times) if click_times else 0
        
        return {
            "total_searches": total_searches,
            "zero_result_rate": round(zero_result_rate, 4),
            "click_through_rate": round(ctr, 4),
            "mean_reciprocal_rank": round(mrr, 4),
            "top3_click_rate": round(top3_rate, 4),
            "conversion_rate": round(conversion_rate, 4),
            "avg_time_to_click_ms": round(avg_time_to_click, 0),
            "health_assessment": self._assess_health(
                zero_result_rate, ctr, mrr, top3_rate
            )
        }
    
    def _assess_health(
        self,
        zero_result_rate: float,
        ctr: float,
        mrr: float,
        top3_rate: float
    ) -> dict:
        """Assess overall search health."""
        
        # Flag any metric worse than "Good" in the quality targets table
        issues = []
        
        if zero_result_rate > 0.10:
            issues.append({
                "severity": "high",
                "metric": "zero_result_rate",
                "message": f"High zero-result rate ({zero_result_rate:.1%}). "
                           "Consider fuzzy matching, synonyms, or spell check."
            })
        
        if ctr < 0.35:
            issues.append({
                "severity": "medium",
                "metric": "click_through_rate",
                "message": f"Low CTR ({ctr:.1%}). "
                           "Results may not be relevant or snippets may be poor."
            })
        
        if mrr < 0.50:
            issues.append({
                "severity": "medium",
                "metric": "mean_reciprocal_rank",
                "message": f"Low MRR ({mrr:.2f}). "
                           "Best results not ranking high enough."
            })
        
        if top3_rate < 0.70:
            issues.append({
                "severity": "low",
                "metric": "top3_click_rate",
                "message": f"Low top-3 click rate ({top3_rate:.1%}). "
                           "Users scrolling past top results."
            })
        
        return {
            "status": "healthy" if not issues else "needs_attention",
            "issues": issues
        }

# services/search_service.py

"""
Complete search service with all features.
"""

import logging
from dataclasses import dataclass
from typing import List, Dict, Any, Optional
from datetime import datetime, timezone

logger = logging.getLogger(__name__)


@dataclass
class SearchConfig:
    """Search service configuration."""
    
    # Field boosts
    name_boost: float = 3.0
    brand_boost: float = 2.0
    description_boost: float = 1.0
    
    # Function score weights
    popularity_weight: float = 2.0
    recency_weight: float = 1.0
    rating_weight: float = 1.5
    
    # Fallback settings
    enable_fuzzy_fallback: bool = True
    min_results_for_fallback: int = 5
    
    # Pagination
    default_page_size: int = 20
    max_page_size: int = 100


@dataclass
class SearchRequest:
    """Search request from client."""
    
    query: str
    filters: Dict[str, Any] = None
    sort: str = "relevance"
    page: int = 1
    page_size: int = 20
    user_id: Optional[str] = None  # For personalization


@dataclass
class SearchResponse:
    """Search response to client."""
    
    products: List[dict]
    total: int
    page: int
    page_size: int
    facets: Dict[str, List[dict]]
    suggestions: List[str]
    query_info: dict  # Debug info about query execution
    took_ms: int


class ProductSearchService:
    """
    Complete product search service.
    
    Features:
    - Multi-field search with boosting
    - Filter/facet support
    - Function score ranking
    - Fallback strategies
    - Spell suggestions
    - Quality tracking
    """
    
    def __init__(
        self,
        es_client,
        cache,
        user_service,
        config: SearchConfig = None
    ):
        self.es = es_client
        self.cache = cache
        self.users = user_service
        self.config = config or SearchConfig()
        
        self.index = "products"
    
    async def search(self, request: SearchRequest) -> SearchResponse:
        """
        Execute product search.
        """
        
        start_time = datetime.now(timezone.utc)
        query_info = {"strategies_tried": []}
        
        # Get user preferences for personalization
        user_prefs = None
        if request.user_id:
            user_prefs = await self.users.get_preferences(request.user_id)
        
        # Build and execute query
        query = self._build_query(request, user_prefs)
        
        result = await self.es.search(
            index=self.index,
            body=query
        )
        query_info["strategies_tried"].append("primary")
        
        # Check if we need fallback
        hit_count = result["hits"]["total"]["value"]
        
        if (hit_count < self.config.min_results_for_fallback 
            and self.config.enable_fuzzy_fallback):
            
            # Try fuzzy search
            fuzzy_query = self._build_fuzzy_query(request)
            fuzzy_result = await self.es.search(
                index=self.index,
                body=fuzzy_query
            )
            query_info["strategies_tried"].append("fuzzy")
            
            if fuzzy_result["hits"]["total"]["value"] > hit_count:
                result = fuzzy_result
                query_info["used_fallback"] = True
        
        # Get suggestions if few/no results
        suggestions = []
        if hit_count < 10:
            suggestions = await self._get_suggestions(request.query)
            query_info["suggestions_fetched"] = True
        
        # Process response
        products = self._process_hits(result["hits"]["hits"])
        facets = self._process_aggregations(result.get("aggregations", {}))
        
        took_ms = int((datetime.now(timezone.utc) - start_time).total_seconds() * 1000)
        
        return SearchResponse(
            products=products,
            total=result["hits"]["total"]["value"],
            page=request.page,
            page_size=request.page_size,
            facets=facets,
            suggestions=suggestions,
            query_info=query_info,
            took_ms=took_ms
        )
    
    def _build_query(
        self,
        request: SearchRequest,
        user_prefs: Optional[dict]
    ) -> dict:
        """Build the main search query."""
        
        # Core search query
        must_query = {
            "multi_match": {
                "query": request.query,
                "fields": [
                    f"name^{self.config.name_boost}",
                    f"brand^{self.config.brand_boost}",
                    f"description^{self.config.description_boost}",
                    "tags"
                ],
                "type": "best_fields",
                "tie_breaker": 0.3,
                "operator": "and"
            }
        }
        
        # Build filters
        filters = self._build_filters(request.filters or {})
        
        # Always filter to in-stock
        filters.append({"term": {"in_stock": True}})
        
        # Build function score
        functions = self._build_score_functions(user_prefs)
        
        # Build should clauses (phrase boost)
        should = [
            {
                "match_phrase": {
                    "name": {
                        "query": request.query,
                        "boost": 2.0,
                        "slop": 1
                    }
                }
            }
        ]
        
        # Complete query
        query = {
            "query": {
                "function_score": {
                    "query": {
                        "bool": {
                            "must": [must_query],
                            "should": should,
                            "filter": filters
                        }
                    },
                    "functions": functions,
                    "score_mode": "sum",
                    "boost_mode": "multiply",
                    "max_boost": 10
                }
            },
            "from": (request.page - 1) * request.page_size,
            "size": min(request.page_size, self.config.max_page_size),
            "aggs": self._build_aggregations(),
            "highlight": {
                "fields": {
                    "name": {},
                    "description": {"fragment_size": 150}
                }
            }
        }
        
        # Add sorting if not relevance
        if request.sort != "relevance":
            query["sort"] = self._build_sort(request.sort)
        
        return query
    
    def _build_filters(self, filters: dict) -> list:
        """Build filter clauses."""
        
        clauses = []
        
        if filters.get("category"):
            clauses.append({"term": {"category": filters["category"]}})
        
        if filters.get("brand"):
            brands = filters["brand"]
            if isinstance(brands, list):
                clauses.append({"terms": {"brand.keyword": brands}})
            else:
                clauses.append({"term": {"brand.keyword": brands}})
        
        if filters.get("price_min") or filters.get("price_max"):
            price_range = {}
            if filters.get("price_min"):
                price_range["gte"] = filters["price_min"]
            if filters.get("price_max"):
                price_range["lte"] = filters["price_max"]
            clauses.append({"range": {"price": price_range}})
        
        if filters.get("rating_min"):
            clauses.append({
                "range": {"rating": {"gte": filters["rating_min"]}}
            })
        
        if filters.get("color"):
            clauses.append({"term": {"colors": filters["color"]}})
        
        return clauses
    
    def _build_score_functions(self, user_prefs: Optional[dict]) -> list:
        """Build function score functions."""
        
        functions = [
            # Popularity boost
            {
                "field_value_factor": {
                    "field": "popularity_score",
                    "modifier": "log1p",
                    "missing": 0
                },
                "weight": self.config.popularity_weight
            },
            
            # Rating quality
            {
                "script_score": {
                    "script": {
                        "source": """
                            double rating = doc['rating'].value;
                            double reviews = doc['review_count'].value;
                            double confidence = Math.min(1.0, reviews / 50.0);
                            return rating * confidence * 0.2;
                        """
                    }
                },
                "weight": self.config.rating_weight
            },
            
            # Recency
            {
                "gauss": {
                    "created_at": {
                        "origin": "now",
                        "scale": "90d",
                        "decay": 0.5
                    }
                },
                "weight": self.config.recency_weight
            },
            
            # Featured products
            {
                "filter": {"term": {"is_featured": True}},
                "weight": 3.0
            }
        ]
        
        # Add personalization if available
        if user_prefs:
            if user_prefs.get("preferred_brands"):
                functions.append({
                    "filter": {
                        "terms": {"brand.keyword": user_prefs["preferred_brands"]}
                    },
                    "weight": 1.5
                })
            
            if user_prefs.get("preferred_categories"):
                functions.append({
                    "filter": {
                        "terms": {"category": user_prefs["preferred_categories"]}
                    },
                    "weight": 1.3
                })
        
        return functions
    
    def _build_fuzzy_query(self, request: SearchRequest) -> dict:
        """Build fuzzy fallback query."""
        
        return {
            "query": {
                "bool": {
                    "must": [
                        {
                            "multi_match": {
                                "query": request.query,
                                "fields": [
                                    f"name^{self.config.name_boost}",
                                    f"brand^{self.config.brand_boost}",
                                    "description"
                                ],
                                "type": "best_fields",
                                "fuzziness": "AUTO",
                                "prefix_length": 2
                            }
                        }
                    ],
                    "filter": [
                        {"term": {"in_stock": True}}
                    ]
                }
            },
            "size": request.page_size
        }
    
    def _build_aggregations(self) -> dict:
        """Build facet aggregations."""
        
        return {
            "categories": {
                "terms": {"field": "category", "size": 20}
            },
            "brands": {
                "terms": {"field": "brand.keyword", "size": 20}
            },
            "price_ranges": {
                "range": {
                    "field": "price",
                    "ranges": [
                        {"key": "under_25", "to": 25},
                        {"key": "25_50", "from": 25, "to": 50},
                        {"key": "50_100", "from": 50, "to": 100},
                        {"key": "100_200", "from": 100, "to": 200},
                        {"key": "over_200", "from": 200}
                    ]
                }
            },
            "ratings": {
                "range": {
                    "field": "rating",
                    "ranges": [
                        {"key": "4_and_up", "from": 4},
                        {"key": "3_and_up", "from": 3},
                        {"key": "2_and_up", "from": 2}
                    ]
                }
            },
            "colors": {
                "terms": {"field": "colors", "size": 15}
            }
        }
    
    def _build_sort(self, sort: str) -> list:
        """Build sort clause."""
        
        sort_map = {
            "price_asc": [{"price": "asc"}],
            "price_desc": [{"price": "desc"}],
            "rating": [{"rating": "desc"}, {"review_count": "desc"}],
            "newest": [{"created_at": "desc"}],
            "popular": [{"popularity_score": "desc"}],
        }
        
        return sort_map.get(sort, [{"_score": "desc"}])
    
    def _process_hits(self, hits: list) -> list:
        """Process search hits into response format."""
        
        products = []
        for hit in hits:
            product = {
                "product_id": hit["_id"],
                "score": hit.get("_score"),
                **hit["_source"]
            }
            
            # Add highlighting if available
            if hit.get("highlight"):
                product["highlights"] = hit["highlight"]
            
            products.append(product)
        
        return products
    
    def _process_aggregations(self, aggs: dict) -> dict:
        """Process aggregations into facets."""
        
        facets = {}
        
        for agg_name, agg_data in aggs.items():
            if "buckets" in agg_data:
                facets[agg_name] = [
                    {"value": b["key"], "count": b["doc_count"]}
                    for b in agg_data["buckets"]
                ]
        
        return facets
    
    async def _get_suggestions(self, query: str) -> list:
        """Get spelling suggestions."""
        
        response = await self.es.search(
            index=self.index,
            body={
                "suggest": {
                    "text": query,
                    "phrase_suggest": {
                        "phrase": {
                            "field": "name.trigram",
                            "size": 3,
                            "gram_size": 3,
                            "direct_generator": [{
                                "field": "name.trigram",
                                "suggest_mode": "popular"
                            }]
                        }
                    }
                }
            }
        )
        
        suggestions = []
        for suggest in response.get("suggest", {}).get("phrase_suggest", []):
            for option in suggest.get("options", []):
                if option["text"] != query:
                    suggestions.append(option["text"])
        
        return suggestions[:3]