#!/usr/bin/env python3

from __future__ import annotations

import gzip
import html
import json
import random
import re
import ssl
import threading
import time
import zlib
from dataclasses import dataclass, replace
from html.parser import HTMLParser
from pathlib import Path
from typing import Any, Callable
from urllib import robotparser
from urllib.error import HTTPError, URLError
from urllib.parse import parse_qsl, quote, quote_plus, unquote, urlencode, urljoin, urlparse, urlunparse
from urllib.request import Request, urlopen


APP_DIR = Path(__file__).resolve().parent
URL_LIST_FILE = APP_DIR / "https-liste.txt"
DEFAULT_URL_LIST_ID = "vw"
URL_LISTS: dict[str, tuple[str, Path]] = {
    "vw": ("VW-Oldtimer", URL_LIST_FILE),
    "citroen": ("Citroen Oldtimer", APP_DIR / "citroen-liste.txt"),
    "porsche": ("Porsche Oldtimer", APP_DIR / "porsche-liste.txt"),
}
BOT_NAME = "DeinPartFinder"
USER_AGENT = (
    f"{BOT_NAME}/1.0 "
    "(+https://wedema.de/finder/bot/; contact: finder@wedema.de)"
)
REQUEST_DELAY_RANGE = (0.15, 0.4)
MAX_ROBOTS_CRAWL_DELAY = 10.0
ROBOTS_CACHE_TTL = 6 * 60 * 60
ROBOTS_TIMEOUT = 1
ROBOTS_MAX_BYTES = 500_000
_robots_cache: dict[str, tuple[float, robotparser.RobotFileParser | None]] = {}
_robots_lock = threading.Lock()

PRICE_RE = re.compile(
    r"(?:(?:ab|from)\s*)?"
    r"(?:(?P<prefix>€|EUR|CHF|USD|\$|£)\s*)?"
    r"(?P<amount>\d{1,4}(?:[.\s]\d{3})*(?:[,.]\d{2}))"
    r"\s*(?P<suffix>€|EUR|CHF|USD|\$|£|Euro|Fr\.?)?",
    re.IGNORECASE,
)
HARD_BREAK_TAG_RE = re.compile(
    r"</?(?:br|p|div|li|tr|td|th|article|section|h[1-6]|ul|ol|table)\b[^>]*>",
    re.IGNORECASE,
)
SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.IGNORECASE | re.DOTALL)
TAG_RE = re.compile(r"<[^>]+>")
WHITESPACE_RE = re.compile(r"\s+")
URL_RE = re.compile(r"\bhref\s*=\s*([\"'])(.*?)\1", re.IGNORECASE | re.DOTALL)
SITEMAP_LOC_RE = re.compile(r"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
RESULT_FRAGMENT_SPLIT_RE = re.compile(
    r"(?i)</(?:article|li|tr|section|div)>|<(?:article|li|tr|section|div)\b"
)
DIV_PRODUCT_FRAGMENT_RE = re.compile(
    r"(?is)<div\b(?=[^>]*(?:\bproduct-wrapper\b|"
    r"\bproduct-box\b|"
    r"\bproduct-container\b|"
    r"\bj-catalog-product\b|"
    r"\bos_list_wrap_all\b|"
    r"\blistingbox\b|"
    r"\blistingrow\b|"
    r"itemtype\s*=\s*['\"]https?://schema\.org/Product['\"]))[^>]*>.*?"
    r"(?=<div\b(?=[^>]*(?:\bproduct-wrapper\b|"
    r"\bproduct-box\b|"
    r"\bproduct-container\b|"
    r"\bj-catalog-product\b|"
    r"\bos_list_wrap_all\b|"
    r"\blistingbox\b|"
    r"\blistingrow\b|"
    r"itemtype\s*=\s*['\"]https?://schema\.org/Product['\"]))[^>]*>|$)"
)
LI_PRODUCT_FRAGMENT_RE = re.compile(
    r"(?is)<li\b(?=[^>]*\bclass\s*=\s*['\"][^'\"]*\bitem\b)[^>]*>.*?</li>"
)

SEARCH_INPUT_HINTS = (
    "search",
    "suche",
    "such",
    "query",
    "q",
    "s",
    "keyword",
    "keywords",
    "term",
    "filter_name",
    "searchparam",
    "ssearch",
    "search_query",
)
SEARCH_ACTION_HINTS = (
    "search",
    "suche",
    "such",
    "catalogsearch",
    "advanced_search",
    "product/search",
)
IGNORED_NAME_LINES = (
    "warenkorb",
    "in den warenkorb",
    "cart",
    "add to cart",
    "inkl",
    "mwst",
    "ust",
    "tax",
    "versand",
    "shipping",
    "lieferzeit",
    "delivery",
    "details",
    "mehr",
    "anzeigen",
    "quick view",
    "compare",
    "wishlist",
    "konto",
    "login",
    "information:",
)
GENERIC_LINK_TEXTS = {
    "artikel",
    "details",
    "mehr",
    "mehr anzeigen",
    "mehr erfahren",
    "mehr lesen",
    "zum artikel",
    "zu den artikeln",
}
SHOP_LABEL_STOP_TOKENS = {
    "artikel",
    "fuer",
    "kt",
    "neu",
    "neuer",
    "teil",
    "teile",
    "und",
    "vw",
}
ENGINE_PART_TOKENS = {
    "dichtung",
    "dichtungen",
    "kolben",
    "kopf",
    "motor",
    "nockenwelle",
    "oelkuehler",
    "ventil",
    "ventildeckel",
    "zylinder",
}
SEARCH_CONTEXT_LINE_RE = re.compile(
    r"\b(?:suche|suchen|suchbegriff|suchergebnis(?:se)?|treffer|"
    r"ergebnis(?:se)?|search|results?|query)\b"
)
FOUND_COUNT_CONTEXT_LINE_RE = re.compile(
    r"\b(?:gefunden|found)\b.*\b(?:artikel|produkte?|products?|items?)\b|"
    r"\b(?:artikel|produkte?|products?|items?)\b.*\b(?:gefunden|found)\b",
    re.IGNORECASE,
)
PAGINATION_URL_RE = re.compile(
    r"(?:[?&](?:p|page|paged|seite|start|offset|limitstart|pg|"
    r"sPage|cPage|pageNumber)=\d+\b|"
    r"/(?:page|seite)/\d+\b|seite-\d+\b)",
    re.IGNORECASE,
)
PAGINATION_WORDS = {
    "next",
    "weiter",
    "weitere",
    "naechste",
    "naechster",
    "mehr",
    "more",
}
LIKE_OPTIONAL_TOKENS = {
    "aussen",
    "front",
    "hinten",
    "hinter",
    "innen",
    "kit",
    "left",
    "links",
    "oben",
    "paar",
    "rear",
    "rechts",
    "right",
    "satz",
    "set",
    "unten",
    "vorn",
    "vorne",
}
PRODUCT_LINK_IGNORED_PARAMS = {
    "cache",
    "nocache",
}
MEDIA_URL_SUFFIXES = (
    ".avif",
    ".gif",
    ".jpeg",
    ".jpg",
    ".png",
    ".svg",
    ".webp",
)
EXACT_SEARCH_HINTS = {"q", "s"}
SITEMAP_FETCH_LIMIT = 4


def search_hint_score(haystack: str, field_type: str = "") -> int:
    score = 100 if field_type == "search" else 0
    tokens = set(haystack.split())
    for hint in SEARCH_INPUT_HINTS:
        if hint in EXACT_SEARCH_HINTS:
            if hint in tokens:
                score += 80
            continue
        if hint == haystack or hint in tokens:
            score += 80
        elif hint in haystack:
            score += 40
    return score


@dataclass(frozen=True)
class SearchConfig:
    query: str
    mode: str
    timeout: int
    max_workers: int
    max_pages_per_site: int
    site_deadline_seconds: int = 90
    vehicle_type: str = ""
    url_list_id: str = DEFAULT_URL_LIST_ID
    report_error: Callable[[str], None] | None = None

    @property
    def effective_query(self) -> str:
        query = normalize_search_query(self.query)
        if self.mode == "name_like":
            query = " ".join(like_search_terms(query)) or query
        return clean_line(" ".join(part for part in (query, self.vehicle_type) if part))

    @property
    def match_mode(self) -> str:
        if self.mode in {"exact", "name"} and is_article_number_query(self.query):
            return "article"
        return self.mode


@dataclass(frozen=True)
class ShopProfile:
    """Per-shop policy for known special cases.

    The generic discovery engine remains the default. Profiles may additionally
    provide a small number of known public category routes. These are tried
    before guessed search URLs and are still subject to robots.txt.
    """

    allow_generic_fallback: bool = True
    use_sitemap_fallback: bool = True
    stop_on_homepage_403: bool = False
    stop_on_homepage_robots: bool = False
    stop_on_search_403: bool = False
    try_host_variants: bool = True
    quiet_expected_failures: bool = False
    try_direct_routes_first: bool = False
    site_deadline_seconds: int | None = None
    max_pages_per_site: int | None = None


@dataclass(frozen=True)
class DirectRoute:
    query_tokens: tuple[str, ...]
    urls: tuple[str, ...]
    vehicle_tokens: tuple[str, ...] = ()
    modes: tuple[str, ...] = ("exact", "name", "name_like")


DEFAULT_SHOP_PROFILE = ShopProfile()

# Only shops with observed special behaviour need an entry here.
# All other shops continue to use automatic search-form discovery.
SHOP_PROFILES: dict[str, ShopProfile] = {
    # Search form exists, but the discovered search route is blocked by robots.
    # Prefer sitemap/product discovery instead of trying many guessed endpoints.
    "busteileshop.de": ShopProfile(allow_generic_fallback=False, site_deadline_seconds=12, max_pages_per_site=1),
    "ahnendorp.com": ShopProfile(
        allow_generic_fallback=False,
        try_direct_routes_first=True,
        site_deadline_seconds=12,
        max_pages_per_site=2,
    ),
    "paruzzi.com": ShopProfile(allow_generic_fallback=False, site_deadline_seconds=12, max_pages_per_site=1),
    "werk34.de": ShopProfile(
        allow_generic_fallback=False,
        try_direct_routes_first=True,
        site_deadline_seconds=12,
        max_pages_per_site=2,
    ),
    "centralbestellung.de": ShopProfile(allow_generic_fallback=False, site_deadline_seconds=10, max_pages_per_site=1),
    "kabel-schmidt.de": ShopProfile(allow_generic_fallback=False, site_deadline_seconds=10, max_pages_per_site=1),

    # These shops currently reject the crawler at the homepage/WAF level.
    # Do not hammer alternate hosts and guessed search paths after a 403.
    "csp-shop.de": ShopProfile(
        allow_generic_fallback=False,
        use_sitemap_fallback=False,
        stop_on_homepage_403=True,
        try_host_variants=False,
        quiet_expected_failures=True,
        try_direct_routes_first=True,
        site_deadline_seconds=6,
        max_pages_per_site=1,
    ),
    "hoffmann-speedster.com": ShopProfile(
        allow_generic_fallback=False,
        use_sitemap_fallback=False,
        stop_on_homepage_403=True,
        try_host_variants=False,
        quiet_expected_failures=True,
        site_deadline_seconds=6,
    ),
    "nlavw.com": ShopProfile(
        allow_generic_fallback=False,
        use_sitemap_fallback=False,
        stop_on_homepage_403=True,
        try_host_variants=False,
        quiet_expected_failures=True,
        site_deadline_seconds=6,
    ),
    "mister-johns-volksshop.de": ShopProfile(
        stop_on_search_403=True,
        quiet_expected_failures=True,
        site_deadline_seconds=8,
        max_pages_per_site=1,
    ),

    # Root access itself is denied by robots.txt; do not try host variants or
    # guessed search endpoints afterwards.
    "vw-entfallteiledienst.de": ShopProfile(
        allow_generic_fallback=False,
        use_sitemap_fallback=False,
        stop_on_homepage_robots=True,
        try_host_variants=False,
        quiet_expected_failures=True,
        site_deadline_seconds=6,
    ),

    # Known timeout candidates: fail quickly so one shop cannot dominate the
    # complete search duration.
    "kieftenklok.nl": ShopProfile(site_deadline_seconds=7, max_pages_per_site=1),
    "heckkraftmotors.de": ShopProfile(site_deadline_seconds=7, max_pages_per_site=1),
    "kummetat-tuning.de": ShopProfile(site_deadline_seconds=7, max_pages_per_site=1),
    "reprowesty.com": ShopProfile(site_deadline_seconds=7, max_pages_per_site=1),
    "vwbusshop.de": ShopProfile(site_deadline_seconds=7, max_pages_per_site=1),
    "vwbuswerkstatt.ch": ShopProfile(site_deadline_seconds=7, max_pages_per_site=1),
}


def shop_profile_for_url(url: str) -> ShopProfile:
    host = urlparse(url).netloc.lower().split("@")[-1].split(":", 1)[0]
    if host.startswith("www."):
        host = host[4:]
    return SHOP_PROFILES.get(host, DEFAULT_SHOP_PROFILE)


SHOP_DIRECT_ROUTES: dict[str, tuple[DirectRoute, ...]] = {
    # Known public category pages. They are not a robots.txt bypass: every URL
    # still passes robots_allows()/fetch_request().
    "werk34.de": (
        DirectRoute(
            query_tokens=("vergaser",),
            vehicle_tokens=("t2",),
            urls=(
                "https://www.werk34.de/de/luftgekuehlt/bus-t2/motor/vergaser-reparatursaetze-einspritzduesen-gluehkerzen/vergaser/",
            ),
        ),
    ),
    "ahnendorp.com": (
        DirectRoute(
            query_tokens=("vergaser",),
            vehicle_tokens=("t2",),
            urls=(
                "https://www.ahnendorp.com/VW-Kaefer-Typ-1-Motorenteile-und-Bearbeitungen/Vergaseranlagen--Ansaugrohre--Zubehoer-48/?language=de",
                "https://www.ahnendorp.com/VW-Typ-4---Porsche-914-Motorenteile-und-Bearbeitungen/Vergaseranlagen--Ansaugrohre--Zubehoer/?language=de",
            ),
        ),
    ),
    "csp-shop.de": (
        DirectRoute(
            query_tokens=("vergaser",),
            urls=(
                "https://www.csp-shop.de/motor/kraftstoffversorgung/vergaser/",
            ),
        ),
    ),
}


def normalized_profile_host(url: str) -> str:
    host = urlparse(url).netloc.lower().split("@")[-1].split(":", 1)[0]
    return host[4:] if host.startswith("www.") else host


def direct_route_requests(url: str, config: SearchConfig) -> list[SearchRequest]:
    """Return narrowly targeted public category routes for a known shop.

    Direct routes are intentionally sparse. If no rule matches, the normal
    automatic discovery engine remains responsible for the shop.
    """
    host = normalized_profile_host(url)
    rules = SHOP_DIRECT_ROUTES.get(host, ())
    if not rules or config.mode == "article":
        return []

    query = normalize_text(config.query)
    vehicle = normalize_text(config.vehicle_type)
    requests: list[SearchRequest] = []
    seen: set[tuple[str, str, bytes | None]] = set()

    for rule in rules:
        if config.mode not in rule.modes:
            continue
        if not all(token in query for token in rule.query_tokens):
            continue
        if rule.vehicle_tokens and vehicle and not any(token in vehicle for token in rule.vehicle_tokens):
            continue
        for target in rule.urls:
            request = SearchRequest(target)
            if request.key not in seen:
                seen.add(request.key)
                requests.append(request)
    return requests


@dataclass(frozen=True)
class SearchRequest:
    url: str
    method: str = "GET"
    data: bytes | None = None

    @property
    def key(self) -> tuple[str, str, bytes | None]:
        return self.method, self.url, self.data


class RobotsDeniedError(PermissionError):
    def __init__(self, url: str) -> None:
        self.url = url
        super().__init__(f"robots.txt erlaubt keinen Zugriff auf {url}")


@dataclass
class ProductHit:
    site: str
    name: str
    availability: str
    price: str
    shipping: str
    url: str
    source: str

    def as_row(self) -> list[str]:
        return [
            self.site,
            self.name,
            self.availability,
            self.price,
            self.url,
        ]


def normalize_search_query(value: str) -> str:
    value = clean_line(value)
    value = re.sub(r"(?i)maschienen", "maschinen", value)
    value = re.sub(r"(?i)maschiene", "maschine", value)
    return value


def normalize_text(value: str) -> str:
    value = html.unescape(normalize_search_query(value)).lower()
    replacements = {
        "ä": "ae",
        "ö": "oe",
        "ü": "ue",
        "ß": "ss",
        "&": " und ",
    }
    for old, new in replacements.items():
        value = value.replace(old, new)
    value = re.sub(r"\b(?:kafer|keafer)\b", "kaefer", value)
    return WHITESPACE_RE.sub(" ", value).strip()


def compact_article(value: str) -> str:
    return re.sub(r"[^a-z0-9]", "", normalize_text(value))


def is_article_number_query(value: str) -> bool:
    compacted = compact_article(value)
    digit_count = sum(1 for char in compacted if char.isdigit())
    letter_count = sum(1 for char in compacted if char.isalpha())
    return len(compacted) >= 6 and digit_count >= 6 and letter_count <= 3


def query_tokens(query: str) -> list[str]:
    return [part for part in re.split(r"[^a-z0-9]+", normalize_text(query)) if len(part) >= 2]


def category_tokens(value: str) -> list[str]:
    return [
        part
        for part in re.split(r"[^a-z0-9]+", normalize_text(value))
        if len(part) >= 2 or part.isdigit()
    ]


def query_terms(query: str) -> list[str]:
    return [
        part
        for part in re.split(r"[^0-9A-Za-zÄÖÜäöüß]+", clean_line(query))
        if len(normalize_text(part)) >= 2
    ]


def like_search_terms(query: str) -> list[str]:
    terms = query_terms(query)
    primary_terms = [
        term for term in terms if normalize_text(term) not in LIKE_OPTIONAL_TOKENS
    ]
    return primary_terms or terms


def like_query_tokens(query: str) -> list[str]:
    tokens = query_tokens(query)
    primary_tokens = [token for token in tokens if token not in LIKE_OPTIONAL_TOKENS]
    return primary_tokens or tokens


def token_variants(token: str) -> list[str]:
    variants = [token]
    if len(token) > 6 and token.endswith("ungen"):
        variants.append(token[:-2])
    return list(dict.fromkeys(variants))


def contains_phrase(haystack: str, query: str, wildcard: bool = False) -> bool:
    normalized_haystack = normalize_text(haystack)
    normalized_query = normalize_text(query)
    if not normalized_query:
        return False
    if wildcard:
        return normalized_query in normalized_haystack

    pattern = r"(?<![a-z0-9])" + re.escape(normalized_query) + r"(?![a-z0-9])"
    if re.search(pattern, normalized_haystack) is not None:
        return True

    tokens = query_tokens(normalized_query)
    if len(tokens) == 1 and tokens[0] == normalized_query:
        for variant in token_variants(tokens[0]):
            if variant == normalized_query:
                continue
            variant_pattern = r"(?<![a-z0-9])" + re.escape(variant) + r"(?![a-z0-9])"
            if re.search(variant_pattern, normalized_haystack) is not None:
                return True
    return False


def contains_like_query(haystack: str, query: str) -> bool:
    normalized_haystack = normalize_text(haystack)
    normalized_query = normalize_text(query)
    if not normalized_query:
        return False
    if normalized_query in normalized_haystack:
        return True

    tokens = like_query_tokens(query)
    return bool(tokens) and all(
        any(variant in normalized_haystack for variant in token_variants(token))
        for token in tokens
    )


def contains_vehicle_token(haystack: str, token: str) -> bool:
    normalized_haystack = normalize_text(haystack)
    if re.fullmatch(r"t\d+", token):
        pattern = r"(?<![a-z0-9])" + re.escape(token) + r"[a-z]?(?![a-z0-9])"
        return re.search(pattern, normalized_haystack) is not None
    return contains_phrase(normalized_haystack, token)


def matches_vehicle_type(haystack: str, vehicle_type: str) -> bool:
    vehicle_tokens = query_tokens(vehicle_type)
    return bool(vehicle_tokens) and all(
        contains_vehicle_token(haystack, token) for token in vehicle_tokens
    )


def matches_query(name: str, query: str, mode: str, *extra_values: str) -> bool:
    haystack = " ".join([name, *extra_values])
    if mode == "article":
        needle = compact_article(query)
        if not needle:
            return False
        return needle in compact_article(haystack)
    if mode == "name_like":
        return contains_like_query(haystack, query)

    return contains_phrase(haystack, query)


def matches_product(name: str, config: SearchConfig, *extra_values: str) -> bool:
    match_mode = config.match_mode
    query_context = extra_values if match_mode in {"article", "name_like"} else ()
    if not matches_query(name, config.query, match_mode, *query_context):
        return False
    if not config.vehicle_type:
        return True

    return matches_vehicle_type(" ".join([name, *extra_values]), config.vehicle_type)


def is_search_context_line(
    line: str,
    config: SearchConfig | None = None,
    *,
    ignore_exact_query: bool = True,
) -> bool:
    normalized = normalize_text(line)
    if SEARCH_CONTEXT_LINE_RE.search(normalized):
        return True
    if FOUND_COUNT_CONTEXT_LINE_RE.search(normalized):
        return True
    if config is not None and ignore_exact_query:
        effective_query = normalize_text(config.effective_query)
        if effective_query and normalized == effective_query:
            return True
        if effective_query and normalized.startswith(f"{effective_query} ") and " shop" in normalized:
            return True
    return False


def is_generic_link_text(text: str) -> bool:
    normalized = normalize_text(text).strip("()[]{}")
    return not normalized or normalized in GENERIC_LINK_TEXTS


def clean_line(value: str) -> str:
    return WHITESPACE_RE.sub(" ", html.unescape(value or "")).strip(" \t\r\n|,-")


ATTRIBUTE_FRAGMENT_RE = re.compile(
    r"\b(?:aria-[\w-]+|class|data-[\w-]+|href|id|style|title)\s*=",
    re.IGNORECASE,
)
TITLE_ATTR_RE = re.compile(r"\btitle\s*=\s*([\"'])(.*?)\1", re.IGNORECASE)
MARKUP_ARTIFACT_LINE_RE = re.compile(
    r"^(?:[a-z0-9_:-]+\s*=\s*['\"]|>)",
    re.IGNORECASE,
)


def clean_availability_line(value: str) -> str:
    line = clean_line(value)
    if ">" in line and ATTRIBUTE_FRAGMENT_RE.search(line.split(">", 1)[0]):
        visible_text = clean_line(line.rsplit(">", 1)[-1])
        if visible_text:
            return visible_text

    title_match = TITLE_ATTR_RE.search(line)
    if title_match:
        return clean_line(title_match.group(2))

    return line


def is_markup_artifact_line(value: str) -> bool:
    line = clean_line(value)
    return bool(MARKUP_ARTIFACT_LINE_RE.search(line) or ATTRIBUTE_FRAGMENT_RE.search(line))


def html_to_lines(markup: str) -> list[str]:
    markup = SCRIPT_STYLE_RE.sub(" ", markup or "")
    markup = HARD_BREAK_TAG_RE.sub("\n", markup)
    text = TAG_RE.sub(" ", markup)
    text = html.unescape(text)
    lines = [clean_line(line) for line in text.splitlines()]
    return [line for line in lines if line]


def sanitize_url(url: str) -> str:
    """Encode whitespace/control characters without destroying existing URL escapes."""
    parsed = urlparse(url)
    path = quote(parsed.path, safe="/%:@-._~!$&'()*+,;=")
    query = quote(parsed.query, safe="=&?/:;+,%@-._~!$'()*[]")
    return urlunparse(parsed._replace(path=path, query=query))


def absolute_url(base_url: str, href: str) -> str:
    href = html.unescape(href or "").strip()
    if not href or href.startswith(("javascript:", "mailto:", "tel:")):
        return ""
    return sanitize_url(urljoin(base_url, href))


def normalized_host(url: str) -> str:
    host = urlparse(url).netloc.lower()
    return host[4:] if host.startswith("www.") else host


def same_site_url(base_url: str, candidate_url: str) -> bool:
    base_host = normalized_host(base_url)
    candidate_host = normalized_host(candidate_url)
    return bool(base_host and candidate_host and base_host == candidate_host)


def canonical_page_url(url: str) -> str:
    parsed = urlparse(url)
    return urlunparse(parsed._replace(fragment=""))


def strip_url_query_params(url: str, ignored_params: set[str]) -> str:
    parsed = urlparse(url)
    if not parsed.query:
        return url

    parsed_params = parse_qsl(parsed.query, keep_blank_values=True)
    params = [
        (key, value)
        for key, value in parsed_params
        if key.lower() not in ignored_params
    ]
    if len(params) == len(parsed_params):
        return url
    return urlunparse(parsed._replace(query=urlencode(params, doseq=True)))


def is_shop_action_url(url: str) -> bool:
    parsed = urlparse(url)
    path = parsed.path.lower()
    params = dict(parse_qsl(parsed.query, keep_blank_values=True))
    action = params.get("action", "").lower()
    if action in {"buy_now", "add_product"} or "BUYproducts_id" in params:
        return True
    if path.startswith("/j/shop/"):
        return True
    return any(
        part in path
        for part in (
            "checkout",
            "login",
            "popup_content",
            "shopping_cart",
        )
    )


def is_view_switch_url(url: str) -> bool:
    params = dict(parse_qsl(urlparse(url).query, keep_blank_values=True))
    return params.get("show", "").lower() in {"box", "list"}


def is_productish_url(url: str) -> bool:
    path = urlparse(url).path.lower()
    return "/app/module/webproduct/goto/" in path


def is_shop_page_url(url: str) -> bool:
    path = urlparse(url).path.lower()
    return path.startswith("/shop/")


def is_media_url(url: str) -> bool:
    parsed = urlparse(url)
    host = parsed.netloc.lower()
    path = parsed.path.lower()
    return host.startswith(("assets.", "image.")) or path.endswith(MEDIA_URL_SUFFIXES)


def site_label(url: str) -> str:
    parsed = urlparse(url)
    return parsed.netloc.replace("www.", "") or url


def host_variants(url: str) -> list[str]:
    parsed = urlparse(url)
    host = parsed.netloc
    if not host:
        return [url]

    variants = [url]
    if host.startswith("www."):
        alt_host = host[4:]
    else:
        alt_host = f"www.{host}"

    if alt_host:
        variants.append(urlunparse(parsed._replace(netloc=alt_host)))
    return list(dict.fromkeys(variants))


def load_urls(path: Path = URL_LIST_FILE) -> list[str]:
    if not path.exists():
        return []
    urls: list[str] = []
    for raw_line in path.read_text(encoding="utf-8").splitlines():
        line = raw_line.strip()
        if not line or line.startswith("#"):
            continue
        if not line.startswith(("http://", "https://")):
            line = f"https://{line}"
        urls.append(line)
    return list(dict.fromkeys(urls))


def normalize_url_list_id(value: str | None) -> str:
    candidate = str(value or "").strip().lower()
    return candidate if candidate in URL_LISTS else DEFAULT_URL_LIST_ID


def url_list_path(url_list_id: str | None = None) -> Path:
    normalized = normalize_url_list_id(url_list_id)
    return URL_LISTS[normalized][1]


def url_list_label(url_list_id: str | None = None) -> str:
    normalized = normalize_url_list_id(url_list_id)
    return URL_LISTS[normalized][0]


def url_list_choices() -> list[tuple[str, str]]:
    return [(key, label) for key, (label, _path) in URL_LISTS.items()]


def load_urls_for_catalog(url_list_id: str | None = None) -> list[str]:
    return load_urls(url_list_path(url_list_id))


class PageParser(HTMLParser):
    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self.forms: list[dict[str, Any]] = []
        self.links: list[tuple[str, str]] = []
        self.next_links: list[str] = []
        self.headings: list[str] = []
        self.scripts: list[tuple[str, str]] = []
        self.meta_tags: list[dict[str, str]] = []
        self.link_tags: list[dict[str, str]] = []
        self.base_href = ""
        self.title = ""
        self._current_form: dict[str, Any] | None = None
        self._current_link: dict[str, Any] | None = None
        self._current_heading: list[str] | None = None
        self._in_title = False
        self._title_parts: list[str] = []
        self._script_type = ""
        self._script_parts: list[str] | None = None

    @staticmethod
    def _attrs(attrs: list[tuple[str, str | None]]) -> dict[str, str]:
        return {name.lower(): value or "" for name, value in attrs}

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        tag = tag.lower()
        data = self._attrs(attrs)

        if tag == "form":
            self._current_form = {
                "method": data.get("method", "GET").upper(),
                "action": data.get("action", ""),
                "inputs": [],
            }
            return

        if tag in {"input", "button", "select"} and self._current_form is not None:
            self._current_form["inputs"].append(
                {
                    "name": data.get("name", ""),
                    "value": data.get("value", ""),
                    "type": data.get("type", "text").lower(),
                    "placeholder": data.get("placeholder", ""),
                    "aria-label": data.get("aria-label", ""),
                }
            )
            return

        if tag == "a":
            self._current_link = {
                "href": data.get("href", ""),
                "rel": data.get("rel", ""),
                "label": " ".join(
                    data.get(name, "")
                    for name in ("title", "aria-label", "data-link-title")
                    if data.get(name, "")
                ),
                "text": [],
            }
            return

        if tag == "link":
            self.link_tags.append(data)
            rel_values = normalize_text(data.get("rel", "")).split()
            href = data.get("href", "")
            if href and "next" in rel_values:
                self.next_links.append(href)
            return

        if tag == "meta":
            self.meta_tags.append(data)
            return

        if tag == "base" and not self.base_href:
            self.base_href = data.get("href", "")
            return

        if tag in {"h1", "h2", "h3"}:
            self._current_heading = []
            return

        if tag == "title":
            self._in_title = True
            self._title_parts = []
            return

        if tag == "script":
            self._script_type = data.get("type", "")
            self._script_parts = []

    def handle_data(self, data: str) -> None:
        if self._script_parts is not None:
            self._script_parts.append(data)
            return
        if self._current_link is not None:
            self._current_link["text"].append(data)
        if self._current_heading is not None:
            self._current_heading.append(data)
        if self._in_title:
            self._title_parts.append(data)

    def handle_endtag(self, tag: str) -> None:
        tag = tag.lower()
        if tag == "form" and self._current_form is not None:
            self.forms.append(self._current_form)
            self._current_form = None
            return

        if tag == "a" and self._current_link is not None:
            link_text = clean_line(" ".join(self._current_link["text"]))
            label = clean_line(self._current_link.get("label", ""))
            if label and is_generic_link_text(link_text):
                link_text = label
            self.links.append((self._current_link["href"], link_text))
            rel_values = normalize_text(self._current_link.get("rel", "")).split()
            if self._current_link["href"] and "next" in rel_values:
                self.next_links.append(self._current_link["href"])
            self._current_link = None
            return

        if tag in {"h1", "h2", "h3"} and self._current_heading is not None:
            heading = clean_line(" ".join(self._current_heading))
            if heading:
                self.headings.append(heading)
            self._current_heading = None
            return

        if tag == "title" and self._in_title:
            self.title = clean_line(" ".join(self._title_parts))
            self._in_title = False
            return

        if tag == "script" and self._script_parts is not None:
            self.scripts.append((self._script_type, "".join(self._script_parts)))
            self._script_type = ""
            self._script_parts = None


def parse_page(markup: str) -> PageParser:
    parser = PageParser()
    try:
        parser.feed(markup)
    except Exception:
        pass
    return parser


def decode_response(raw: bytes, headers: Any) -> str:
    encoding = headers.get("Content-Encoding", "").lower()
    if "gzip" in encoding:
        raw = gzip.decompress(raw)
    elif "deflate" in encoding:
        try:
            raw = zlib.decompress(raw)
        except zlib.error:
            raw = zlib.decompress(raw, -zlib.MAX_WBITS)
    elif raw.startswith(b"\x1f\x8b"):
        raw = gzip.decompress(raw)

    content_type = headers.get("Content-Type", "")
    charset_match = re.search(r"charset=([\w.-]+)", content_type, re.IGNORECASE)
    charset = charset_match.group(1) if charset_match else ""
    if not charset:
        head = raw[:4096].decode("ascii", errors="ignore")
        meta_match = re.search(r"charset=[\"']?([\w.-]+)", head, re.IGNORECASE)
        charset = meta_match.group(1) if meta_match else "utf-8"

    try:
        return raw.decode(charset, errors="replace")
    except LookupError:
        return raw.decode("utf-8", errors="replace")


def robots_cache_key(url: str) -> str:
    parsed = urlparse(url)
    return f"{parsed.scheme}://{parsed.netloc}"


def robots_url(url: str) -> str:
    parsed = urlparse(url)
    return urlunparse(parsed._replace(path="/robots.txt", params="", query="", fragment=""))


def load_robots_parser(url: str, timeout: int = ROBOTS_TIMEOUT) -> robotparser.RobotFileParser | None:
    cache_key = robots_cache_key(url)
    now = time.time()
    with _robots_lock:
        cached = _robots_cache.get(cache_key)
        if cached and now - cached[0] < ROBOTS_CACHE_TTL:
            return cached[1]

    parser = robotparser.RobotFileParser(robots_url(url))
    request = Request(
        robots_url(url),
        headers={
            "User-Agent": USER_AGENT,
            "Accept": "text/plain,*/*;q=0.8",
            "Connection": "close",
        },
    )
    try:
        context = ssl.create_default_context()
        with urlopen(request, timeout=timeout, context=context) as response:
            raw = response.read(ROBOTS_MAX_BYTES)
            parser.parse(decode_response(raw, response.headers).splitlines())
    except HTTPError as exc:
        # Ein HTTP-4xx beim Abruf von /robots.txt ist kein explizites
        # robots.txt-Verbot. Nur erfolgreich geladene Regeln werden als
        # Verbote ausgewertet. Dadurch legt z. B. ein WAF-403 auf
        # /robots.txt nicht pauschal den gesamten Shop fuer Stunden still.
        if 400 <= exc.code < 500:
            parser.allow_all = True
        else:
            parser = None
    except (URLError, TimeoutError, OSError, ssl.SSLError, ValueError):
        parser = None

    with _robots_lock:
        _robots_cache[cache_key] = (now, parser)
    return parser


def robots_allows(url: str, timeout: float = ROBOTS_TIMEOUT) -> bool:
    parser = load_robots_parser(url, timeout=max(0.5, min(float(timeout), ROBOTS_TIMEOUT)))
    if parser is None:
        return True
    # robotparser erwartet fuer die Regelzuordnung den Bot-/Product-Token.
    # Der vollstaendige HTTP User-Agent beginnt mit "Mozilla/..." und
    # waere deshalb fuer User-agent: T2Finder gerade die falsche Wahl.
    return parser.can_fetch(BOT_NAME, url)


def robots_crawl_delay(url: str, timeout: float = ROBOTS_TIMEOUT) -> float | None:
    parser = load_robots_parser(url, timeout=max(0.5, min(float(timeout), ROBOTS_TIMEOUT)))
    if parser is None:
        return None
    delay = parser.crawl_delay(BOT_NAME)
    if delay is None:
        return None
    try:
        return max(0.0, float(delay))
    except (TypeError, ValueError):
        return None


def polite_request_delay(url: str, timeout: float = ROBOTS_TIMEOUT) -> float:
    delay = random.uniform(*REQUEST_DELAY_RANGE)
    crawl_delay = robots_crawl_delay(url, timeout=timeout)
    if crawl_delay is not None:
        delay = max(delay, min(crawl_delay, MAX_ROBOTS_CRAWL_DELAY))
    return delay


def notable_fetch_error(url: str, exc: BaseException) -> str | None:
    if isinstance(exc, RobotsDeniedError):
        return f"robots.txt blockiert {exc.url}"
    if isinstance(exc, HTTPError):
        target_url = getattr(exc, "url", None) or url
        if exc.code == 429:
            return f"Rate Limit (HTTP 429) bei {target_url}"
        if exc.code == 403:
            return f"Zugriff verweigert (HTTP 403) bei {target_url}"
    return None


def report_fetch_error(config: SearchConfig, url: str, exc: BaseException) -> None:
    if config.report_error is None:
        return
    message = notable_fetch_error(url, exc)
    if message:
        config.report_error(message)


def fetch_request(search_request: SearchRequest, timeout: float) -> tuple[str, str]:
    headers = {
        "User-Agent": USER_AGENT,
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "de-DE,de;q=0.9,en;q=0.6",
        "Accept-Encoding": "gzip, deflate",
        "Connection": "close",
    }
    if search_request.method == "POST":
        headers["Content-Type"] = "application/x-www-form-urlencoded"

    robots_timeout = max(0.5, min(float(timeout), ROBOTS_TIMEOUT))
    if not robots_allows(search_request.url, timeout=robots_timeout):
        raise RobotsDeniedError(search_request.url)

    time.sleep(polite_request_delay(search_request.url, timeout=robots_timeout))

    request = Request(
        search_request.url,
        data=search_request.data,
        headers=headers,
        method=search_request.method,
    )
    context = ssl.create_default_context()
    with urlopen(request, timeout=timeout, context=context) as response:
        raw = response.read(3_500_000)
        return response.geturl(), decode_response(raw, response.headers)


def fetch_url(url: str, timeout: float) -> tuple[str, str]:
    return fetch_request(SearchRequest(url=url), timeout)


def form_is_search(form: dict[str, Any]) -> bool:
    action = normalize_text(form.get("action", ""))
    if any(hint in action for hint in SEARCH_ACTION_HINTS):
        return True
    for field in form.get("inputs", []):
        haystack = normalize_text(
            " ".join(
                [
                    field.get("name", ""),
                    field.get("type", ""),
                    field.get("placeholder", ""),
                    field.get("aria-label", ""),
                ]
            )
        )
        if search_hint_score(haystack, field.get("type", "")):
            return True
    return False


def choose_search_field(form: dict[str, Any]) -> str:
    candidates: list[tuple[int, str]] = []
    for field in form.get("inputs", []):
        name = field.get("name", "")
        if not name:
            continue
        field_type = field.get("type", "text")
        if field_type in {"submit", "button", "image", "reset", "checkbox", "radio"}:
            continue

        haystack = normalize_text(
            " ".join(
                [
                    name,
                    field_type,
                    field.get("placeholder", ""),
                    field.get("aria-label", ""),
                ]
            )
        )
        score = search_hint_score(haystack, field_type)
        if score:
            candidates.append((score, name))

    if candidates:
        candidates.sort(reverse=True)
        return candidates[0][1]

    for field in form.get("inputs", []):
        name = field.get("name", "")
        field_type = field.get("type", "text")
        if name and field_type in {"text", "search", ""}:
            return name
    return "q"


def build_form_request(base_url: str, form: dict[str, Any], query: str) -> SearchRequest | None:
    if not form_is_search(form):
        return None

    action = form.get("action", "") or base_url
    target_url = urljoin(base_url, action)
    method = form.get("method", "GET").upper()
    if method not in {"GET", "POST"}:
        method = "GET"

    search_field = choose_search_field(form)
    params: list[tuple[str, str]] = []
    seen_search_field = False
    for field in form.get("inputs", []):
        name = field.get("name", "")
        field_type = field.get("type", "text")
        if not name or field_type in {"submit", "button", "image", "reset"}:
            continue
        if name == search_field:
            params.append((name, query))
            seen_search_field = True
        elif field_type in {"hidden", "select"} and field.get("value"):
            params.append((name, field.get("value", "")))

    if not seen_search_field:
        params.append((search_field, query))

    encoded = urlencode(params, doseq=True)
    if method == "POST":
        return SearchRequest(url=target_url, method="POST", data=encoded.encode("utf-8"))

    parsed = urlparse(target_url)
    existing_params = parse_qsl(parsed.query, keep_blank_values=True)
    target = parsed._replace(query=urlencode(existing_params + params, doseq=True))
    return SearchRequest(url=urlunparse(target))


def common_search_requests(base_url: str, query: str) -> list[SearchRequest]:
    parsed = urlparse(base_url)
    root = f"{parsed.scheme}://{parsed.netloc}/"
    encoded_query = quote_plus(query)
    paths = [
        f"de/advanced_search_result.php?keywords={encoded_query}",
        f"advanced_search_result.php?keywords={encoded_query}",
        f"catalog/advanced_search_result.php?keywords={encoded_query}",
        f"search?search={encoded_query}",
        f"search?q={encoded_query}",
        f"search?sSearch={encoded_query}",
        f"?s={encoded_query}",
        f"catalogsearch/result/?q={encoded_query}",
        f"index.php?controller=search&s={encoded_query}",
        f"index.php?route=product/search&search={encoded_query}",
        f"suche?search={encoded_query}",
        f"suche?q={encoded_query}",
        f"suche?sSearch={encoded_query}",
        f"de/search?sSearch={encoded_query}",
        f"de/search?search={encoded_query}",
        f"de/suche?search={encoded_query}",
        f"catalogsearch/result/index/?q={encoded_query}",
        f"shopware.php?sViewport=search&sSearch={encoded_query}",
        f"?search={encoded_query}",
        f"?q={encoded_query}",
    ]
    return [SearchRequest(urljoin(root, path)) for path in paths]


def shop_link_score(
    url: str,
    text: str,
    product_labels: list[str],
    config: SearchConfig,
) -> int:
    parsed = urlparse(url)
    path_parts = [part for part in parsed.path.split("/") if part]
    haystack = " ".join([text, parsed.path.replace("-", " ").replace("/", " ")])
    normalized_haystack = normalize_text(haystack)
    query_parts = set(category_tokens(config.query))
    label_parts = set()
    for label in product_labels:
        label_parts.update(category_tokens(label))
    label_parts -= query_parts | SHOP_LABEL_STOP_TOKENS

    score = max(0, len(path_parts) - 1)
    score += 60 * sum(1 for token in query_parts if token in normalized_haystack)
    score += 20 * sum(1 for token in label_parts if token in normalized_haystack)
    if query_parts & ENGINE_PART_TOKENS and "motor" in normalized_haystack:
        score += 15
    return score


def discover_shop_page_requests(
    base_url: str,
    markup: str,
    config: SearchConfig,
) -> list[SearchRequest]:
    parser = parse_page(markup)
    product_labels: list[str] = []
    product_requests: list[SearchRequest] = []
    shop_links: list[tuple[str, str]] = []

    for href, text in parser.links:
        target_url = absolute_url(base_url, href)
        if not target_url or not same_site_url(base_url, target_url):
            continue
        if is_shop_action_url(target_url) or is_view_switch_url(target_url):
            continue

        if is_productish_url(target_url) and text and matches_query(text, config.query, config.mode):
            product_labels.append(clean_line(text))
            product_requests.append(SearchRequest(target_url))
            continue

        if is_shop_page_url(target_url):
            shop_links.append((target_url, clean_line(text)))

    if not product_labels:
        return []

    scored_shop_links: list[tuple[int, int, SearchRequest]] = []
    for index, (target_url, text) in enumerate(shop_links):
        score = shop_link_score(target_url, text, product_labels, config)
        if score > 0:
            scored_shop_links.append((score, index, SearchRequest(target_url)))

    scored_shop_links.sort(key=lambda item: (-item[0], item[1]))
    requests = [request for _score, _index, request in scored_shop_links]
    requests.extend(product_requests)

    deduped: list[SearchRequest] = []
    seen: set[tuple[str, str, bytes | None]] = set()
    for request in requests:
        if request.key in seen:
            continue
        seen.add(request.key)
        deduped.append(request)
    return deduped


def discover_search_requests(base_url: str, homepage_html: str, query: str) -> list[SearchRequest]:
    """Return only search requests actually discovered in the page markup.

    Generic guessed search URLs are intentionally handled later as a fallback
    in search_site(). This avoids trying many unrelated shop-system routes when
    the shop already exposes a usable search form.
    """
    parser = parse_page(homepage_html)
    form_base_url = absolute_url(base_url, parser.base_href) or base_url
    requests: list[SearchRequest] = []
    seen: set[tuple[str, str, bytes | None]] = set()

    for form in parser.forms:
        request = build_form_request(form_base_url, form, query)
        if request and request.key not in seen:
            seen.add(request.key)
            requests.append(request)

    return requests


def common_sitemap_urls(base_url: str) -> list[str]:
    parsed = urlparse(base_url)
    root = f"{parsed.scheme}://{parsed.netloc}/"
    paths: list[str] = []
    path_parts = [part for part in parsed.path.split("/") if part]
    if path_parts and re.fullmatch(r"[a-z]{2}(?:-[a-z]{2})?", path_parts[0].lower()):
        paths.append(f"{path_parts[0]}/sitemap.xml")
    paths.extend(["sitemap.xml", "de/sitemap.xml", "en/sitemap.xml"])
    return list(dict.fromkeys(urljoin(root, path) for path in paths))


def sitemap_locations(markup: str) -> list[str]:
    return [clean_line(html.unescape(match.group(1))) for match in SITEMAP_LOC_RE.finditer(markup or "")]


def is_sitemap_location(url: str) -> bool:
    path = urlparse(url).path.lower()
    return path.endswith((".xml", ".xml.gz")) and "sitemap" in path


def sitemap_product_url_matches(url: str, config: SearchConfig) -> bool:
    parsed = urlparse(url)
    if parsed.query or is_media_url(url) or is_sitemap_location(url):
        return False
    path_text = unquote(parsed.path).replace("-", " ").replace("_", " ").replace("/", " ")
    return matches_query(path_text, config.query, config.match_mode)


def discover_sitemap_product_requests(
    base_url: str,
    config: SearchConfig,
    timeout: float,
) -> list[SearchRequest]:
    limit = max(1, int(config.max_pages_per_site))
    requests: list[SearchRequest] = []
    seen_requests: set[tuple[str, str, bytes | None]] = set()
    seen_sitemaps: set[str] = set()
    sitemap_queue = common_sitemap_urls(base_url)
    fetched_sitemaps = 0

    while sitemap_queue and fetched_sitemaps < SITEMAP_FETCH_LIMIT and len(requests) < limit:
        sitemap_url = sitemap_queue.pop(0)
        sitemap_key = canonical_page_url(sitemap_url)
        if sitemap_key in seen_sitemaps:
            continue
        seen_sitemaps.add(sitemap_key)
        fetched_sitemaps += 1

        try:
            final_url, markup = fetch_request(SearchRequest(sitemap_url), timeout)
        except (HTTPError, URLError, TimeoutError, OSError, ssl.SSLError, ValueError):
            # Diese Sitemap-Adressen sind nur vorsichtige Fallback-Vermutungen.
            # Ein 403/404/robots-Verbot hier ist kein relevanter Shop-Fehler
            # und soll deshalb die Benutzerhinweise nicht aufblaehen.
            continue

        for loc in sitemap_locations(markup):
            target_url = absolute_url(final_url, loc)
            if not target_url or not same_site_url(base_url, target_url):
                continue
            if is_sitemap_location(target_url):
                if target_url not in seen_sitemaps:
                    sitemap_queue.append(target_url)
                continue
            if not sitemap_product_url_matches(target_url, config):
                continue

            request = SearchRequest(target_url)
            if request.key in seen_requests:
                continue
            seen_requests.add(request.key)
            requests.append(request)
            if len(requests) >= limit:
                break

    return requests


def is_pagination_url(url: str) -> bool:
    parsed = urlparse(url)
    path_and_query = f"{parsed.path}?{parsed.query}"
    return PAGINATION_URL_RE.search(path_and_query) is not None


def is_pagination_link(text: str, url: str) -> bool:
    raw_text = clean_line(text)
    normalized = normalize_text(raw_text)
    page_url = is_pagination_url(url)
    if raw_text in {">", ">>", "»", "›", "→"}:
        return page_url
    if normalized in PAGINATION_WORDS or normalized.startswith("naechst"):
        return True
    if re.fullmatch(r"\d{1,3}", normalized):
        return page_url
    if "seite" in normalized and re.search(r"\d", normalized):
        return True
    return page_url and len(normalized) <= 12


def discover_pagination_requests(page_url: str, markup: str) -> list[SearchRequest]:
    parser = parse_page(markup)
    requests: list[SearchRequest] = []
    seen: set[tuple[str, str, bytes | None]] = set()
    current_url = canonical_page_url(page_url)
    candidates = [(href, "next") for href in parser.next_links]
    candidates.extend(parser.links)

    for href, text in candidates:
        target_url = absolute_url(page_url, href)
        if not target_url or not same_site_url(page_url, target_url):
            continue
        if is_shop_action_url(target_url) or is_view_switch_url(target_url):
            continue
        target_url = canonical_page_url(target_url)
        if target_url == current_url:
            continue
        if text != "next" and not is_pagination_link(text, target_url):
            continue

        request = SearchRequest(target_url)
        if request.key in seen:
            continue
        seen.add(request.key)
        requests.append(request)
    return requests


def normalize_price(value: str) -> str:
    value = clean_line(value)
    value = re.sub(r"\s+", " ", value)
    return value


def price_number(value: str) -> float | None:
    text = clean_line(value)
    match = re.search(r"\d+(?:[.,]\d+)?", text)
    if not match:
        return None
    number = match.group(0).replace(".", "").replace(",", ".")
    try:
        return float(number)
    except ValueError:
        return None


def is_zero_price(value: str) -> bool:
    number = price_number(value)
    return number is not None and abs(number) < 0.005


def first_price(lines: list[str]) -> str:
    for line in lines:
        matches = [
            match
            for match in PRICE_RE.finditer(line)
            if match.group("prefix") or match.group("suffix")
        ]
        if not matches:
            continue
        shipping_match = re.search(r"versand|shipping|porto", line, re.IGNORECASE)
        if shipping_match:
            product_prices = [match for match in matches if match.start() < shipping_match.start()]
            for match in product_prices:
                price = normalize_price(match.group(0))
                if not is_zero_price(price):
                    return price
            continue
        for match in matches:
            price = normalize_price(match.group(0))
            if not is_zero_price(price):
                return price
    return ""


def find_shipping(lines: list[str]) -> str:
    for line in lines:
        normalized = normalize_text(line)
        if not any(word in normalized for word in ("versand", "shipping", "porto")):
            continue
        matches = [
            match
            for match in PRICE_RE.finditer(line)
            if match.group("prefix") or match.group("suffix")
        ]
        if matches:
            keyword_match = re.search(r"versand|shipping|porto", line, re.IGNORECASE)
            if keyword_match:
                keyword_pos = keyword_match.start()
                for match in matches:
                    if match.start() >= keyword_pos:
                        return normalize_price(match.group(0))
            return normalize_price(matches[-1].group(0))
        if "kostenlos" in normalized or "free" in normalized:
            return "kostenlos"
        return "siehe Shop"
    return ""


def find_availability(lines: list[str]) -> str:
    joined = normalize_text(" | ".join(lines))
    negative = [
        ("nicht lieferbar", "nicht lieferbar"),
        ("nicht verfuegbar", "nicht verfügbar"),
        ("ausverkauft", "ausverkauft"),
        ("out of stock", "out of stock"),
        ("sold out", "sold out"),
        ("unavailable", "unavailable"),
    ]
    positive = [
        ("sofort lieferbar", "sofort lieferbar"),
        ("lieferbar", "lieferbar"),
        ("verfuegbar", "verfügbar"),
        ("vorraetig", "vorrätig"),
        ("auf lager", "auf Lager"),
        ("in stock", "in stock"),
        ("op voorraad", "op voorraad"),
    ]
    for needle, label in negative:
        if needle in joined:
            return label
    for needle, label in positive:
        if needle in joined:
            return label

    for line in lines:
        normalized = normalize_text(line)
        if any(word in normalized for word in ("lieferzeit", "delivery time", "versandfertig")):
            return clean_availability_line(line)
    return ""


def link_candidates_from_fragment(fragment: str, base_url: str) -> list[tuple[str, str]]:
    links: list[tuple[str, str]] = []

    parser = parse_page(fragment)
    for href, text in parser.links:
        abs_href = absolute_url(base_url, href)
        if abs_href and not is_shop_action_url(abs_href) and not is_media_url(abs_href):
            abs_href = strip_url_query_params(abs_href, PRODUCT_LINK_IGNORED_PARAMS)
            links.append((abs_href, text))

    seen = {href for href, _text in links}
    for match in URL_RE.finditer(fragment):
        href = absolute_url(base_url, match.group(2))
        if href:
            href = strip_url_query_params(href, PRODUCT_LINK_IGNORED_PARAMS)
        if href and href not in seen and not is_shop_action_url(href) and not is_media_url(href):
            seen.add(href)
            links.append((href, ""))
    return links


def link_match_contexts(links: list[tuple[str, str]]) -> list[str]:
    contexts: list[str] = []
    for href, text in links:
        if text:
            contexts.append(text)
        path = urlparse(href).path
        if path:
            contexts.append(path.replace("/", " ").replace("-", " ").replace("_", " "))
    return contexts


def choose_name(lines: list[str], config: SearchConfig, link_texts: list[str]) -> str:
    match_mode = config.match_mode
    for link_text in link_texts:
        if (
            link_text
            and not is_search_context_line(link_text, config, ignore_exact_query=False)
            and matches_query(link_text, config.query, match_mode)
        ):
            return clean_line(link_text)

    useful_lines = []
    for line in lines:
        normalized = normalize_text(line)
        if len(line) < 3 or len(line) > 180:
            continue
        if is_markup_artifact_line(line):
            continue
        if is_search_context_line(line, config):
            continue
        if any(ignored in normalized for ignored in IGNORED_NAME_LINES):
            continue
        if PRICE_RE.search(line):
            continue
        useful_lines.append(line)

    for line in useful_lines:
        if matches_query(line, config.query, match_mode):
            if match_mode == "name_like":
                for link_text in link_texts:
                    normalized_link = normalize_text(link_text)
                    if (
                        link_text
                        and not is_search_context_line(link_text, config, ignore_exact_query=False)
                        and not any(ignored in normalized_link for ignored in IGNORED_NAME_LINES)
                    ):
                        return clean_line(link_text)
            return clean_line(line)
    if match_mode == "article":
        return clean_line(useful_lines[0]) if useful_lines else ""
    return ""


def product_objects_from_json_ld(markup: str) -> list[dict[str, Any]]:
    parser = parse_page(markup)
    products: list[dict[str, Any]] = []

    def visit(value: Any) -> None:
        if isinstance(value, list):
            for item in value:
                visit(item)
            return
        if not isinstance(value, dict):
            return

        type_value = value.get("@type") or value.get("type")
        type_values = type_value if isinstance(type_value, list) else [type_value]
        normalized_types = {normalize_text(str(item)) for item in type_values if item}
        if "product" in normalized_types:
            products.append(value)

        for key in (
            "@graph",
            "itemListElement",
            "item",
            "mainEntity",
            "mainEntityOfPage",
            "hasVariant",
        ):
            if key in value:
                visit(value[key])

    for script_type, script_text in parser.scripts:
        if "ld+json" not in script_type.lower():
            continue
        raw = script_text.strip()
        if not raw:
            continue
        try:
            visit(json.loads(raw))
        except json.JSONDecodeError:
            repaired = re.sub(r"</?script[^>]*>", "", raw, flags=re.IGNORECASE)
            try:
                visit(json.loads(repaired))
            except json.JSONDecodeError:
                continue
    return products


def value_as_text(value: Any) -> str:
    if value is None:
        return ""
    if isinstance(value, str):
        return clean_line(value)
    if isinstance(value, (int, float)):
        return str(value)
    if isinstance(value, list):
        return clean_line(", ".join(value_as_text(item) for item in value if value_as_text(item)))
    if isinstance(value, dict):
        for key in ("name", "value", "@id", "url"):
            if key in value:
                return value_as_text(value[key])
    return clean_line(str(value))


def offers_from_product(product: dict[str, Any]) -> list[dict[str, Any]]:
    offers = product.get("offers")
    if not offers:
        return [{}]
    if isinstance(offers, list):
        return [offer for offer in offers if isinstance(offer, dict)] or [{}]
    if isinstance(offers, dict):
        if "offers" in offers and isinstance(offers["offers"], list):
            return [offer for offer in offers["offers"] if isinstance(offer, dict)] or [offers]
        return [offers]
    return [{}]


def price_from_offer(offer: dict[str, Any]) -> str:
    price = next(
        (
            offer[key]
            for key in ("price", "lowPrice", "highPrice")
            if key in offer and offer[key] is not None and offer[key] != ""
        ),
        None,
    )
    currency = offer.get("priceCurrency", "")
    if price is not None:
        if currency:
            return clean_line(f"{price} {currency}")
        return clean_line(str(price))

    spec = offer.get("priceSpecification")
    if isinstance(spec, dict):
        price = spec.get("price")
        currency = spec.get("priceCurrency", currency)
        if price is not None and price != "":
            return clean_line(f"{price} {currency}".strip())
    return ""


def availability_from_offer(offer: dict[str, Any]) -> str:
    raw = value_as_text(offer.get("availability"))
    if not raw:
        return ""
    tail = raw.rstrip("/").split("/")[-1]
    mapping = {
        "instock": "in stock",
        "outofstock": "out of stock",
        "preorder": "preorder",
        "backorder": "backorder",
        "soldout": "sold out",
        "discontinued": "discontinued",
        "limitedavailability": "limited availability",
    }
    return mapping.get(normalize_text(tail).replace(" ", ""), tail or raw)


def first_meta_content(parser: PageParser, *keys: str) -> str:
    wanted = {normalize_text(key) for key in keys}
    for item in parser.meta_tags:
        item_keys = {
            normalize_text(item.get("property", "")),
            normalize_text(item.get("name", "")),
            normalize_text(item.get("itemprop", "")),
        }
        if wanted & item_keys:
            content = clean_line(item.get("content", ""))
            if content:
                return content
    return ""


def first_link_href(parser: PageParser, *keys: str) -> str:
    wanted = {normalize_text(key) for key in keys}
    for item in parser.link_tags:
        item_keys = {
            normalize_text(item.get("rel", "")),
            normalize_text(item.get("itemprop", "")),
        }
        if wanted & item_keys:
            href = clean_line(item.get("href", ""))
            if href:
                return href
    return ""


def clean_meta_product_name(value: str) -> str:
    name = clean_line(value)
    if " | " in name:
        head = clean_line(name.split(" | ", 1)[0])
        if head:
            return head
    return name


def hits_from_meta(markup: str, base_url: str, site: str, config: SearchConfig) -> list[ProductHit]:
    parser = parse_page(markup)
    name = clean_meta_product_name(
        first_meta_content(parser, "og:title", "twitter:title", "name")
    )
    description = first_meta_content(parser, "og:description", "twitter:description", "description")
    sku = first_meta_content(parser, "product:retailer_item_id", "sku")
    mpn = first_meta_content(parser, "mpn")
    if not name or not matches_product(name, config, sku, mpn, description):
        return []

    price = first_meta_content(parser, "product:price:amount", "price")
    currency = first_meta_content(parser, "product:price:currency", "priceCurrency")
    price = clean_line(f"{price} {currency}".strip()) if price else ""
    if is_zero_price(price):
        return []

    availability = first_link_href(parser, "availability")
    url = (
        first_meta_content(parser, "product:product_link", "og:url")
        or first_link_href(parser, "canonical")
        or base_url
    )

    return [
        ProductHit(
            site=site,
            name=name,
            availability=availability_from_offer({"availability": availability}) or "-",
            price=price or "-",
            shipping="-",
            url=absolute_url(base_url, url) or base_url,
            source="Meta",
        )
    ]


def shipping_from_offer(offer: dict[str, Any]) -> str:
    details = offer.get("shippingDetails") or offer.get("shippingRate")
    if isinstance(details, list):
        for item in details:
            text = shipping_from_offer({"shippingDetails": item})
            if text:
                return text
    if isinstance(details, dict):
        rate = details.get("shippingRate") or details.get("price") or details.get("value")
        if isinstance(rate, dict):
            price = rate.get("value") or rate.get("price")
            currency = rate.get("currency") or rate.get("priceCurrency")
            if price:
                return clean_line(f"{price} {currency or ''}".strip())
        if rate:
            return value_as_text(rate)
    return ""


def hits_from_json_ld(markup: str, base_url: str, site: str, config: SearchConfig) -> list[ProductHit]:
    hits: list[ProductHit] = []
    for product in product_objects_from_json_ld(markup):
        name = value_as_text(product.get("name"))
        sku = value_as_text(product.get("sku"))
        mpn = value_as_text(product.get("mpn"))
        description = value_as_text(product.get("description"))
        if not name or not matches_product(name, config, sku, mpn, description):
            continue

        for offer in offers_from_product(product):
            url = value_as_text(offer.get("url")) or value_as_text(product.get("url")) or base_url
            price = price_from_offer(offer)
            if is_zero_price(price):
                continue
            hits.append(
                ProductHit(
                    site=site,
                    name=name,
                    availability=availability_from_offer(offer) or "-",
                    price=price or "-",
                    shipping=shipping_from_offer(offer) or "-",
                    url=absolute_url(base_url, url) or base_url,
                    source="JSON-LD",
                )
            )
    return hits




CATEGORY_PRODUCT_ANCHOR_RE = re.compile(
    r"(?is)<a\b[^>]*href\s*=\s*([\"'])(?P<href>.*?)\1[^>]*>(?P<body>.*?)</a>"
)


def hits_from_category_page(
    markup: str,
    base_url: str,
    site: str,
    config: SearchConfig,
) -> list[ProductHit]:
    """Parse product-heavy category pages used by shops such as Werk34/Ahnendorp.

    The normal fragment parser expects specific product-card classes. Some shops render
    perfectly usable category pages with different markup, so we fall back to scanning
    product-looking anchors and a short following HTML window for price/availability.
    Vehicle matching is intentionally not repeated here: these pages are only used when
    a shop profile already selected a vehicle-specific category.
    """
    hits: list[ProductHit] = []
    matches = list(CATEGORY_PRODUCT_ANCHOR_RE.finditer(markup or ""))
    if not matches:
        return hits

    for index, match in enumerate(matches):
        href = absolute_url(base_url, match.group("href"))
        if not href or not same_site_url(base_url, href):
            continue
        if is_shop_action_url(href) or is_view_switch_url(href) or is_media_url(href):
            continue

        name_lines = html_to_lines(match.group("body"))
        name = clean_line(" ".join(name_lines))
        if not name or is_generic_link_text(name) or len(name) < 4 or len(name) > 220:
            continue
        name_matches = matches_query(name, config.query, config.match_mode)
        if not name_matches and config.match_mode == "exact":
            # Category pages are already preselected for the requested topic; allow
            # German compounds such as "Doppelvergaseranlage" for query "vergaser".
            name_matches = normalize_text(config.query) in normalize_text(name)
        if not name_matches:
            continue

        next_start = matches[index + 1].start() if index + 1 < len(matches) else len(markup)
        window_end = min(next_start, match.end() + 5000)
        window = markup[match.end():window_end]
        lines = html_to_lines(window)
        price = first_price(lines)
        if not price or is_zero_price(price):
            continue

        hits.append(
            ProductHit(
                site=site,
                name=name,
                availability=find_availability(lines[:60]) or "-",
                price=price,
                shipping=find_shipping(lines[:80]) or "-",
                url=href,
                source="CategoryHTML",
            )
        )

    return dedupe_hits(hits)


def split_result_fragments(markup: str) -> list[str]:
    product_matches = [
        (match.start(), match.group(0))
        for pattern in (DIV_PRODUCT_FRAGMENT_RE, LI_PRODUCT_FRAGMENT_RE)
        for match in pattern.finditer(markup)
    ]
    product_matches.sort(key=lambda item: item[0])
    product_fragments = [fragment for _start, fragment in product_matches]
    if product_fragments:
        return product_fragments

    fragments: list[str] = []
    current = ""
    for chunk in RESULT_FRAGMENT_SPLIT_RE.split(markup):
        if not chunk.strip():
            continue
        if len(current) + len(chunk) < 7000:
            current += "\n" + chunk
        else:
            fragments.append(current)
            current = chunk
        if PRICE_RE.search(current):
            fragments.append(current)
            current = ""
    if current:
        fragments.append(current)
    return fragments


def fragment_allows_missing_price(fragment: str) -> bool:
    return "os_list_wrap_all" in fragment


def hits_from_html(markup: str, base_url: str, site: str, config: SearchConfig) -> list[ProductHit]:
    hits: list[ProductHit] = []
    for fragment in split_result_fragments(markup):
        lines = html_to_lines(fragment)
        if not lines:
            continue

        links = link_candidates_from_fragment(fragment, base_url)
        link_texts = [text for _, text in links if text]
        name = choose_name(lines, config, link_texts)
        if not name:
            continue
        product_context = " ".join(
            line for line in lines[:12] if not is_search_context_line(line, config)
        )
        match_mode = config.match_mode
        if match_mode in {"article", "name_like"}:
            match_contexts = [product_context]
            if match_mode == "article":
                match_contexts.extend(link_match_contexts(links))
            product_matches = matches_product(
                name,
                config,
                *match_contexts,
            )
        else:
            product_matches = matches_product(name, config)
        if not product_matches:
            continue

        price = first_price(lines)
        if is_zero_price(price):
            continue
        if not price and not fragment_allows_missing_price(fragment):
            continue

        url = next((href for href, text in links if text and clean_line(text) == name), "")
        if not url:
            url = next(
                (
                    href
                    for href, text in links
                    if text
                    and not is_search_context_line(text, config, ignore_exact_query=False)
                    and matches_query(text, config.query, config.mode)
                ),
                "",
            )
        if not url and links:
            url = links[0][0]

        hits.append(
            ProductHit(
                site=site,
                name=name,
                availability=find_availability(lines) or "-",
                price=price or "-",
                shipping=find_shipping(lines) or "-",
                url=url or base_url,
                source="HTML",
            )
        )

    if hits:
        return hits

    lines = html_to_lines(markup)
    price = first_price(lines)
    if not price or is_zero_price(price):
        return []

    parser = parse_page(markup)
    candidates = parser.headings + ([parser.title] if parser.title else [])
    name = next(
        (
            item
            for item in candidates
            if not is_search_context_line(item, config) and matches_product(item, config)
        ),
        "",
    )
    if name:
        hits.append(
            ProductHit(
                site=site,
                name=name,
                availability=find_availability(lines[:80]) or "-",
                price=price,
                shipping=find_shipping(lines[:120]) or "-",
                url=base_url,
                source="HTML",
            )
        )
    return hits


def dedupe_hits(hits: list[ProductHit]) -> list[ProductHit]:
    seen_url: dict[tuple[str, str], int] = {}
    seen_name_price: dict[tuple[str, str, str], int] = {}
    deduped: list[ProductHit] = []
    for hit in hits:
        canonical_url = canonical_page_url(hit.url)
        url_key = (
            normalize_text(hit.site),
            canonical_url,
        ) if canonical_url else None
        name_price_key = (
            normalize_text(hit.site),
            normalize_text(hit.name),
            normalize_text(hit.price),
        ) if hit.name and hit.price != "-" else None

        existing_index = None
        if canonical_url:
            existing_index = seen_url.get((normalize_text(hit.site), canonical_url))
        if existing_index is None and name_price_key is not None:
            existing_index = seen_name_price.get(name_price_key)

        if existing_index is not None:
            deduped[existing_index] = merge_duplicate_hit(deduped[existing_index], hit)
            if url_key is not None:
                seen_url[url_key] = existing_index
            if name_price_key is not None:
                seen_name_price[name_price_key] = existing_index
            continue
        deduped.append(hit)
        new_index = len(deduped) - 1
        if url_key is not None:
            seen_url[url_key] = new_index
        if name_price_key is not None:
            seen_name_price[name_price_key] = new_index
    return deduped


def merge_duplicate_hit(base: ProductHit, other: ProductHit) -> ProductHit:
    source = base.source
    if other.source and other.source not in source:
        source = f"{source}+{other.source}"
    return ProductHit(
        site=base.site,
        name=base.name or other.name,
        availability=base.availability if base.availability != "-" else other.availability,
        price=base.price if base.price != "-" else other.price,
        shipping=base.shipping if base.shipping != "-" else other.shipping,
        url=base.url or other.url,
        source=source,
    )


def parse_hits(markup: str, base_url: str, site: str, config: SearchConfig) -> list[ProductHit]:
    hits = hits_from_json_ld(markup, base_url, site, config)
    hits.extend(hits_from_meta(markup, base_url, site, config))
    hits.extend(hits_from_html(markup, base_url, site, config))

    # Werk34 and Ahnendorp expose useful, robots-allowed category pages whose
    # product-card markup does not match the generic fragment patterns above.
    # Only use this fallback when the generic parsers found nothing.
    if not hits and normalized_host(base_url) in {"werk34.de", "ahnendorp.com"}:
        hits.extend(hits_from_category_page(markup, base_url, site, config))

    return dedupe_hits(hits)


def merge_hit(base: ProductHit, detail: ProductHit) -> ProductHit:
    return ProductHit(
        site=base.site,
        name=base.name,
        availability=detail.availability if detail.availability != "-" else base.availability,
        price=detail.price if base.price == "-" and detail.price != "-" else base.price,
        shipping=detail.shipping if detail.shipping != "-" else base.shipping,
        url=base.url or detail.url,
        source=f"{base.source}+Detail",
    )


def enrich_hits(
    hits: list[ProductHit],
    config: SearchConfig,
    stop_event: threading.Event,
    timeout: int,
) -> list[ProductHit]:
    enriched: list[ProductHit] = []
    for hit in hits[:10]:
        if stop_event.is_set():
            break
        if not hit.url.startswith(("http://", "https://")):
            enriched.append(hit)
            continue
        needs_detail = hit.availability == "-" or hit.price == "-"
        if not needs_detail:
            enriched.append(hit)
            continue

        try:
            detail_url, detail_markup = fetch_url(hit.url, timeout)
        except (HTTPError, URLError, TimeoutError, OSError, ssl.SSLError, ValueError) as exc:
            report_fetch_error(config, hit.url, exc)
            enriched.append(hit)
            continue

        detail_hits = parse_hits(detail_markup, detail_url, hit.site, config)
        if detail_hits:
            detail_hit = next(
                (
                    candidate
                    for candidate in detail_hits
                    if canonical_page_url(candidate.url) == canonical_page_url(hit.url)
                ),
                detail_hits[0],
            )
            enriched.append(merge_hit(hit, detail_hit))
        else:
            enriched.append(hit)

    if len(hits) > len(enriched):
        enriched.extend(hits[len(enriched) :])
    return dedupe_hits(enriched)


def add_new_requests(
    pending: list[SearchRequest],
    seen_requests: set[tuple[str, str, bytes | None]],
    requests: list[SearchRequest],
    *,
    prepend: bool = False,
) -> None:
    new_requests: list[SearchRequest] = []
    for request in requests:
        if request.key in seen_requests:
            continue
        seen_requests.add(request.key)
        new_requests.append(request)
    if prepend:
        pending[:0] = new_requests
    else:
        pending.extend(new_requests)


def fetch_direct_profile_hits(
    url: str,
    config: SearchConfig,
    profile: ShopProfile,
    stop_event: threading.Event,
    timeout_for_request: Callable[[], float],
    deadline_expired: Callable[[], bool],
) -> tuple[list[ProductHit], bool]:
    """Try known public category routes with a very small request budget.

    Returns (hits, attempted). A successful category hit lets search_site skip
    homepage/search-form probing, which is especially useful for shops whose
    search endpoint or homepage is WAF-blocked.
    """
    if not profile.try_direct_routes_first:
        return [], False

    requests = direct_route_requests(url, config)
    if not requests:
        return [], False

    site = site_label(url)
    hits: list[ProductHit] = []
    attempted = False
    budget = max(1, int(profile.max_pages_per_site or 1))

    for request in requests[:budget]:
        if stop_event.is_set() or deadline_expired():
            break
        if not robots_allows(request.url, timeout=timeout_for_request()):
            continue
        attempted = True
        try:
            final_url, markup = fetch_request(request, timeout_for_request())
        except (HTTPError, URLError, TimeoutError, OSError, ssl.SSLError, ValueError):
            continue
        hits.extend(parse_hits(markup, final_url, site, config))

    return dedupe_hits(hits), attempted


def search_site(url: str, config: SearchConfig, stop_event: threading.Event) -> list[ProductHit]:
    site = site_label(url)
    profile = shop_profile_for_url(url)
    started_at = time.monotonic()
    site_deadline_seconds = float(profile.site_deadline_seconds or config.site_deadline_seconds)
    max_pages_per_site = int(profile.max_pages_per_site or config.max_pages_per_site)

    # Avoid flooding the UI with the same 403/robots message for one shop.
    reported_messages: set[str] = set()
    upstream_reporter = config.report_error

    def report_once(message: str) -> None:
        if message in reported_messages:
            return
        reported_messages.add(message)
        if upstream_reporter is not None:
            upstream_reporter(message)

    site_config = replace(config, report_error=report_once)

    def remaining_site_seconds() -> float:
        return site_deadline_seconds - (time.monotonic() - started_at)

    def request_timeout() -> float:
        return max(0.5, min(float(site_config.timeout), remaining_site_seconds()))

    def deadline_expired() -> bool:
        return remaining_site_seconds() <= 0

    def report_site_deadline() -> None:
        report_once(f"[TIMEOUT] {site}: Zeitlimit erreicht; nur dieser Suchlauf wird beendet")

    if stop_event.is_set():
        return []

    direct_hits, direct_attempted = fetch_direct_profile_hits(
        url,
        site_config,
        profile,
        stop_event,
        request_timeout,
        deadline_expired,
    )
    if direct_hits:
        return enrich_hits(direct_hits, site_config, stop_event, request_timeout())

    homepage_url, homepage_html = url, ""
    base_url = url
    homepage_candidates = host_variants(url) if profile.try_host_variants else [url]

    for candidate_url in homepage_candidates:
        if stop_event.is_set():
            return []
        if deadline_expired():
            report_site_deadline()
            return []
        try:
            homepage_url, homepage_html = fetch_url(candidate_url, request_timeout())
            base_url = homepage_url
            break
        except (HTTPError, URLError, TimeoutError, OSError, ssl.SSLError, ValueError) as exc:
            # For known fail-fast shops report one concise classification instead
            # of both the low-level fetch error and a second summary line.
            if isinstance(exc, RobotsDeniedError) and profile.stop_on_homepage_robots:
                report_once(f"[ROBOTS] {site}: Einstieg laut robots.txt nicht erlaubt; nur dieser Suchlauf wird uebersprungen")
                return []

            if isinstance(exc, HTTPError) and exc.code == 403 and profile.stop_on_homepage_403:
                report_once(f"[HTTP403] {site}: Zugriff aktuell abgewiesen; nur dieser Suchlauf wird uebersprungen")
                return []

            report_fetch_error(site_config, candidate_url, exc)
            continue

    if not homepage_html and not homepage_url:
        return []

    # 1) Search routes actually discovered in the shop have highest priority.
    form_requests = discover_search_requests(
        homepage_url, homepage_html, site_config.effective_query
    )
    shop_page_requests = discover_shop_page_requests(homepage_url, homepage_html, site_config)
    primary_requests = form_requests + shop_page_requests

    allowed_primary = [
        request
        for request in primary_requests
        if robots_allows(request.url, timeout=request_timeout())
    ]

    requests: list[SearchRequest] = []
    if allowed_primary:
        requests = allowed_primary
    else:
        if primary_requests:
            fallback_label = "zulaessiger Fallback wird geprueft" if (profile.allow_generic_fallback or profile.use_sitemap_fallback) else "Shop wird fuer diesen Suchlauf uebersprungen"
            report_once(
                f"[ROBOTS] {site}: erkannter Suchweg nicht erlaubt; {fallback_label}"
            )

        # 2) Generic guessed routes are only used for shops where the profile
        # explicitly allows this. Known-problematic shops skip this noisy step.
        if profile.allow_generic_fallback:
            generic_requests: list[SearchRequest] = []
            generic_hosts = host_variants(base_url) if profile.try_host_variants else [base_url]
            for candidate_url in generic_hosts:
                generic_requests.extend(
                    common_search_requests(candidate_url, site_config.effective_query)
                )

            requests = [
                request
                for request in generic_requests
                if robots_allows(request.url, timeout=request_timeout())
            ]

        # 3) Sitemap/product discovery is the last fallback and may be disabled
        # per shop when it is known to be unhelpful or blocked.
        if not requests and profile.use_sitemap_fallback and not deadline_expired():
            requests = discover_sitemap_product_requests(
                homepage_url, site_config, request_timeout()
            )

    seen_requests: set[tuple[str, str, bytes | None]] = set()
    pending_requests: list[SearchRequest] = []
    add_new_requests(pending_requests, seen_requests, requests)

    hits: list[ProductHit] = []
    checked_pages = 0
    while pending_requests and checked_pages < max_pages_per_site:
        request = pending_requests.pop(0)
        if stop_event.is_set():
            break
        if deadline_expired():
            report_site_deadline()
            break

        # Pagination or follow-up links can be forbidden even when page 1 was
        # allowed. Skip them silently instead of producing one warning per page.
        if not robots_allows(request.url, timeout=request_timeout()):
            continue

        checked_pages += 1
        try:
            final_url, markup = fetch_request(request, request_timeout())
        except (HTTPError, URLError, TimeoutError, OSError, ssl.SSLError, ValueError) as exc:
            if isinstance(exc, HTTPError) and exc.code == 403 and profile.stop_on_search_403:
                report_once(f"[HTTP403] {site}: Suchzugriff aktuell abgewiesen; weitere Suchwege werden in diesem Lauf uebersprungen")
                break
            report_fetch_error(site_config, request.url, exc)
            continue

        page_hits = parse_hits(markup, final_url, site, site_config)
        if page_hits:
            hits.extend(page_hits)

        add_new_requests(
            pending_requests,
            seen_requests,
            discover_pagination_requests(final_url, markup),
            prepend=True,
        )

    if deadline_expired():
        report_site_deadline()
        return dedupe_hits(hits)

    return enrich_hits(dedupe_hits(hits), site_config, stop_event, request_timeout())