# Copyright (c) DevSecCode. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root.

"""
Async robots.txt compliance checker.

Fetches and caches robots.txt per domain (1-hour TTL).  Used by browser
agent and web scraper nodes before navigating to external URLs.

Usage::

    checker = RobotsChecker()
    allowed = await checker.is_allowed("https://example.com/page", user_agent="DevSecCode")
"""

from __future__ import annotations

import time
import asyncio
from urllib.parse import urlparse
from urllib.robotparser import RobotFileParser
from typing import Any

import httpx


_CACHE_TTL_SECONDS = 3600  # 1 hour


class _CachedRobots:
	__slots__ = ('parser', 'fetched_at')

	def __init__(self, parser: RobotFileParser, fetched_at: float) -> None:
		self.parser = parser
		self.fetched_at = fetched_at

	@property
	def expired(self) -> bool:
		return (time.monotonic() - self.fetched_at) > _CACHE_TTL_SECONDS


class RobotsChecker:
	"""Async robots.txt checker with per-domain caching."""

	def __init__(self, timeout: float = 10.0) -> None:
		self._cache: dict[str, _CachedRobots] = {}
		self._locks: dict[str, asyncio.Lock] = {}
		self._timeout = timeout

	async def is_allowed(
		self,
		url: str,
		user_agent: str = 'DevSecCode',
	) -> bool:
		"""Check whether *url* is allowed by the domain's robots.txt.

		Returns ``True`` if robots.txt cannot be fetched (fail-open for
		usability — the browser agent still respects explicit blocks).
		"""
		parsed = urlparse(url)
		domain = f"{parsed.scheme}://{parsed.netloc}"

		parser = await self._get_parser(domain)
		if parser is None:
			return True  # fail-open

		return parser.can_fetch(user_agent, url)

	async def _get_parser(self, domain: str) -> RobotFileParser | None:
		cached = self._cache.get(domain)
		if cached and not cached.expired:
			return cached.parser

		# One fetch at a time per domain
		if domain not in self._locks:
			self._locks[domain] = asyncio.Lock()

		async with self._locks[domain]:
			# Double-check after acquiring lock
			cached = self._cache.get(domain)
			if cached and not cached.expired:
				return cached.parser

			return await self._fetch(domain)

	async def _fetch(self, domain: str) -> RobotFileParser | None:
		robots_url = f"{domain}/robots.txt"
		try:
			async with httpx.AsyncClient(timeout=self._timeout) as client:
				resp = await client.get(robots_url, follow_redirects=True)

			if resp.status_code != 200:
				# No robots.txt → allow everything
				self._cache[domain] = _CachedRobots(
					parser=self._permissive_parser(robots_url),
					fetched_at=time.monotonic(),
				)
				return self._cache[domain].parser

			parser = RobotFileParser()
			parser.set_url(robots_url)
			parser.parse(resp.text.splitlines())

			self._cache[domain] = _CachedRobots(
				parser=parser,
				fetched_at=time.monotonic(),
			)
			return parser

		except Exception:
			# Network error → fail-open, cache a permissive parser
			self._cache[domain] = _CachedRobots(
				parser=self._permissive_parser(robots_url),
				fetched_at=time.monotonic(),
			)
			return self._cache[domain].parser

	@staticmethod
	def _permissive_parser(url: str) -> RobotFileParser:
		"""Create a parser that allows everything."""
		parser = RobotFileParser()
		parser.set_url(url)
		parser.parse([])  # empty → allow all
		return parser

	def clear_cache(self) -> None:
		"""Evict all cached robots.txt entries."""
		self._cache.clear()
