"""Compliance evaluation engine.

Fetches control definitions from Firestore, evaluates findings against
mapped CWEs, and produces coverage reports per framework.

Stateful records (attestations, baselines) flow through the storage
abstraction in ``compliance_storage`` so the same code path serves the
Firestore cloud default, the SQLite local fallback, and the in-memory test
backend. Selection is by ``DSC_COMPLIANCE_BACKEND``.
"""

from __future__ import annotations

import logging
import time
import uuid
from datetime import datetime, timezone
from typing import Any

from dsc_api.models.compliance_models import (
    Attestation,
    AttestationActivity,
    BulkAttestResult,
    BulkRevokeResult,
    ComplianceBaseline,
    ComplianceFramework,
    ComplianceReport,
    ControlEvaluation,
    ControlEvidence,
    ControlStatus,
    DriftReport,
    EvidenceItem,
    EvidencePackage,
    FrameworkReport,
    GroupBreakdown,
)
from dsc_api.services.compliance_storage import get_storage, utc_now_iso

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Firestore helpers
# ---------------------------------------------------------------------------

_CACHE_TTL_S = 15 * 60  # 15 minutes


def _get_db():
    """Return a Firestore client or None if unavailable.

    Validates ADC credentials before creating the client to avoid gRPC
    auth callbacks that block the thread pool on expired/missing credentials.
    """
    try:
        import os
        import google.auth
        credentials, _project = google.auth.default()
        if hasattr(credentials, 'expired') and credentials.expired:
            return None

        from google.cloud import firestore
        return firestore.Client(
            project=os.environ.get("GCP_PROJECT_ID", "devseccode-9dc41"),
            database="dsc-app",
            credentials=credentials,
        )
    except Exception:
        return None


# ---------------------------------------------------------------------------
# Cached control/framework data
# ---------------------------------------------------------------------------

_frameworks_cache: list[dict[str, Any]] = []
_controls_cache: dict[str, list[dict[str, Any]]] = {}  # framework_id -> controls
_cache_ts: float = 0.0


def _merge_custom_frameworks() -> None:
    """Inject user-defined custom frameworks into the compliance caches.

    Custom frameworks live as workspace-local YAML files (see
    ``custom_frameworks.py``). They are merged after the Firestore fetch
    so all downstream paths (evaluate, baseline, evidence) treat them
    identically to built-in frameworks.
    """
    try:
        from dsc_api.services.custom_frameworks import merge_into_caches
        merge_into_caches(_frameworks_cache, _controls_cache)
    except Exception as exc:
        logger.warning("Failed to merge custom frameworks: %s", exc)


def _refresh_cache_if_needed() -> None:
    global _frameworks_cache, _controls_cache, _cache_ts

    now = time.monotonic()
    if _frameworks_cache and (now - _cache_ts) < _CACHE_TTL_S:
        return

    # Fetch directly from Cloud Run proxy (not via localhost, which would
    # deadlock the single-worker uvicorn).
    import urllib.request
    import json as _json

    from dsc_api.services.product_config import get_llm_proxy_url, get_llm_proxy_key, is_service_auth_allowed

    proxy_url = get_llm_proxy_url()
    if not proxy_url:
        logger.warning("Compliance proxy URL missing; loading custom frameworks only")
        _frameworks_cache = []
        _controls_cache = {}
        _merge_custom_frameworks()
        _cache_ts = now
        return

    # Build auth header: prefer caller's Bearer token, fall back to proxy key
    # only when service auth is explicitly enabled.
    headers: dict[str, str] = {}
    try:
        from dsc_api.services.auth_context import get_current_principal
        principal = get_current_principal()
        if principal and principal.bearer_token:
            headers["Authorization"] = f"Bearer {principal.bearer_token}"
    except Exception:
        pass
    if "Authorization" not in headers and is_service_auth_allowed():
        proxy_key = get_llm_proxy_key()
        if proxy_key:
            headers["X-DSC-Proxy-Key"] = proxy_key

    # Fetch frameworks (graceful on failure: custom frameworks still load)
    try:
        req = urllib.request.Request(
            f"{proxy_url}/vuln/compliance-frameworks",
            headers=headers,
        )
        with urllib.request.urlopen(req, timeout=15) as resp:
            _frameworks_cache = _json.loads(resp.read())
    except Exception as exc:
        logger.error("Failed to fetch compliance_frameworks: %s", exc)
        _frameworks_cache = []

    # Fetch controls grouped by framework
    try:
        req = urllib.request.Request(
            f"{proxy_url}/vuln/compliance-controls",
            headers=headers,
        )
        with urllib.request.urlopen(req, timeout=15) as resp:
            all_controls = _json.loads(resp.read())
        grouped: dict[str, list[dict[str, Any]]] = {}
        for d in all_controls:
            fw = d.get("framework", "")
            grouped.setdefault(fw, []).append(d)
        _controls_cache = grouped
    except Exception as exc:
        logger.error("Failed to fetch compliance_controls: %s", exc)
        _controls_cache = {}

    _merge_custom_frameworks()

    _cache_ts = now
    total_controls = sum(len(v) for v in _controls_cache.values())
    logger.info(
        "Compliance cache refreshed: %d frameworks, %d controls",
        len(_frameworks_cache),
        total_controls,
    )


def invalidate_cache() -> None:
    """Force the next call to re-fetch frameworks and controls.

    Called after custom framework CRUD operations so the newly saved
    framework is immediately visible to the evaluator.
    """
    global _cache_ts
    _cache_ts = 0.0


# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------


def get_frameworks() -> list[ComplianceFramework]:
    """Return all available compliance frameworks."""
    _refresh_cache_if_needed()
    result: list[ComplianceFramework] = []
    for fw in _frameworks_cache:
        cats = []
        for c in fw.get("categories", []):
            cats.append({
                "id": c.get("id", ""),
                "name": c.get("name", ""),
                "controlCount": c.get("control_count", c.get("controlCount", 0)),
            })
        result.append(ComplianceFramework(
            frameworkId=fw.get("framework_id", ""),
            displayName=fw.get("display_name", ""),
            fullName=fw.get("full_name", ""),
            version=fw.get("version", ""),
            totalControls=fw.get("total_controls", 0),
            codeRelevantControls=fw.get("code_relevant_controls", 0),
            description=fw.get("description", ""),
            defaultAttestationExpiryDays=fw.get("default_attestation_expiry_days", 90),
            categories=cats,
        ))
    return result


def get_controls(framework_id: str) -> list[dict[str, Any]]:
    """Return raw control dicts for a framework."""
    _refresh_cache_if_needed()
    # Try exact match first, then case-insensitive
    controls = _controls_cache.get(framework_id)
    if controls is None:
        upper = framework_id.upper()
        for key, val in _controls_cache.items():
            if key.upper() == upper:
                controls = val
                break
    return controls or []


def _finding_fingerprint(finding: dict[str, Any]) -> str:
    """Return a stable fingerprint for a finding, using metadata when present."""
    fingerprint = finding.get("fingerprint")
    if isinstance(fingerprint, str) and fingerprint:
        return fingerprint

    metadata = finding.get("metadata")
    if isinstance(metadata, dict):
        meta_fingerprint = metadata.get("fingerprint")
        if isinstance(meta_fingerprint, str) and meta_fingerprint:
            return meta_fingerprint

    parts = [
        str(finding.get("rule_id", "")),
        str(finding.get("cwe", "")),
        str(finding.get("file_path", "")),
        str(finding.get("line_start", "")),
        str(finding.get("line_end", "")),
        str(finding.get("message", "")),
    ]
    return "|".join(parts)


def _group_findings_by_cwe(
    findings: list[dict[str, Any]] | None,
) -> tuple[dict[str, int], dict[str, list[dict[str, Any]]]]:
    """Return finding counts and finding rows grouped by CWE."""
    cwe_finding_counts: dict[str, int] = {}
    findings_by_cwe: dict[str, list[dict[str, Any]]] = {}

    for raw in findings or []:
        if not isinstance(raw, dict):
            continue
        cwe = str(raw.get("cwe", ""))
        if not cwe:
            continue
        cwe_finding_counts[cwe] = cwe_finding_counts.get(cwe, 0) + 1
        findings_by_cwe.setdefault(cwe, []).append(raw)

    return cwe_finding_counts, findings_by_cwe


def _build_breakdown(
    evaluations: list[ControlEvaluation],
    *,
    key_fn,
) -> dict[str, GroupBreakdown]:
    """Aggregate evaluations into per-group breakdowns (Phase 5).

    The same shape serves ``category_breakdown`` (one bucket per category)
    and ``family_breakdown`` (one bucket per family; a control may
    contribute to multiple families). Groups with zero applicable controls
    report 100.0% coverage / clean to match the framework-level convention.
    """
    buckets: dict[str, dict[ControlStatus, int]] = {}
    for ev in evaluations:
        for key in key_fn(ev):
            counts = buckets.setdefault(key, {s: 0 for s in ControlStatus})
            counts[ev.status] = counts.get(ev.status, 0) + 1

    out: dict[str, GroupBreakdown] = {}
    for key in sorted(buckets):
        c = buckets[key]
        total = sum(c.values())
        na = c.get(ControlStatus.NOT_APPLICABLE, 0)
        applicable = total - na
        clean = c.get(ControlStatus.COVERED_CLEAN, 0)
        viol = c.get(ControlStatus.COVERED_VIOLATIONS, 0)
        attested = c.get(ControlStatus.ATTESTED, 0)
        partial = c.get(ControlStatus.PARTIALLY_COVERED, 0)
        if applicable > 0:
            coverage_pct = round((clean + viol + attested) / applicable * 100, 1)
            clean_pct = round((clean + attested) / applicable * 100, 1)
        else:
            coverage_pct = 100.0
            clean_pct = 100.0
        out[key] = GroupBreakdown(
            name=key,
            totalCodeRelevant=total,
            coveredClean=clean,
            coveredViolations=viol,
            partiallyCovered=partial,
            notCovered=c.get(ControlStatus.NOT_COVERED, 0),
            attestationRequired=c.get(ControlStatus.ATTESTATION_REQUIRED, 0),
            attested=attested,
            notApplicable=na,
            coveragePct=coverage_pct,
            cleanPct=clean_pct,
        )
    return out


def _matched_findings_for_control(
    mapped_cwes: list[str],
    findings_by_cwe: dict[str, list[dict[str, Any]]],
) -> list[dict[str, Any]]:
    """Return unique findings that map to a control's CWEs."""
    by_fingerprint: dict[str, dict[str, Any]] = {}
    for cwe in mapped_cwes:
        for finding in findings_by_cwe.get(cwe, []):
            by_fingerprint[_finding_fingerprint(finding)] = finding
    return list(by_fingerprint.values())


def _finding_fingerprints_for_control(
    mapped_cwes: list[str],
    findings_by_cwe: dict[str, list[dict[str, Any]]],
) -> list[str]:
    """Return sorted finding fingerprints relevant to a control."""
    return sorted(
        _finding_fingerprint(finding)
        for finding in _matched_findings_for_control(mapped_cwes, findings_by_cwe)
    )


def evaluate(
    frameworks: list[str],
    findings: list[dict[str, Any]] | None = None,
    active_detector_cwes: set[str] | None = None,
    repo_identifier: str = "",
    user_id: str = "",
) -> ComplianceReport:
    """Evaluate compliance for the given frameworks against findings.

    The caller must supply the active detector CWE set from the completed
    scan. This keeps coverage aligned with the actual scan configuration
    instead of inferring it from the full detector registry.
    """
    _refresh_cache_if_needed()
    if active_detector_cwes is None:
        raise ValueError("active_detector_cwes is required")

    _, findings_by_cwe = _group_findings_by_cwe(findings)
    finding_cwes = set(findings_by_cwe.keys())
    active_detector_cwes = {str(cwe) for cwe in active_detector_cwes if cwe}
    active_detector_cwes.update(finding_cwes)

    # Fetch active attestations for this user/repo
    attestations_by_key: dict[str, dict[str, Any]] = {}
    if user_id and repo_identifier:
        for fw_id in frameworks:
            for att in _fetch_attestations(user_id, repo_identifier, fw_id.upper()):
                key = f"{att.get('framework', '')}:{att.get('control_id', '')}"
                attestations_by_key[key] = att

    fw_reports: dict[str, FrameworkReport] = {}
    finding_fingerprints_by_framework: dict[str, dict[str, list[str]]] = {}

    for fw_id in frameworks:
        fw_key = fw_id.upper()
        controls = get_controls(fw_key)
        if not controls:
            continue

        # Find framework metadata
        fw_meta = None
        for fw in _frameworks_cache:
            if fw.get("framework_id", "").upper() == fw_key:
                fw_meta = fw
                break

        default_expiry_days = (
            fw_meta.get("default_attestation_expiry_days", 90) if fw_meta else 90
        )

        evaluations: list[ControlEvaluation] = []
        counts = {s: 0 for s in ControlStatus}
        control_fingerprints: dict[str, list[str]] = {}

        for ctrl in controls:
            if not ctrl.get("code_relevant", True):
                continue

            control_id = ctrl.get("control_id", "")
            detection_method = ctrl.get("detection_method", "scanner")
            mapped = [str(cwe) for cwe in ctrl.get("mapped_cwes", [])]
            matching_fingerprints = _finding_fingerprints_for_control(mapped, findings_by_cwe)
            control_fingerprints[control_id] = matching_fingerprints

            # Check for active attestation on this control
            att_key = f"{fw_key}:{control_id}"
            attestation = attestations_by_key.get(att_key)
            att_active = _is_attestation_active(attestation, ctrl, default_expiry_days)

            if att_active and attestation:
                att_status = attestation.get("status", "attested")
                if att_status == "not_applicable":
                    status = ControlStatus.NOT_APPLICABLE
                else:
                    status = ControlStatus.ATTESTED
                covered: list[str] = list(mapped)
                uncovered: list[str] = []
                fc = 0
            elif detection_method == "attestation":
                status = ControlStatus.ATTESTATION_REQUIRED
                covered = []
                uncovered = list(mapped)
                fc = 0
            else:
                # Determine which mapped CWEs have active detectors
                covered = [c for c in mapped if c in active_detector_cwes]
                uncovered = [c for c in mapped if c not in active_detector_cwes]

                # Count findings for this control
                fc = len(matching_fingerprints)

                if not covered:
                    status = ControlStatus.NOT_COVERED
                elif uncovered and fc > 0:
                    status = ControlStatus.COVERED_VIOLATIONS
                elif uncovered:
                    status = ControlStatus.PARTIALLY_COVERED
                elif fc > 0:
                    status = ControlStatus.COVERED_VIOLATIONS
                else:
                    status = ControlStatus.COVERED_CLEAN

            counts[status] = counts.get(status, 0) + 1

            evaluations.append(ControlEvaluation(
                controlId=control_id,
                framework=fw_key,
                controlName=ctrl.get("control_name", ""),
                status=status,
                severityWeight=ctrl.get("severity_weight", "medium"),
                mappedCwes=mapped,
                coveredCwes=covered,
                uncoveredCwes=uncovered,
                findingCount=fc,
                evidenceGuidance=ctrl.get("evidence_guidance", ""),
                category=str(ctrl.get("category", "")),
                families=[str(f) for f in (ctrl.get("families") or [])],
            ))

        covered_clean = counts.get(ControlStatus.COVERED_CLEAN, 0)
        covered_violations = counts.get(ControlStatus.COVERED_VIOLATIONS, 0)
        attested = counts.get(ControlStatus.ATTESTED, 0)
        partially_covered = counts.get(ControlStatus.PARTIALLY_COVERED, 0)
        na = counts.get(ControlStatus.NOT_APPLICABLE, 0)

        # Exclude NOT_APPLICABLE from the denominator -- those controls are
        # removed from scope entirely and should not deflate percentages.
        applicable_total = len(evaluations) - na
        if applicable_total > 0:
            coverage_pct = round(
                (covered_clean + covered_violations + attested) / applicable_total * 100, 1
            )
            partial_coverage_pct = round(partially_covered / applicable_total * 100, 1)
            clean_pct = round((covered_clean + attested) / applicable_total * 100, 1)
        else:
            coverage_pct = 100.0
            partial_coverage_pct = 0.0
            clean_pct = 100.0

        category_breakdown = _build_breakdown(
            evaluations, key_fn=lambda e: [e.category] if e.category else [],
        )
        family_breakdown = _build_breakdown(
            evaluations, key_fn=lambda e: list(e.families) if e.families else [],
        )

        fw_reports[fw_key] = FrameworkReport(
            frameworkId=fw_key,
            displayName=fw_meta.get("display_name", fw_key) if fw_meta else fw_key,
            totalCodeRelevant=len(evaluations),
            coveredClean=covered_clean,
            coveredViolations=covered_violations,
            partiallyCovered=partially_covered,
            notCovered=counts.get(ControlStatus.NOT_COVERED, 0),
            attestationRequired=counts.get(ControlStatus.ATTESTATION_REQUIRED, 0),
            attested=attested,
            notApplicable=counts.get(ControlStatus.NOT_APPLICABLE, 0),
            coveragePct=coverage_pct,
            partialCoveragePct=partial_coverage_pct,
            cleanPct=clean_pct,
            controls=evaluations,
            categoryBreakdown=category_breakdown,
            familyBreakdown=family_breakdown,
        )
        finding_fingerprints_by_framework[fw_key] = control_fingerprints

    # Overall score: weighted average of clean_pct by applicable control count
    # (total_code_relevant minus notApplicable).  Frameworks with zero
    # applicable controls contribute zero weight.
    total_applicable = sum(
        r.total_code_relevant - r.not_applicable for r in fw_reports.values()
    )
    if total_applicable > 0:
        overall = sum(
            r.clean_pct * (r.total_code_relevant - r.not_applicable)
            for r in fw_reports.values()
        ) / total_applicable
    else:
        overall = 0.0

    # Compute drift against baselines
    drift: DriftReport | None = None
    if user_id and repo_identifier:
        drift = _compute_drift_for_report(
            fw_reports,
            user_id,
            repo_identifier,
            finding_fingerprints_by_framework,
        )

    # Phase 6: attach the scope envelope (canonical disclaimer + per-framework
    # warnings) so any consumer of /api/compliance/evaluate sees the scope
    # without parsing free-text fields elsewhere.
    from dsc.compliance.scope_disclaimer import api_scope_envelope

    return ComplianceReport(
        reportId=str(uuid.uuid4()),
        timestamp=datetime.now(timezone.utc).isoformat(),
        repoIdentifier=repo_identifier,
        frameworks=fw_reports,
        overallScore=round(overall, 1),
        drift=drift,
        scope=api_scope_envelope(list(fw_reports.keys())),
    )


# ---------------------------------------------------------------------------
# Attestation helpers
# ---------------------------------------------------------------------------

def _is_attestation_active(
    attestation: dict[str, Any] | None,
    ctrl: dict[str, Any],
    default_expiry_days: int,
) -> bool:
    """Return True if the attestation exists and has not expired.

    Fail-closed: missing, empty, or unparseable ``expires_at`` values are
    treated as expired.  No attestation should be immortal.
    """
    if not attestation:
        return False
    expires_at = attestation.get("expires_at", "")
    if not expires_at:
        return False
    try:
        expiry = datetime.fromisoformat(expires_at)
        if expiry.tzinfo is None:
            expiry = expiry.replace(tzinfo=timezone.utc)
        return datetime.now(timezone.utc) < expiry
    except (ValueError, TypeError):
        return False


def _fetch_attestations(
    user_id: str,
    repo_identifier: str,
    framework: str,
) -> list[dict[str, Any]]:
    """Fetch active attestations for a user/repo/framework via storage backend."""
    return get_storage().list_attestations(user_id, repo_identifier, framework)


def save_attestation(
    user_id: str,
    user_email: str,
    framework: str,
    control_id: str,
    repo_identifier: str,
    note: str,
    status: str = "attested",
    expiry_days: int | None = None,
    activity_event: str = "added",
) -> Attestation:
    """Create or update an attestation via the configured storage backend."""
    _refresh_cache_if_needed()
    framework_key = framework.upper()
    storage = get_storage()

    if expiry_days is None:
        expiry_days = _resolve_expiry_days(framework_key, control_id)

    now = datetime.now(timezone.utc)
    from datetime import timedelta
    expires_at = (now + timedelta(days=expiry_days)).isoformat() if expiry_days > 0 else ""
    now_iso = now.isoformat()

    # Capture IDs of any existing active attestations for this control so the
    # activity log can record a "superseded" event for each one before the
    # storage layer marks them revoked. Preserves the audit trail instead of
    # overwriting.
    superseded_ids: list[str] = [
        a.get("attestation_id", "")
        for a in storage.list_attestations(user_id, repo_identifier, framework_key)
        if a.get("control_id") == control_id and not a.get("revoked")
    ]
    storage.supersede_active_attestations(
        user_id, repo_identifier, framework_key, control_id,
        at=now_iso,
        by=user_id,
    )

    attestation_id = str(uuid.uuid4())
    record = {
        "attestation_id": attestation_id,
        "user_id": user_id,
        "user_email": user_email,
        "repo_identifier": repo_identifier,
        "framework": framework_key,
        "control_id": control_id,
        "status": status,
        "note": note,
        "attested_at": now_iso,
        "expires_at": expires_at,
        "revoked": False,
        "revoked_at": "",
        "revoked_by": "",
        "revoked_reason": "",
    }
    storage.save_attestation(record)
    logger.info("Attestation saved: %s / %s / %s", framework_key, control_id, status)

    # Record activity events: one for any superseded prior, one for the new add.
    for prior_id in superseded_ids:
        _record_activity(
            user_id=user_id,
            user_email=user_email,
            framework=framework_key,
            control_id=control_id,
            repo_identifier=repo_identifier,
            event="superseded",
            note=f"Replaced by {attestation_id}",
            attestation_id=prior_id,
            timestamp=now,
        )
    _record_activity(
        user_id=user_id,
        user_email=user_email,
        framework=framework_key,
        control_id=control_id,
        repo_identifier=repo_identifier,
        event=activity_event,
        note=note,
        attestation_id=attestation_id,
        timestamp=now,
    )

    return Attestation(
        attestationId=attestation_id,
        userId=user_id,
        userEmail=user_email,
        repoIdentifier=repo_identifier,
        framework=framework_key,
        controlId=control_id,
        status=status,
        note=note,
        attestedAt=now_iso,
        expiresAt=expires_at,
    )


def get_attestations(
    user_id: str,
    repo_identifier: str,
    framework: str,
) -> list[Attestation]:
    """Return all attestations for a user/repo/framework."""
    raw = _fetch_attestations(user_id, repo_identifier, framework)
    result: list[Attestation] = []
    for att in raw:
        result.append(Attestation(
            attestationId=att.get("attestation_id", ""),
            userId=att.get("user_id", ""),
            userEmail=att.get("user_email", ""),
            repoIdentifier=att.get("repo_identifier", ""),
            framework=att.get("framework", ""),
            controlId=att.get("control_id", ""),
            status=att.get("status", "attested"),
            note=att.get("note", ""),
            attestedAt=att.get("attested_at", ""),
            expiresAt=att.get("expires_at", ""),
        ))
    return result


def revoke_attestation(
    attestation_id: str,
    user_id: str,
    reason: str = "",
    activity_event: str = "removed",
) -> bool:
    """Soft-delete an attestation by ID.  Returns True if found and revoked.

    Sets ``revoked=True``, ``revoked_at``, and ``revoked_by`` on the record
    rather than deleting it, preserving an audit trail.
    """
    revoked_at = datetime.now(timezone.utc)
    record = get_storage().revoke_attestation(
        attestation_id, user_id, at=revoked_at.isoformat(),
    )
    if record is None:
        return False
    logger.info("Attestation revoked: %s", attestation_id)
    _record_activity(
        user_id=user_id,
        user_email=record.get("user_email", ""),
        framework=record.get("framework", ""),
        control_id=record.get("control_id", ""),
        repo_identifier=record.get("repo_identifier", ""),
        event=activity_event,
        note=reason,
        attestation_id=attestation_id,
        timestamp=revoked_at,
    )
    return True


# ---------------------------------------------------------------------------
# Bulk attestation operations
# ---------------------------------------------------------------------------

def bulk_attest(
    user_id: str,
    user_email: str,
    framework: str,
    repo_identifier: str,
    control_ids: list[str],
    note: str = "",
    status: str = "attested",
) -> BulkAttestResult:
    """Attest a batch of controls in a single call.

    Best-effort: each control is attempted independently.  Per-control
    failures are reported in ``failed`` rather than aborting the batch.
    Each successful save still goes through ``save_attestation`` so the
    soft-revoke and activity-log invariants are preserved.
    """
    succeeded: list[Attestation] = []
    failed: list[dict] = []
    seen: set[str] = set()
    for control_id in control_ids:
        if not control_id or control_id in seen:
            continue
        seen.add(control_id)
        try:
            att = save_attestation(
                user_id=user_id,
                user_email=user_email,
                framework=framework,
                control_id=control_id,
                repo_identifier=repo_identifier,
                note=note,
                status=status,
                activity_event="bulk_added",
            )
            succeeded.append(att)
        except Exception as exc:
            logger.error(
                "bulk_attest failed for %s/%s: %s", framework, control_id, exc,
            )
            failed.append({"controlId": control_id, "error": str(exc)})
    return BulkAttestResult(succeeded=succeeded, failed=failed)


def bulk_revoke(
    user_id: str,
    attestation_ids: list[str],
    reason: str = "",
) -> BulkRevokeResult:
    """Revoke a batch of attestations.  Reports IDs that were not found."""
    revoked = 0
    not_found: list[str] = []
    for aid in attestation_ids:
        if not aid:
            continue
        try:
            ok = revoke_attestation(
                aid, user_id, reason=reason, activity_event="bulk_removed",
            )
            if ok:
                revoked += 1
            else:
                not_found.append(aid)
        except Exception as exc:
            logger.error("bulk_revoke failed for %s: %s", aid, exc)
            not_found.append(aid)
    return BulkRevokeResult(revoked=revoked, notFound=not_found)


# ---------------------------------------------------------------------------
# Attestation activity log
# ---------------------------------------------------------------------------

def _record_activity(
    user_id: str,
    user_email: str,
    framework: str,
    control_id: str,
    repo_identifier: str,
    event: str,
    note: str,
    attestation_id: str,
    timestamp: datetime,
) -> None:
    """Append an activity event to the per-repo/framework attestation log.

    Activity events are write-only; the UI renders them as a timeline and the
    evidence package can snapshot them.  Best-effort: write failures log a
    warning but never raise so the attestation operation itself still
    succeeds.
    """
    db = _get_db()
    if db is None:
        return
    try:
        activity_id = str(uuid.uuid4())
        data = {
            "activity_id": activity_id,
            "user_id": user_id,
            "user_email": user_email,
            "framework": framework.upper(),
            "control_id": control_id,
            "repo_identifier": repo_identifier,
            "event": event,
            "note": note,
            "attestation_id": attestation_id,
            "timestamp": timestamp.isoformat(),
        }
        db.collection("compliance_attestation_activity").document(activity_id).set(data)
    except Exception as exc:
        logger.warning("Failed to record attestation activity: %s", exc)


def get_attestation_activity(
    user_id: str,
    repo_identifier: str,
    framework: str,
    limit: int = 200,
) -> list[AttestationActivity]:
    """Return the attestation activity log for a user/repo/framework.

    Returned events are sorted by timestamp descending (most recent first)
    and capped at ``limit``.  When Firestore is unavailable the log is
    treated as empty rather than raising; the UI renders an empty
    timeline in that case.
    """
    db = _get_db()
    if db is None:
        return []
    framework_key = framework.upper()
    try:
        docs = (
            db.collection("compliance_attestation_activity")
            .where("user_id", "==", user_id)
            .where("repo_identifier", "==", repo_identifier)
            .where("framework", "==", framework_key)
            .stream()
        )
        events: list[AttestationActivity] = []
        for doc in docs:
            data = doc.to_dict()
            events.append(AttestationActivity(
                activityId=data.get("activity_id", doc.id),
                userId=data.get("user_id", ""),
                userEmail=data.get("user_email", ""),
                repoIdentifier=data.get("repo_identifier", ""),
                framework=data.get("framework", ""),
                controlId=data.get("control_id", ""),
                event=data.get("event", ""),
                note=data.get("note", ""),
                timestamp=data.get("timestamp", ""),
                attestationId=data.get("attestation_id", ""),
            ))
        events.sort(key=lambda e: e.timestamp, reverse=True)
        return events[:limit]
    except Exception as exc:
        logger.warning("Failed to fetch attestation activity: %s", exc)
        return []


def _resolve_expiry_days(framework: str, control_id: str) -> int:
    """Determine attestation expiry days from control, then framework defaults."""
    # Check control-level override
    controls = get_controls(framework)
    for ctrl in controls:
        if ctrl.get("control_id") == control_id:
            ctrl_expiry = ctrl.get("attestation_expiry_days")
            if ctrl_expiry is not None:
                return int(ctrl_expiry)
            break

    # Check framework-level default
    for fw in _frameworks_cache:
        if fw.get("framework_id", "").upper() == framework.upper():
            return fw.get("default_attestation_expiry_days", 90)

    return 90


# ---------------------------------------------------------------------------
# Baseline management
# ---------------------------------------------------------------------------

def save_baseline(
    user_id: str,
    framework: str,
    repo_identifier: str,
    findings: list[dict[str, Any]] | None,
    active_detector_cwes: set[str],
) -> ComplianceBaseline:
    """Snapshot current compliance state as a baseline via storage backend."""
    _refresh_cache_if_needed()
    framework_key = framework.upper()
    report = evaluate(
        frameworks=[framework_key],
        findings=findings,
        active_detector_cwes=active_detector_cwes,
        repo_identifier=repo_identifier,
        user_id=user_id,
    )
    fw_report = report.frameworks.get(framework_key)
    if not fw_report:
        raise ValueError(f"No controls found for framework '{framework}'")

    control_statuses: dict[str, str] = {
        ctrl.control_id: ctrl.status.value for ctrl in fw_report.controls
    }

    _, findings_by_cwe = _group_findings_by_cwe(findings)
    control_finding_fingerprints = {
        ctrl.control_id: _finding_fingerprints_for_control(list(ctrl.mapped_cwes), findings_by_cwe)
        for ctrl in fw_report.controls
    }

    now = datetime.now(timezone.utc)
    baseline_id = str(uuid.uuid4())

    fw_version_at = ""
    for fw in _frameworks_cache:
        if fw.get("framework_id", "").upper() == framework.upper():
            fw_version_at = fw.get("updated_at", "")
            break

    finding_count = sum(c.finding_count for c in fw_report.controls)

    record = {
        "baseline_id": baseline_id,
        "user_id": user_id,
        "repo_identifier": repo_identifier,
        "framework": framework_key,
        "created_at": now.isoformat(),
        "is_latest": True,
        "framework_version_at_creation": fw_version_at,
        "coverage_pct": fw_report.coverage_pct,
        "clean_pct": fw_report.clean_pct,
        "control_statuses": control_statuses,
        "control_finding_fingerprints": control_finding_fingerprints,
        "finding_count": finding_count,
    }
    get_storage().save_baseline(record)
    logger.info("Baseline saved: %s / %s", framework_key, baseline_id)

    return ComplianceBaseline(
        baselineId=baseline_id,
        framework=framework_key,
        createdAt=now.isoformat(),
        coveragePct=fw_report.coverage_pct,
        cleanPct=fw_report.clean_pct,
        controlStatuses=control_statuses,
        controlFindingFingerprints=control_finding_fingerprints,
        findingCount=finding_count,
    )


def get_baseline(
    user_id: str,
    repo_identifier: str,
    framework: str,
) -> ComplianceBaseline | None:
    """Return the latest baseline for a user/repo/framework, or None."""
    framework_key = framework.upper()
    record = get_storage().get_latest_baseline(user_id, repo_identifier, framework_key)
    if not record:
        return None
    return ComplianceBaseline(
        baselineId=record.get("baseline_id", ""),
        framework=record.get("framework", ""),
        createdAt=record.get("created_at", ""),
        coveragePct=record.get("coverage_pct", 0.0),
        cleanPct=record.get("clean_pct", 0.0),
        controlStatuses=record.get("control_statuses", {}),
        controlFindingFingerprints=record.get("control_finding_fingerprints", {}),
        findingCount=record.get("finding_count", 0),
    )


# ---------------------------------------------------------------------------
# Drift computation
# ---------------------------------------------------------------------------

def compute_drift(
    user_id: str,
    repo_identifier: str,
    framework: str,
    findings: list[dict[str, Any]] | None,
    active_detector_cwes: set[str],
) -> DriftReport | None:
    """Compute drift between current state and the latest baseline."""
    framework_key = framework.upper()
    baseline = get_baseline(user_id, repo_identifier, framework_key)
    if not baseline:
        return None

    report = evaluate(
        frameworks=[framework_key],
        findings=findings,
        active_detector_cwes=active_detector_cwes,
        repo_identifier=repo_identifier,
        user_id=user_id,
    )
    fw_report = report.frameworks.get(framework_key)
    if not fw_report:
        return None

    _, findings_by_cwe = _group_findings_by_cwe(findings)
    current_fingerprints = {
        ctrl.control_id: _finding_fingerprints_for_control(list(ctrl.mapped_cwes), findings_by_cwe)
        for ctrl in fw_report.controls
    }

    return _build_drift(
        baseline,
        fw_report,
        user_id,
        repo_identifier,
        framework_key,
        current_fingerprints,
    )


def _compute_drift_for_report(
    fw_reports: dict[str, FrameworkReport],
    user_id: str,
    repo_identifier: str,
    finding_fingerprints_by_framework: dict[str, dict[str, list[str]]],
) -> DriftReport | None:
    """Compute drift across all evaluated frameworks against their baselines.

    Returns a single combined drift report (or None if no baselines exist).
    """
    all_new_violations: list[str] = []
    all_resolved: list[str] = []
    all_new_gaps: list[str] = []
    all_closed_gaps: list[str] = []
    all_expired: list[str] = []
    total_coverage_change = 0.0
    baseline_count = 0
    latest_baseline_date = ""

    for fw_id, fw_report in fw_reports.items():
        baseline = get_baseline(user_id, repo_identifier, fw_id)
        if not baseline:
            continue
        drift = _build_drift(
            baseline,
            fw_report,
            user_id,
            repo_identifier,
            fw_id,
            finding_fingerprints_by_framework.get(fw_id, {}),
        )
        if not drift:
            continue

        baseline_count += 1
        total_coverage_change += drift.coverage_change
        all_new_violations.extend(drift.new_violations)
        all_resolved.extend(drift.resolved_violations)
        all_new_gaps.extend(drift.new_gaps)
        all_closed_gaps.extend(drift.closed_gaps)
        all_expired.extend(drift.attestations_expired)
        if drift.baseline_date > latest_baseline_date:
            latest_baseline_date = drift.baseline_date

    if baseline_count == 0:
        return None

    avg_change = round(total_coverage_change / baseline_count, 1)

    # Determine direction
    if len(all_resolved) > len(all_new_violations) and avg_change >= 0:
        direction = "improving"
    elif len(all_new_violations) > len(all_resolved) or avg_change < -1:
        direction = "regressing"
    else:
        direction = "stable"

    return DriftReport(
        baselineDate=latest_baseline_date,
        coverageChange=avg_change,
        newViolations=all_new_violations,
        resolvedViolations=all_resolved,
        newGaps=all_new_gaps,
        closedGaps=all_closed_gaps,
        attestationsExpired=all_expired,
        overallDirection=direction,
    )


def _build_drift(
    baseline: ComplianceBaseline,
    fw_report: FrameworkReport,
    user_id: str,
    repo_identifier: str,
    framework: str,
    current_finding_fingerprints: dict[str, list[str]],
) -> DriftReport | None:
    """Build a drift report comparing a framework report to its baseline."""
    baseline_statuses = baseline.control_statuses
    current_statuses: dict[str, str] = {}
    for ctrl in fw_report.controls:
        current_statuses[ctrl.control_id] = ctrl.status.value

    new_violations: list[str] = []
    resolved: list[str] = []
    new_gaps: list[str] = []
    closed_gaps: list[str] = []

    clean_like = {"covered_clean", "attested"}
    violation_like = {"covered_violations"}
    gap_like = {"not_covered", "partially_covered"}

    if baseline.control_finding_fingerprints:
        all_fingerprint_control_ids = set(baseline.control_finding_fingerprints.keys()) | set(current_finding_fingerprints.keys())
        for cid in all_fingerprint_control_ids:
            old_fingerprints = set(baseline.control_finding_fingerprints.get(cid, []))
            new_fingerprints = set(current_finding_fingerprints.get(cid, []))
            if new_fingerprints - old_fingerprints:
                new_violations.append(cid)
            elif old_fingerprints and not new_fingerprints:
                resolved.append(cid)
    else:
        # Older baselines only tracked control status, so keep the legacy
        # comparison for backward compatibility.
        all_control_ids = set(baseline_statuses.keys()) | set(current_statuses.keys())
        for cid in all_control_ids:
            old = baseline_statuses.get(cid, "")
            new = current_statuses.get(cid, "")
            if not old or not new:
                continue  # new or retired control

            if old in clean_like and new in violation_like:
                new_violations.append(cid)
            elif old in violation_like and new in clean_like:
                resolved.append(cid)

    all_control_ids = set(baseline_statuses.keys()) | set(current_statuses.keys())
    for cid in all_control_ids:
        old = baseline_statuses.get(cid, "")
        new = current_statuses.get(cid, "")
        if not old or not new:
            continue  # new or retired control

        if old not in gap_like and new in gap_like:
            new_gaps.append(cid)
        elif old in gap_like and new not in gap_like:
            closed_gaps.append(cid)

    # Check for attestations expiring within 14 days
    attestations_expired: list[str] = []
    now = datetime.now(timezone.utc)
    from datetime import timedelta
    warning_threshold = now + timedelta(days=14)

    for att in _fetch_attestations(user_id, repo_identifier, framework):
        expires_at = att.get("expires_at", "")
        if not expires_at:
            continue
        try:
            expiry = datetime.fromisoformat(expires_at)
            if expiry.tzinfo is None:
                expiry = expiry.replace(tzinfo=timezone.utc)
            if expiry <= warning_threshold:
                attestations_expired.append(att.get("control_id", ""))
        except (ValueError, TypeError):
            pass

    coverage_change = round(fw_report.coverage_pct - baseline.coverage_pct, 1)

    if len(resolved) > len(new_violations) and coverage_change >= 0:
        direction = "improving"
    elif len(new_violations) > len(resolved) or coverage_change < -1:
        direction = "regressing"
    else:
        direction = "stable"

    return DriftReport(
        baselineDate=baseline.created_at,
        coverageChange=coverage_change,
        newViolations=new_violations,
        resolvedViolations=resolved,
        newGaps=new_gaps,
        closedGaps=closed_gaps,
        attestationsExpired=attestations_expired,
        overallDirection=direction,
    )


# ---------------------------------------------------------------------------
# Evidence packaging (Phase 3)
# ---------------------------------------------------------------------------

def generate_evidence(
    framework: str,
    fmt: str = "json",
    repo_identifier: str = "",
    user_id: str = "",
    user_email: str = "",
    findings: list[dict[str, Any]] | None = None,
    active_detector_cwes: set[str] | None = None,
) -> EvidencePackage | str:
    """Generate an evidence package for a single framework.

    Returns an ``EvidencePackage`` when *fmt* is ``"json"``, or an HTML
    string when *fmt* is ``"html"``.
    """
    _refresh_cache_if_needed()

    # Run evaluation to get the current report
    report = evaluate(
        frameworks=[framework.upper()],
        findings=findings,
        active_detector_cwes=active_detector_cwes,
        repo_identifier=repo_identifier,
        user_id=user_id,
    )
    framework_key = framework.upper()
    fw_report = report.frameworks.get(framework_key)
    if not fw_report:
        raise ValueError(f"No controls found for framework '{framework}'")

    # Resolve framework metadata
    fw_version = ""
    for fw in _frameworks_cache:
        if fw.get("framework_id", "").upper() == framework_key:
            fw_version = fw.get("version", "")
            break

    # Fetch attestations for evidence items
    attestations_by_control: dict[str, dict[str, Any]] = {}
    if user_id and repo_identifier:
        for att in _fetch_attestations(user_id, repo_identifier, framework_key):
            attestations_by_control[att.get("control_id", "")] = att

    _, findings_by_cwe = _group_findings_by_cwe(findings)

    # Build per-control evidence
    controls_evidence: list[ControlEvidence] = []
    for ctrl in fw_report.controls:
        items: list[EvidenceItem] = []
        matched_findings = _matched_findings_for_control(list(ctrl.mapped_cwes), findings_by_cwe)

        if ctrl.status in (ControlStatus.COVERED_VIOLATIONS, ControlStatus.COVERED_CLEAN, ControlStatus.PARTIALLY_COVERED):
            items.append(EvidenceItem(
                evidenceType="scan_result",
                timestamp=report.timestamp,
                details={
                    "findingCount": len(matched_findings),
                    "findingFingerprints": [
                        _finding_fingerprint(finding)
                        for finding in matched_findings
                    ],
                    "files": sorted({
                        str(finding.get("file_path", ""))
                        for finding in matched_findings
                        if finding.get("file_path")
                    }),
                    "ruleIds": sorted({
                        str(finding.get("rule_id", ""))
                        for finding in matched_findings
                        if finding.get("rule_id")
                    }),
                    "coveredCwes": list(ctrl.covered_cwes),
                    "uncoveredCwes": list(ctrl.uncovered_cwes),
                    "status": ctrl.status.value,
                },
            ))

        att = attestations_by_control.get(ctrl.control_id)
        if att:
            items.append(EvidenceItem(
                evidenceType="attestation",
                timestamp=att.get("attested_at", ""),
                details={
                    "attestedBy": att.get("user_email", ""),
                    "status": att.get("status", ""),
                    "note": att.get("note", ""),
                    "expiresAt": att.get("expires_at", ""),
                },
            ))

        if not items:
            items.append(EvidenceItem(
                evidenceType="no_evidence",
                timestamp=report.timestamp,
                details={"reason": ctrl.status.value},
            ))

        # Fetch full control description from cache
        description = ""
        for cached_ctrl in get_controls(framework_key):
            if cached_ctrl.get("control_id") == ctrl.control_id:
                description = cached_ctrl.get("description", "")
                break

        controls_evidence.append(ControlEvidence(
            controlId=ctrl.control_id,
            controlName=ctrl.control_name,
            description=description,
            status=ctrl.status,
            evidenceGuidance=ctrl.evidence_guidance,
            evidenceItems=items,
        ))

    now = datetime.now(timezone.utc)
    package = EvidencePackage(
        packageId=str(uuid.uuid4()),
        generatedAt=now.isoformat(),
        framework=framework_key,
        frameworkVersion=fw_version,
        repoIdentifier=repo_identifier,
        generatedBy=user_email,
        summary=fw_report,
        controls=controls_evidence,
        metadata={
            "scannerVersion": "1.0.0",
            "evaluationTimestamp": report.timestamp,
            "reportId": report.report_id,
        },
    )

    if fmt == "html":
        return _render_evidence_html(package)

    return package


def _render_evidence_html(package: EvidencePackage) -> str:
    """Render an evidence package to a self-contained HTML report."""
    import pathlib
    from jinja2 import Environment, FileSystemLoader

    template_dir = pathlib.Path(__file__).resolve().parent.parent / "templates"
    env = Environment(
        loader=FileSystemLoader(str(template_dir)),
        autoescape=True,
    )
    template = env.get_template("compliance_report.html.j2")

    # Prepare template context from the Pydantic model
    ctx = package.model_dump(by_alias=True)

    # Add convenience helpers for the template
    status_labels = {
        "covered_clean": "Clean",
        "covered_violations": "Violations Found",
        "partially_covered": "Partially Covered",
        "not_covered": "Gap",
        "attestation_required": "Attestation Required",
        "attested": "Attested",
        "not_applicable": "Not Applicable",
    }
    status_colors = {
        "covered_clean": "#3fb950",
        "covered_violations": "#f85149",
        "partially_covered": "#d29922",
        "not_covered": "#8b949e",
        "attestation_required": "#79c0ff",
        "attested": "#3fb950",
        "not_applicable": "#8b949e",
    }
    ctx["statusLabels"] = status_labels
    ctx["statusColors"] = status_colors

    return template.render(**ctx)
