#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
RACF Audit Reader V2
Advanced analyzer for RACF / ICH / IRR / security audit text exports.

The tool works on offline text exports and normalized command traces. It does not
connect to a live z/OS system. It parses security events, access denials,
privileged changes, sensitive dataset access, group membership drift, profile
changes, and policy risks.
"""

from __future__ import annotations

import argparse
import csv
import fnmatch
import html
import json
import os
import re
import sys
from collections import Counter, defaultdict
from dataclasses import asdict, dataclass, field
from datetime import datetime
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple

SEVERITY_ORDER = {"INFO": 1, "WARNING": 2, "ERROR": 3, "CRITICAL": 4}
EXIT_BY_SEVERITY = {"INFO": 0, "WARNING": 2, "ERROR": 3, "CRITICAL": 4}

DEFAULT_POLICY: Dict[str, Any] = {
    "sensitive_dataset_patterns": [
        "SYS1.*", "SYS2.*", "RACF.*", "SECURITY.*", "PAYROLL.*", "HR.*",
        "FIN.*", "PROD.*", "DB2.*", "DSN.*", "CICS.*", "IMS.*", "APPL.PROD.*"
    ],
    "privileged_groups": [
        "SYS1", "SPECIAL", "SECADMIN", "RACFADM", "AUDIT", "OPERATIONS", "DBA", "CICSADM"
    ],
    "privileged_authorities": ["SPECIAL", "OPERATIONS", "AUDITOR", "ROAUDIT", "CLAUTH", "GROUP-SPECIAL", "GRP-SPECIAL"],
    "dangerous_uacc": ["UPDATE", "CONTROL", "ALTER"],
    "critical_classes": ["DATASET", "FACILITY", "SURROGAT", "OPERCMDS", "STARTED", "PROGRAM", "APPL", "SERVER"],
    "critical_commands": ["ALTUSER", "ADDUSER", "DELUSER", "CONNECT", "REMOVE", "PERMIT", "RDEFINE", "RALTER", "RDELETE", "SETROPTS", "RVARY", "RACDCERT", "ADDSD", "ALTDSD", "DELDSD"],
    "service_account_patterns": ["STC*", "BATCH*", "CICS*", "DB2*", "IMS*", "ZOS*"],
    "denied_threshold_per_user": 3,
    "denied_threshold_per_dataset": 3,
    "stale_user_days": 90,
    "max_events_in_html": 100,
    "max_findings_in_html": 150,
}

PROFILE_MULTIPLIERS = {
    "production": 1.00,
    "security": 1.15,
    "compliance": 1.20,
    "audit": 1.10,
    "privilege": 1.20,
    "training": 0.75,
    "strict": 1.35,
}

COMMAND_RE = re.compile(r"\b(ADDUSER|ALTUSER|DELUSER|CONNECT|REMOVE|PERMIT|RDEFINE|RALTER|RDELETE|SETROPTS|RVARY|RACDCERT|ADDSD|ALTDSD|DELDSD|ADDGROUP|ALTGROUP|DELGROUP|PASSWORD)\b", re.I)
DATASET_RE = re.compile(r"\b(?:DATASET|DATA SET|DSN|DSNAME|PROFILE)\s*[=:\(]?\s*'?([A-Z0-9$#@][A-Z0-9$#@._\-]{2,})'?", re.I)
CLASS_RE = re.compile(r"\bCLASS\s*[=:\(]?\s*'?([A-Z0-9$#@_\-]{2,})'?", re.I)
USER_RE = re.compile(r"\b(?:USER|USERID|ID|LOGONID)\s*[=:\(]?\s*'?([A-Z0-9$#@_\-]{2,8})'?", re.I)
GROUP_RE = re.compile(r"\b(?:GROUP|OWNER)\s*[=:\(]?\s*'?([A-Z0-9$#@_\-]{2,8})'?", re.I)
ACCESS_RE = re.compile(r"\b(READ|UPDATE|CONTROL|ALTER|EXECUTE|NONE)\b", re.I)
AUTH_RE = re.compile(r"\b(SPECIAL|OPERATIONS|AUDITOR|ROAUDIT|CLAUTH|GRP-SPECIAL|GROUP-SPECIAL|UID\(0\)|BPX\.SUPERUSER|SURROGAT|OPERCMDS)\b", re.I)
ABEND_RE = re.compile(r"\bS913\b|\bABEND\s*=\s*S913\b", re.I)
RC_RE = re.compile(r"\b(?:RC|RETCODE|RETURN CODE|MAXCC)\s*[=:\-]?\s*([0-9]{1,4})\b", re.I)
TIME_RE = re.compile(r"(?P<ts>\b\d{2}:\d{2}:\d{2}(?:\.\d+)?\b|\b\d{4}[-/]\d{2}[-/]\d{2}[ T]\d{2}:\d{2}:\d{2}\b)")
ICH_RE = re.compile(r"\bICH\d{3,5}[A-Z]?\b")
IRR_RE = re.compile(r"\bIRR\d{3,5}[A-Z]?\b")


@dataclass
class AuditEvent:
    line_no: int
    timestamp: str = ""
    family: str = "OTHER"
    event_type: str = "message"
    severity: str = "INFO"
    user: str = ""
    group: str = ""
    dataset: str = ""
    resource_class: str = ""
    command: str = ""
    access: str = ""
    result: str = ""
    authority: str = ""
    message_id: str = ""
    raw: str = ""


@dataclass
class Finding:
    severity: str
    code: str
    category: str
    title: str
    detail: str
    recommendation: str
    line_no: int = 0
    user: str = ""
    dataset: str = ""
    group: str = ""
    command: str = ""
    evidence: str = ""
    score: int = 0


@dataclass
class EntitySummary:
    name: str
    count: int = 0
    denied_count: int = 0
    critical_count: int = 0
    warning_count: int = 0
    commands: List[str] = field(default_factory=list)
    datasets: List[str] = field(default_factory=list)
    groups: List[str] = field(default_factory=list)
    authorities: List[str] = field(default_factory=list)


@dataclass
class AnalysisResult:
    source_name: str
    profile: str
    status: str
    highest_severity: str
    risk_score: int
    lines: int
    events: List[AuditEvent]
    findings: List[Finding]
    users: List[EntitySummary]
    datasets: List[EntitySummary]
    groups: List[EntitySummary]
    commands: List[EntitySummary]
    categories: List[Dict[str, Any]]
    compare: List[Dict[str, Any]]
    policy: Dict[str, Any]


def load_policy(path: Optional[str]) -> Dict[str, Any]:
    policy = dict(DEFAULT_POLICY)
    if not path:
        return policy
    with open(path, "r", encoding="utf-8") as handle:
        custom = json.load(handle)
    for key, value in custom.items():
        if isinstance(value, list) and isinstance(policy.get(key), list):
            merged = list(policy[key])
            for item in value:
                if item not in merged:
                    merged.append(item)
            policy[key] = merged
        else:
            policy[key] = value
    return policy


def read_lines_from_file(path: str) -> List[str]:
    with open(path, "r", encoding="utf-8", errors="replace") as handle:
        return [line.rstrip("\n") for line in handle]


def redact_text(text: str) -> str:
    text = re.sub(r"\b([A-Z0-9$#@]+\.){1,8}[A-Z0-9$#@]+\b", "<DATASET>", text)
    text = re.sub(r"\b(USER|USERID|ID)=?([A-Z0-9$#@_\-]{2,8})\b", r"\1=<USER>", text, flags=re.I)
    return text


def extract_timestamp(line: str) -> str:
    match = TIME_RE.search(line)
    return match.group("ts") if match else ""


def detect_family(line: str) -> Tuple[str, str]:
    ich = ICH_RE.search(line.upper())
    if ich:
        return "ICH", ich.group(0)
    irr = IRR_RE.search(line.upper())
    if irr:
        return "IRR", irr.group(0)
    upper = line.upper()
    if "RACF" in upper:
        return "RACF", "RACF"
    if "SAF" in upper:
        return "SAF", "SAF"
    if "$HASP" in upper:
        return "JES", "$HASP"
    if "BPX" in upper:
        return "USS", "BPX"
    return "OTHER", ""


def first_match(regex: re.Pattern, text: str) -> str:
    match = regex.search(text)
    if not match:
        return ""
    if match.lastindex:
        return match.group(1) or ""
    return match.group(0) or ""


def extract_dataset(line: str) -> str:
    match = DATASET_RE.search(line)
    if match:
        return normalize_dataset(match.group(1))
    candidates = re.findall(r"\b(?:SYS1|SYS2|RACF|SECURITY|PAYROLL|HR|FIN|PROD|DB2|DSN|CICS|IMS|APPL)(?:\.[A-Z0-9$#@_\-]+)+\b", line.upper())
    return normalize_dataset(candidates[0]) if candidates else ""


def normalize_dataset(value: str) -> str:
    return value.strip("'\" .,;)").upper()


def extract_user(line: str) -> str:
    upper = line.upper()
    match = USER_RE.search(upper)
    if match:
        return match.group(1).upper()
    for pattern in [
        r"\bUSER\s+([A-Z0-9$#@_\-]{2,8})\b",
        r"\bID\s+([A-Z0-9$#@_\-]{2,8})\b",
        r"\bFOR\s+USER\s+([A-Z0-9$#@_\-]{2,8})\b",
        r"\b(?:ALTUSER|ADDUSER|DELUSER|PASSWORD|CONNECT|REMOVE)\s+([A-Z0-9$#@_\-]{2,8})\b",
    ]:
        found = re.search(pattern, upper)
        if found:
            return found.group(1).upper()
    return ""


def extract_command(line: str) -> str:
    match = COMMAND_RE.search(line)
    return match.group(1).upper() if match else ""


def extract_group(line: str) -> str:
    match = GROUP_RE.search(line)
    return match.group(1).upper() if match else ""


def extract_class(line: str) -> str:
    match = CLASS_RE.search(line)
    return match.group(1).upper() if match else ""


def extract_access(line: str) -> str:
    match = ACCESS_RE.search(line)
    return match.group(1).upper() if match else ""


def extract_authority(line: str, policy: Dict[str, Any]) -> str:
    authorities = set(a.upper() for a in AUTH_RE.findall(line.upper()))
    for authority in policy.get("privileged_authorities", []):
        if re.search(r"\b" + re.escape(authority.upper()) + r"\b", line.upper()):
            authorities.add(authority.upper())
    return ",".join(sorted(authorities))


def is_sensitive_dataset(dataset: str, policy: Dict[str, Any]) -> bool:
    if not dataset:
        return False
    for pattern in policy.get("sensitive_dataset_patterns", []):
        if fnmatch.fnmatch(dataset.upper(), pattern.upper()):
            return True
    return False


def is_privileged_group(group: str, policy: Dict[str, Any]) -> bool:
    if not group:
        return False
    return group.upper() in {g.upper() for g in policy.get("privileged_groups", [])}


def parse_event(line_no: int, line: str, policy: Dict[str, Any], redact: bool = False) -> AuditEvent:
    raw = redact_text(line) if redact else line
    upper = line.upper()
    family, message_id = detect_family(line)
    command = extract_command(line)
    user = extract_user(line)
    group = extract_group(line)
    dataset = extract_dataset(line)
    resource_class = extract_class(line)
    access = extract_access(line)
    authority = extract_authority(line, policy)
    result = ""
    event_type = "message"
    severity = "INFO"

    if "ACCESS DENIED" in upper or "INSUFFICIENT ACCESS" in upper or "NOT AUTHORIZED" in upper or "VIOLATION" in upper:
        result = "DENIED"
        event_type = "access_denied"
        severity = "ERROR"
    elif "ACCESS ALLOWED" in upper or "AUTHORIZED" in upper or "PERMITTED" in upper:
        result = "ALLOWED"
        event_type = "access_allowed"
    elif command:
        event_type = "command"
        result = "CHANGE"
        severity = "WARNING" if command not in {"LISTUSER", "LISTGRP", "SEARCH"} else "INFO"
    elif ABEND_RE.search(upper):
        event_type = "security_abend"
        result = "ABEND"
        severity = "ERROR"

    if command in {"ALTUSER", "ADDUSER", "DELUSER", "CONNECT", "REMOVE", "PERMIT", "RDEFINE", "RALTER", "RDELETE", "SETROPTS", "RVARY", "RACDCERT", "ADDSD", "ALTDSD", "DELDSD", "ADDGROUP", "ALTGROUP", "DELGROUP"}:
        severity = "WARNING"
    if command in {"SETROPTS", "RVARY", "RDELETE", "DELUSER", "DELGROUP"}:
        severity = "ERROR"
    if authority:
        severity = "ERROR"
    if result == "DENIED" and dataset and is_sensitive_dataset(dataset, policy):
        severity = "CRITICAL"
    if command in {"ALTUSER", "CONNECT", "PERMIT", "RDEFINE", "RALTER", "ADDSD", "ALTDSD", "SETROPTS"} and (authority or is_privileged_group(group, policy)):
        severity = "CRITICAL"
    if "UACC(ALTER" in upper or "UACC(CONTROL" in upper or "UACC(UPDATE" in upper:
        severity = "CRITICAL"
    if "NOAUDIT" in upper or "NOWARNING" in upper or "NOGENERIC" in upper or "NOPROTECTALL" in upper:
        severity = "CRITICAL"

    return AuditEvent(
        line_no=line_no,
        timestamp=extract_timestamp(line),
        family=family,
        event_type=event_type,
        severity=severity,
        user=user,
        group=group,
        dataset=dataset,
        resource_class=resource_class,
        command=command,
        access=access,
        result=result,
        authority=authority,
        message_id=message_id,
        raw=raw,
    )


def make_finding(severity: str, code: str, category: str, title: str, detail: str, recommendation: str, event: Optional[AuditEvent] = None, score: int = 0) -> Finding:
    return Finding(
        severity=severity,
        code=code,
        category=category,
        title=title,
        detail=detail,
        recommendation=recommendation,
        line_no=event.line_no if event else 0,
        user=event.user if event else "",
        dataset=event.dataset if event else "",
        group=event.group if event else "",
        command=event.command if event else "",
        evidence=event.raw if event else "",
        score=score,
    )


def generate_event_findings(events: Sequence[AuditEvent], policy: Dict[str, Any]) -> List[Finding]:
    findings: List[Finding] = []
    for event in events:
        upper = event.raw.upper()
        if event.result == "DENIED":
            severity = "CRITICAL" if event.dataset and is_sensitive_dataset(event.dataset, policy) else "ERROR"
            findings.append(make_finding(
                severity,
                "ACCESS_DENIED",
                "Access control",
                "Access denied event detected",
                f"User={event.user or 'unknown'} dataset={event.dataset or 'unknown'} class={event.resource_class or 'unknown'}",
                "Verify expected authorization, RACF profile, access list, group membership, and whether this is a real violation or a missing permit.",
                event,
                18 if severity == "CRITICAL" else 12,
            ))
        if event.dataset and is_sensitive_dataset(event.dataset, policy):
            if event.result == "ALLOWED":
                findings.append(make_finding(
                    "WARNING",
                    "SENSITIVE_ACCESS_ALLOWED",
                    "Sensitive data",
                    "Sensitive dataset access detected",
                    f"Dataset {event.dataset} was accessed by {event.user or 'unknown'}.",
                    "Confirm business justification and check whether the dataset should be monitored with additional audit controls.",
                    event,
                    7,
                ))
            elif event.event_type == "command" and event.command in {"PERMIT", "RDEFINE", "RALTER", "ADDSD", "ALTDSD", "DELDSD", "RDELETE"}:
                findings.append(make_finding(
                    "CRITICAL",
                    "SENSITIVE_PROFILE_CHANGE",
                    "Profile change",
                    "Sensitive dataset profile changed",
                    f"Command {event.command} affects sensitive dataset/profile {event.dataset}.",
                    "Validate the change ticket, approval, requested access level, and rollback procedure.",
                    event,
                    20,
                ))
        if event.command in policy.get("critical_commands", []):
            severity = "ERROR" if event.command in {"SETROPTS", "RVARY", "RDELETE", "DELUSER", "DELGROUP"} else "WARNING"
            findings.append(make_finding(
                severity,
                "SECURITY_COMMAND",
                "Change control",
                "Security administration command detected",
                f"Command {event.command} found for user={event.user or 'unknown'} group={event.group or 'unknown'} dataset={event.dataset or 'unknown'}.",
                "Map the command to an approved change record and verify whether it modified privileged access.",
                event,
                8 if severity == "WARNING" else 14,
            ))
        if event.authority:
            findings.append(make_finding(
                "CRITICAL",
                "PRIVILEGED_AUTHORITY",
                "Privilege management",
                "Privileged authority mentioned or granted",
                f"Authority {event.authority} detected on line {event.line_no}.",
                "Validate whether the privilege is temporary, approved, and limited to the minimum required authority.",
                event,
                22,
            ))
        if event.command == "CONNECT" and is_privileged_group(event.group, policy):
            findings.append(make_finding(
                "CRITICAL",
                "PRIVILEGED_GROUP_CONNECT",
                "Group membership",
                "Connection to privileged group detected",
                f"User {event.user or 'unknown'} connected to privileged group {event.group}.",
                "Verify group owner approval and review the user's effective privileges after the connection.",
                event,
                22,
            ))
        if event.command == "PERMIT" and re.search(r"\b(ALTER|CONTROL|UPDATE)\b", upper) and (event.dataset and is_sensitive_dataset(event.dataset, policy)):
            findings.append(make_finding(
                "CRITICAL",
                "HIGH_ACCESS_PERMIT",
                "Privilege management",
                "High access level granted on sensitive resource",
                "PERMIT grants UPDATE/CONTROL/ALTER on a sensitive profile.",
                "Reduce the access level if possible, define a time-bound access process, and enable auditing for the profile.",
                event,
                24,
            ))
        if "UACC(ALTER" in upper or "UACC(CONTROL" in upper or "UACC(UPDATE" in upper:
            findings.append(make_finding(
                "CRITICAL",
                "DANGEROUS_UACC",
                "Profile exposure",
                "Dangerous UACC value detected",
                "The profile appears to expose UPDATE/CONTROL/ALTER access through UACC.",
                "Set UACC to NONE or READ according to policy and grant explicit access only where required.",
                event,
                25,
            ))
        if "SETROPTS" in upper and ("NOAUDIT" in upper or "NOGENERIC" in upper or "NOPROTECTALL" in upper or "NOWARNING" in upper):
            findings.append(make_finding(
                "CRITICAL",
                "GLOBAL_RACF_OPTION_RISK",
                "System policy",
                "Risky SETROPTS option detected",
                "A global RACF option appears to reduce audit/protection behavior.",
                "Review security standards before activating global RACF option changes and verify SETROPTS LIST after change.",
                event,
                28,
            ))
        if "PASSWORD" in upper and ("INVALID" in upper or "FAILED" in upper or "REVOKED" in upper):
            findings.append(make_finding(
                "ERROR",
                "AUTHENTICATION_FAILURE",
                "Authentication",
                "Authentication failure or revoked user activity",
                f"Authentication-related event for user {event.user or 'unknown'}.",
                "Check whether this is a user error, expired credentials, revoked account attempt, or brute-force pattern.",
                event,
                12,
            ))
        if ABEND_RE.search(upper):
            findings.append(make_finding(
                "ERROR",
                "S913_SECURITY_ABEND",
                "Runtime security",
                "S913 security abend detected",
                "A job or step ended with an authorization failure.",
                "Correlate with nearby ICH408I/IRR messages and verify the dataset or resource access profile.",
                event,
                16,
            ))
        if "BPX.SUPERUSER" in upper or "UID(0" in upper:
            findings.append(make_finding(
                "CRITICAL",
                "USS_SUPERUSER_RISK",
                "USS privilege",
                "USS superuser level authority detected",
                "UID(0) or BPX.SUPERUSER authority was mentioned or changed.",
                "Confirm strict approval, monitor commands executed under this identity, and avoid permanent UID(0) assignment.",
                event,
                28,
            ))
    return findings


def generate_aggregate_findings(events: Sequence[AuditEvent], policy: Dict[str, Any]) -> List[Finding]:
    findings: List[Finding] = []
    denied_by_user = Counter(e.user for e in events if e.result == "DENIED" and e.user)
    denied_by_dataset = Counter(e.dataset for e in events if e.result == "DENIED" and e.dataset)
    command_by_user = Counter(e.user for e in events if e.command and e.user)
    threshold_user = int(policy.get("denied_threshold_per_user", 3))
    threshold_dataset = int(policy.get("denied_threshold_per_dataset", 3))

    for user, count in denied_by_user.items():
        if count >= threshold_user:
            findings.append(Finding(
                severity="ERROR",
                code="REPEATED_DENIALS_BY_USER",
                category="Behavior anomaly",
                title="Repeated access denials by user",
                detail=f"User {user} has {count} denied events in the analyzed window.",
                recommendation="Review recent activity, job submissions, credential use, and whether the denials indicate reconnaissance or missing access provisioning.",
                user=user,
                score=min(30, 8 + count * 4),
            ))
    for dataset, count in denied_by_dataset.items():
        if count >= threshold_dataset:
            sev = "CRITICAL" if is_sensitive_dataset(dataset, policy) else "ERROR"
            findings.append(Finding(
                severity=sev,
                code="REPEATED_DENIALS_BY_DATASET",
                category="Resource targeting",
                title="Repeated access denials against dataset",
                detail=f"Dataset {dataset} has {count} denied events in the analyzed window.",
                recommendation="Check whether multiple users or jobs are targeting the same protected profile and verify audit alerts for this resource.",
                dataset=dataset,
                score=min(32, 10 + count * 4),
            ))
    for user, count in command_by_user.items():
        if count >= 4:
            findings.append(Finding(
                severity="WARNING",
                code="HIGH_ADMIN_COMMAND_VOLUME",
                category="Change control",
                title="High security command volume by user",
                detail=f"User {user} issued {count} RACF/security administration commands in the analyzed window.",
                recommendation="Validate the change batch, ensure dual control where required, and compare against approved implementation plan.",
                user=user,
                score=min(18, 4 + count * 2),
            ))
    return findings


def summarize_entities(events: Sequence[AuditEvent], attr: str) -> List[EntitySummary]:
    bucket: Dict[str, EntitySummary] = {}
    for e in events:
        name = getattr(e, attr)
        if not name:
            continue
        item = bucket.setdefault(name, EntitySummary(name=name))
        item.count += 1
        if e.result == "DENIED":
            item.denied_count += 1
        if e.severity == "CRITICAL":
            item.critical_count += 1
        if e.severity == "WARNING":
            item.warning_count += 1
        if e.command and e.command not in item.commands:
            item.commands.append(e.command)
        if e.dataset and e.dataset not in item.datasets:
            item.datasets.append(e.dataset)
        if e.group and e.group not in item.groups:
            item.groups.append(e.group)
        if e.authority:
            for authority in e.authority.split(","):
                if authority and authority not in item.authorities:
                    item.authorities.append(authority)
    return sorted(bucket.values(), key=lambda x: (x.critical_count, x.denied_count, x.count), reverse=True)


def summarize_commands(events: Sequence[AuditEvent]) -> List[EntitySummary]:
    bucket: Dict[str, EntitySummary] = {}
    for e in events:
        if not e.command:
            continue
        item = bucket.setdefault(e.command, EntitySummary(name=e.command))
        item.count += 1
        if e.severity == "CRITICAL":
            item.critical_count += 1
        if e.severity == "WARNING":
            item.warning_count += 1
        if e.user and e.user not in item.groups:
            item.groups.append(e.user)
    return sorted(bucket.values(), key=lambda x: (x.critical_count, x.warning_count, x.count), reverse=True)


def summarize_categories(findings: Sequence[Finding]) -> List[Dict[str, Any]]:
    by_category: Dict[str, Dict[str, Any]] = defaultdict(lambda: {"category": "", "findings": 0, "critical": 0, "error": 0, "warning": 0, "score": 0})
    for f in findings:
        item = by_category[f.category]
        item["category"] = f.category
        item["findings"] += 1
        item["score"] += f.score
        if f.severity == "CRITICAL":
            item["critical"] += 1
        elif f.severity == "ERROR":
            item["error"] += 1
        elif f.severity == "WARNING":
            item["warning"] += 1
    return sorted(by_category.values(), key=lambda x: (x["critical"], x["error"], x["score"]), reverse=True)


def compute_risk(findings: Sequence[Finding], profile: str) -> int:
    base = sum(f.score for f in findings)
    multiplier = PROFILE_MULTIPLIERS.get(profile, 1.0)
    return min(100, int(round(base * multiplier)))


def highest_severity(findings: Sequence[Finding], events: Sequence[AuditEvent]) -> str:
    values = [f.severity for f in findings] + [e.severity for e in events]
    if not values:
        return "INFO"
    return max(values, key=lambda s: SEVERITY_ORDER.get(s, 0))


def status_from(highest: str, risk: int) -> str:
    if highest == "CRITICAL" or risk >= 90:
        return "BLOCKED"
    if highest == "ERROR" or risk >= 70:
        return "HIGH_RISK"
    if highest == "WARNING" or risk >= 35:
        return "REVIEW"
    return "OK"


def analyze_lines(lines: Sequence[str], source_name: str, profile: str, policy: Dict[str, Any], redact: bool = False, baseline_lines: Optional[Sequence[str]] = None) -> AnalysisResult:
    events = [parse_event(i + 1, line, policy, redact=redact) for i, line in enumerate(lines) if line.strip()]
    findings = generate_event_findings(events, policy)
    findings.extend(generate_aggregate_findings(events, policy))
    compare = compare_baseline(lines, baseline_lines, policy) if baseline_lines else []
    for row in compare:
        if row.get("severity") in {"ERROR", "CRITICAL"}:
            findings.append(Finding(
                severity=row["severity"],
                code=row.get("code", "BASELINE_DRIFT"),
                category="Baseline drift",
                title=row.get("title", "Baseline drift detected"),
                detail=row.get("detail", ""),
                recommendation="Review the current security state against the approved baseline and validate every drift item.",
                score=18 if row["severity"] == "ERROR" else 24,
            ))
    risk = compute_risk(findings, profile)
    high = highest_severity(findings, events)
    return AnalysisResult(
        source_name=source_name,
        profile=profile,
        status=status_from(high, risk),
        highest_severity=high,
        risk_score=risk,
        lines=len(lines),
        events=events,
        findings=sorted(findings, key=lambda f: (SEVERITY_ORDER.get(f.severity, 0), f.score, f.line_no), reverse=True),
        users=summarize_entities(events, "user"),
        datasets=summarize_entities(events, "dataset"),
        groups=summarize_entities(events, "group"),
        commands=summarize_commands(events),
        categories=summarize_categories(findings),
        compare=compare,
        policy=policy,
    )


def compare_baseline(current_lines: Sequence[str], baseline_lines: Optional[Sequence[str]], policy: Dict[str, Any]) -> List[Dict[str, Any]]:
    if not baseline_lines:
        return []
    current_events = [parse_event(i + 1, line, policy) for i, line in enumerate(current_lines) if line.strip()]
    baseline_events = [parse_event(i + 1, line, policy) for i, line in enumerate(baseline_lines) if line.strip()]
    current_priv = {(e.user, e.group, e.authority, e.command) for e in current_events if e.authority or e.command in {"CONNECT", "ALTUSER", "PERMIT", "RDEFINE", "RALTER", "SETROPTS"}}
    baseline_priv = {(e.user, e.group, e.authority, e.command) for e in baseline_events if e.authority or e.command in {"CONNECT", "ALTUSER", "PERMIT", "RDEFINE", "RALTER", "SETROPTS"}}
    new_priv = sorted(current_priv - baseline_priv)
    current_sensitive_profiles = {e.dataset for e in current_events if e.dataset and is_sensitive_dataset(e.dataset, policy)}
    baseline_sensitive_profiles = {e.dataset for e in baseline_events if e.dataset and is_sensitive_dataset(e.dataset, policy)}
    new_sensitive = sorted(current_sensitive_profiles - baseline_sensitive_profiles)
    rows: List[Dict[str, Any]] = []
    for user, group, authority, command in new_priv:
        rows.append({
            "severity": "CRITICAL" if authority or is_privileged_group(group, policy) else "ERROR",
            "code": "NEW_PRIVILEGE_OR_SECURITY_CHANGE",
            "title": "New privileged/security change compared with baseline",
            "detail": f"user={user or '-'} group={group or '-'} authority={authority or '-'} command={command or '-'}",
        })
    for dataset in new_sensitive:
        rows.append({
            "severity": "ERROR",
            "code": "NEW_SENSITIVE_RESOURCE_IN_SCOPE",
            "title": "New sensitive dataset/profile compared with baseline",
            "detail": f"dataset={dataset}",
        })
    return rows


def write_csv(path: str, rows: Iterable[Dict[str, Any]], fieldnames: Optional[List[str]] = None) -> None:
    rows = list(rows)
    if not fieldnames:
        keys = []
        for row in rows:
            for key in row.keys():
                if key not in keys:
                    keys.append(key)
        fieldnames = keys or ["empty"]
    with open(path, "w", newline="", encoding="utf-8") as handle:
        writer = csv.DictWriter(handle, fieldnames=fieldnames)
        writer.writeheader()
        for row in rows:
            writer.writerow({key: serialize_cell(row.get(key, "")) for key in fieldnames})


def serialize_cell(value: Any) -> str:
    if isinstance(value, (list, dict)):
        return json.dumps(value, ensure_ascii=False)
    return "" if value is None else str(value)


def entity_to_dict(e: EntitySummary) -> Dict[str, Any]:
    return asdict(e)


def result_to_dict(result: AnalysisResult) -> Dict[str, Any]:
    return {
        "source_name": result.source_name,
        "profile": result.profile,
        "status": result.status,
        "highest_severity": result.highest_severity,
        "risk_score": result.risk_score,
        "lines": result.lines,
        "event_count": len(result.events),
        "finding_count": len(result.findings),
        "user_count": len(result.users),
        "dataset_count": len(result.datasets),
        "group_count": len(result.groups),
        "command_count": len(result.commands),
        "events": [asdict(e) for e in result.events],
        "findings": [asdict(f) for f in result.findings],
        "users": [entity_to_dict(x) for x in result.users],
        "datasets": [entity_to_dict(x) for x in result.datasets],
        "groups": [entity_to_dict(x) for x in result.groups],
        "commands": [entity_to_dict(x) for x in result.commands],
        "categories": result.categories,
        "compare": result.compare,
    }


def write_json(path: str, result: AnalysisResult) -> None:
    with open(path, "w", encoding="utf-8") as handle:
        json.dump(result_to_dict(result), handle, indent=2, ensure_ascii=False)


def severity_class(sev: str) -> str:
    return {
        "CRITICAL": "sev-critical",
        "ERROR": "sev-error",
        "WARNING": "sev-warning",
        "INFO": "sev-info",
    }.get(sev, "sev-info")


def write_html(path: str, result: AnalysisResult) -> None:
    max_events = int(result.policy.get("max_events_in_html", 100))
    max_findings = int(result.policy.get("max_findings_in_html", 150))
    h = html.escape
    event_rows = "\n".join(
        f"<tr><td>{e.line_no}</td><td>{h(e.timestamp)}</td><td>{h(e.family)}</td><td>{h(e.event_type)}</td><td class='{severity_class(e.severity)}'>{h(e.severity)}</td><td>{h(e.user)}</td><td>{h(e.dataset)}</td><td>{h(e.command)}</td><td><code>{h(e.raw)}</code></td></tr>"
        for e in result.events[:max_events]
    )
    finding_rows = "\n".join(
        f"<tr><td class='{severity_class(f.severity)}'>{h(f.severity)}</td><td>{h(f.code)}</td><td>{h(f.category)}</td><td>{h(f.title)}</td><td>{h(f.detail)}</td><td>{h(f.recommendation)}</td><td><code>{h(f.evidence)}</code></td></tr>"
        for f in result.findings[:max_findings]
    )
    category_rows = "\n".join(
        f"<tr><td>{h(str(c.get('category','')))}</td><td>{c.get('findings',0)}</td><td>{c.get('critical',0)}</td><td>{c.get('error',0)}</td><td>{c.get('warning',0)}</td><td>{c.get('score',0)}</td></tr>"
        for c in result.categories
    )
    user_rows = "\n".join(
        f"<tr><td>{h(u.name)}</td><td>{u.count}</td><td>{u.denied_count}</td><td>{u.critical_count}</td><td>{h(', '.join(u.commands[:8]))}</td><td>{h(', '.join(u.groups[:8]))}</td></tr>"
        for u in result.users[:50]
    )
    dataset_rows = "\n".join(
        f"<tr><td>{h(d.name)}</td><td>{d.count}</td><td>{d.denied_count}</td><td>{d.critical_count}</td><td>{h(', '.join(d.commands[:8]))}</td></tr>"
        for d in result.datasets[:50]
    )
    compare_rows = "\n".join(
        f"<tr><td class='{severity_class(str(row.get('severity','INFO')))}'>{h(str(row.get('severity','')))}</td><td>{h(str(row.get('code','')))}</td><td>{h(str(row.get('title','')))}</td><td>{h(str(row.get('detail','')))}</td></tr>"
        for row in result.compare
    ) or "<tr><td colspan='4'>No baseline drift exported.</td></tr>"
    doc = f"""<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>RACF Audit Reader V2 Report</title>
<style>
body {{ font-family: Segoe UI, Arial, sans-serif; margin: 0; background: #f5f7fa; color: #111827; }}
.wrapper {{ max-width: 1180px; margin: 28px auto; background: white; border-radius: 12px; box-shadow: 0 8px 28px rgba(0,0,0,.08); overflow: hidden; }}
.header {{ background: linear-gradient(135deg,#0b2533,#103f55); color: white; padding: 24px 28px; }}
.header h1 {{ margin: 0 0 6px; }}
.content {{ padding: 26px 28px 36px; }}
.kpis {{ display:grid; grid-template-columns: repeat(5, 1fr); gap: 12px; margin: 18px 0 24px; }}
.kpi {{ border: 1px solid #dce3ea; border-radius: 10px; padding: 14px; background:#fbfdff; }}
.kpi .value {{ font-size: 1.35rem; font-weight: 800; color:#0077b6; }}
.kpi .label {{ color:#667085; font-size:.88rem; }}
table {{ width:100%; border-collapse: collapse; margin: 14px 0 28px; font-size:.9rem; }}
th {{ background:#2c3e50; color:white; padding: 8px 9px; text-align:left; }}
td {{ border:1px solid #dce3ea; padding: 8px 9px; vertical-align:top; }}
tr:nth-child(even) td {{ background:#f8fafc; }}
code {{ font-family: Consolas, monospace; font-size:.82rem; white-space: pre-wrap; }}
.sev-critical {{ color:#b71c1c; font-weight:800; }}
.sev-error {{ color:#d84315; font-weight:800; }}
.sev-warning {{ color:#ef6c00; font-weight:800; }}
.sev-info {{ color:#1565c0; font-weight:700; }}
.section-title {{ border-left:5px solid #00b4d8; padding-left: 12px; margin-top: 30px; }}
@media(max-width:900px) {{ .kpis {{ grid-template-columns: 1fr 1fr; }} .wrapper {{ margin: 0; border-radius:0; }} }}
</style>
</head>
<body>
<div class="wrapper">
  <div class="header">
    <h1>RACF Audit Reader V2</h1>
    <div>Source: {h(result.source_name)} | Profile: {h(result.profile)}</div>
  </div>
  <div class="content">
    <div class="kpis">
      <div class="kpi"><div class="value">{h(result.status)}</div><div class="label">Status</div></div>
      <div class="kpi"><div class="value">{result.risk_score}/100</div><div class="label">Risk score</div></div>
      <div class="kpi"><div class="value">{h(result.highest_severity)}</div><div class="label">Highest severity</div></div>
      <div class="kpi"><div class="value">{len(result.events)}</div><div class="label">Events</div></div>
      <div class="kpi"><div class="value">{len(result.findings)}</div><div class="label">Findings</div></div>
    </div>
    <h2 class="section-title">Risk categories</h2>
    <table><thead><tr><th>Category</th><th>Findings</th><th>Critical</th><th>Error</th><th>Warning</th><th>Score</th></tr></thead><tbody>{category_rows}</tbody></table>
    <h2 class="section-title">Findings</h2>
    <table><thead><tr><th>Severity</th><th>Code</th><th>Category</th><th>Title</th><th>Detail</th><th>Recommendation</th><th>Evidence</th></tr></thead><tbody>{finding_rows}</tbody></table>
    <h2 class="section-title">Top users</h2>
    <table><thead><tr><th>User</th><th>Events</th><th>Denied</th><th>Critical</th><th>Commands</th><th>Groups</th></tr></thead><tbody>{user_rows}</tbody></table>
    <h2 class="section-title">Top datasets/resources</h2>
    <table><thead><tr><th>Dataset/resource</th><th>Events</th><th>Denied</th><th>Critical</th><th>Commands</th></tr></thead><tbody>{dataset_rows}</tbody></table>
    <h2 class="section-title">Baseline drift</h2>
    <table><thead><tr><th>Severity</th><th>Code</th><th>Title</th><th>Detail</th></tr></thead><tbody>{compare_rows}</tbody></table>
    <h2 class="section-title">Event timeline</h2>
    <table><thead><tr><th>Line</th><th>Time</th><th>Family</th><th>Type</th><th>Severity</th><th>User</th><th>Dataset</th><th>Command</th><th>Raw</th></tr></thead><tbody>{event_rows}</tbody></table>
  </div>
</div>
</body>
</html>"""
    with open(path, "w", encoding="utf-8") as handle:
        handle.write(doc)


def export_all(result: AnalysisResult, args: argparse.Namespace) -> None:
    if args.json:
        write_json(args.json, result)
    if args.html:
        write_html(args.html, result)
    if args.csv_events:
        write_csv(args.csv_events, [asdict(e) for e in result.events])
    if args.csv_findings:
        write_csv(args.csv_findings, [asdict(f) for f in result.findings])
    if args.csv_users:
        write_csv(args.csv_users, [entity_to_dict(e) for e in result.users])
    if args.csv_datasets:
        write_csv(args.csv_datasets, [entity_to_dict(e) for e in result.datasets])
    if args.csv_groups:
        write_csv(args.csv_groups, [entity_to_dict(e) for e in result.groups])
    if args.csv_commands:
        write_csv(args.csv_commands, [entity_to_dict(e) for e in result.commands])
    if args.csv_categories:
        write_csv(args.csv_categories, result.categories)
    if args.csv_compare:
        write_csv(args.csv_compare, result.compare)


def print_console_summary(result: AnalysisResult) -> None:
    print("RACF Audit Reader V2")
    print(f"Status           : {result.status}")
    print(f"Highest severity : {result.highest_severity}")
    print(f"Risk score       : {result.risk_score}/100")
    print(f"Profile          : {result.profile}")
    print(f"Lines            : {result.lines}")
    print(f"Events           : {len(result.events)}")
    print(f"Findings         : {len(result.findings)}")
    print(f"Users            : {len(result.users)}")
    print(f"Datasets         : {len(result.datasets)}")
    print(f"Groups           : {len(result.groups)}")
    print(f"Commands         : {len(result.commands)}")
    if result.findings:
        print("\nTop findings:")
        for f in result.findings[:10]:
            print(f"- [{f.severity}] {f.code}: {f.title} :: {f.detail}")


def demo_lines(kind: str) -> List[str]:
    if kind == "baseline":
        return [
            "08:00:01 RACF COMMAND ALTUSER PAYADM NAME('PAYROLL ADMIN')",
            "08:00:02 RACF COMMAND CONNECT PAYADM GROUP(PAYROLL) AUTHORITY(USE)",
            "08:00:03 ICH70001I PAYADM LAST ACCESS AT 2026-05-20 07:51:11",
            "08:00:04 ICH408I USER(BATCH01) GROUP(BATCH) NAME(BATCH SERVICE) DATASET(PROD.PAYROLL.REPORT) CL(DATASET) INSUFFICIENT ACCESS AUTHORITY ACCESS INTENT(READ) ACCESS ALLOWED(READ)",
        ]
    if kind == "privilege":
        return [
            "09:10:01 RACF COMMAND ALTUSER PAYADM SPECIAL OPERATIONS AUDITOR",
            "09:10:04 RACF COMMAND CONNECT PAYADM GROUP(SECADMIN) AUTHORITY(USE)",
            "09:10:07 RACF COMMAND PERMIT SYS1.PARMLIB CLASS(DATASET) ID(PAYADM) ACCESS(ALTER)",
            "09:10:11 RACF COMMAND SETROPTS NOAUDIT NOWARNING",
            "09:10:14 RACF COMMAND ALTUSER STCDB2 OMVS(UID(0))",
        ]
    if kind == "compliance":
        return [
            "10:02:01 ICH408I USER(AUDT01) GROUP(AUDIT) DATASET(SYS1.RACF.PRIMARY) CL(DATASET) INSUFFICIENT ACCESS AUTHORITY ACCESS INTENT(READ) ACCESS ALLOWED(NONE)",
            "10:02:08 IRR010I RACF ACCESS DENIED USER=AUDT01 CLASS=DATASET PROFILE=SYS1.RACF.PRIMARY",
            "10:03:20 RACF COMMAND RALTER DATASET PROD.** UACC(UPDATE)",
            "10:04:02 RACF COMMAND RDEFINE FACILITY BPX.SUPERUSER UACC(READ)",
            "10:05:33 ICH408I USER(APPUSR1) GROUP(APPL) DATASET(PAYROLL.PROD.MASTER) CL(DATASET) INSUFFICIENT ACCESS AUTHORITY ACCESS INTENT(UPDATE) ACCESS ALLOWED(READ)",
        ]
    return [
        "08:53:01 $HASP373 PAYSEC01 STARTED - INIT 1 - CLASS A - SYS ZOS1",
        "08:53:08 ICH408I USER(PAYUSR1) GROUP(PAYROLL) NAME(PAY USER) DATASET(PAYROLL.PROD.MASTER) CL(DATASET) INSUFFICIENT ACCESS AUTHORITY ACCESS INTENT(UPDATE) ACCESS ALLOWED(READ)",
        "08:53:12 IRR010I RACF ACCESS DENIED USER=PAYUSR1 CLASS=DATASET PROFILE=PAYROLL.PROD.MASTER",
        "08:53:22 ICH408I USER(PAYUSR1) GROUP(PAYROLL) DATASET(PAYROLL.PROD.MASTER) CL(DATASET) INSUFFICIENT ACCESS AUTHORITY ACCESS INTENT(UPDATE) ACCESS ALLOWED(READ)",
        "08:53:35 ICH408I USER(PAYUSR1) GROUP(PAYROLL) DATASET(PAYROLL.PROD.MASTER) CL(DATASET) INSUFFICIENT ACCESS AUTHORITY ACCESS INTENT(UPDATE) ACCESS ALLOWED(READ)",
        "08:54:01 RACF COMMAND CONNECT DEV001 GROUP(SECADMIN) AUTHORITY(USE)",
        "08:54:04 RACF COMMAND ALTUSER DEV001 SPECIAL OPERATIONS",
        "08:54:09 RACF COMMAND PERMIT SYS1.PARMLIB CLASS(DATASET) ID(DEV001) ACCESS(ALTER)",
        "08:54:14 RACF COMMAND RALTER DATASET PROD.** UACC(ALTER)",
        "08:54:20 RACF COMMAND SETROPTS NOAUDIT",
        "08:55:10 IEF450I PAYSEC01 STEP010 - ABEND=S913 REASON=00000004",
        "08:55:11 $HASP395 PAYSEC01 ENDED - RC=0012",
        "08:56:00 ICH70001I USERID DB2STC HAS BEEN REVOKED DUE TO INVALID PASSWORD ATTEMPTS",
        "08:56:30 RACF COMMAND ALTUSER OMVSADM OMVS(UID(0))",
        "08:57:01 IRR012I RACF ACCESS ALLOWED USER=SYSOPR CLASS=FACILITY PROFILE=BPX.SUPERUSER",
    ]


def build_arg_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="RACF Audit Reader V2")
    parser.add_argument("input_file", nargs="?", help="RACF/ICH/IRR/SYSLOG text export to analyze")
    parser.add_argument("--demo", choices=["mixed", "privilege", "compliance", "baseline"], help="Run an embedded demo scenario")
    parser.add_argument("--baseline", help="Optional baseline text export for drift comparison")
    parser.add_argument("--policy", help="Custom policy JSON")
    parser.add_argument("--profile", default="production", choices=sorted(PROFILE_MULTIPLIERS.keys()), help="Analysis profile")
    parser.add_argument("--redact", action="store_true", help="Redact dataset/user details in evidence lines")
    parser.add_argument("--fail-on", choices=["INFO", "WARNING", "ERROR", "CRITICAL"], help="Return non-zero exit code if the highest severity reaches this level")
    parser.add_argument("--json", help="Write full JSON analysis")
    parser.add_argument("--html", help="Write HTML report")
    parser.add_argument("--csv-events", help="Write normalized events CSV")
    parser.add_argument("--csv-findings", help="Write findings CSV")
    parser.add_argument("--csv-users", help="Write users CSV")
    parser.add_argument("--csv-datasets", help="Write datasets CSV")
    parser.add_argument("--csv-groups", help="Write groups CSV")
    parser.add_argument("--csv-commands", help="Write commands CSV")
    parser.add_argument("--csv-categories", help="Write categories CSV")
    parser.add_argument("--csv-compare", help="Write baseline compare CSV")
    return parser


def main(argv: Optional[Sequence[str]] = None) -> int:
    parser = build_arg_parser()
    args = parser.parse_args(argv)
    policy = load_policy(args.policy)
    if args.demo:
        lines = demo_lines(args.demo)
        source_name = f"demo:{args.demo}"
    elif args.input_file:
        lines = read_lines_from_file(args.input_file)
        source_name = os.path.basename(args.input_file)
    else:
        lines = demo_lines("mixed")
        source_name = "demo:mixed"
    baseline_lines = read_lines_from_file(args.baseline) if args.baseline else None
    result = analyze_lines(lines, source_name, args.profile, policy, redact=args.redact, baseline_lines=baseline_lines)
    print_console_summary(result)
    export_all(result, args)
    if args.fail_on and SEVERITY_ORDER.get(result.highest_severity, 0) >= SEVERITY_ORDER.get(args.fail_on, 0):
        return EXIT_BY_SEVERITY.get(result.highest_severity, 1)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())