#!/usr/bin/env python3
import argparse
import csv
import json
import hashlib
import re
from collections import Counter, defaultdict
from pathlib import Path
from urllib.parse import urlparse

ROOT = Path('.')
DATA = ROOT / 'data'
OUT = Path('audit-results.json')


def load_csv(name):
    path = DATA / name
    with path.open(newline='', encoding='utf-8-sig') as f:
        reader = csv.reader(f)
        rows = list(reader)
    header = rows[0]
    malformed = []
    dict_rows = []
    for line_no, row in enumerate(rows[1:], start=2):
        if len(row) != len(header):
            malformed.append({'line': line_no, 'field_count': len(row), 'expected': len(header), 'row': row})
        padded = row[:len(header)] + [''] * max(0, len(header) - len(row))
        dict_rows.append(dict(zip(header, padded)))
    return path, header, dict_rows, malformed


def blank(value):
    return value is None or str(value).strip() == ''


def norm_text(value):
    if value is None:
        return ''
    value = str(value).strip().lower()
    value = re.sub(r"[^a-z0-9]+", " ", value)
    return ' '.join(value.split())


def summarize_duplicates(rows, fields):
    groups = defaultdict(list)
    for i, row in enumerate(rows, start=2):
        key = tuple(norm_text(row.get(field)) for field in fields)
        if all(key):
            groups[key].append({'line': i, 'evidence_id': row.get('evidence_id'), 'yard_id': row.get('yard_id')})
    return [
        {'fields': fields, 'key': list(key), 'records': records}
        for key, records in groups.items() if len(records) > 1
    ]


def main():
    global ROOT, DATA, OUT
    parser = argparse.ArgumentParser(
        description='Validate an extracted AsbestosAtlas v1.6-compatible release directory.'
    )
    parser.add_argument(
        '--deposit-root', required=True,
        help='Directory containing README.md, METHODOLOGY.md, metadata/, and data/.'
    )
    parser.add_argument(
        '--output', default='audit-results.json',
        help='Path for the JSON audit result (default: ./audit-results.json).'
    )
    args = parser.parse_args()
    ROOT = Path(args.deposit_root).expanduser().resolve()
    DATA = ROOT / 'data'
    OUT = Path(args.output).expanduser().resolve()
    OUT.parent.mkdir(parents=True, exist_ok=True)

    registry_path, registry_header, registry, registry_malformed = load_csv('shipyard_registry_v1.6.csv')
    evidence_path, evidence_header, evidence, evidence_malformed = load_csv('shipyard_evidence_v1.6.csv')
    employment_path, employment_header, employment, employment_malformed = load_csv('shipyard_employment_v1.6.csv')
    canonical = json.loads((DATA / 'shipyards.json').read_text(encoding='utf-8'))
    manifest = json.loads((ROOT / 'metadata/shipyard-v1.6-release-manifest.json').read_text(encoding='utf-8'))

    yard_ids = [r['yard_id'] for r in registry]
    yard_id_set = set(yard_ids)
    evidence_ids = [r['evidence_id'] for r in evidence]
    evidence_id_set = set(evidence_ids)

    nested_evidence = []
    nested_products = []
    normalized_products = []
    for yard in canonical:
        for item in yard.get('evidence', []):
            nested_evidence.append((yard.get('yard_id'), item))
        for item in yard.get('products_documented', []):
            nested_products.append((yard.get('yard_id'), item))
        for item in yard.get('product_normalized', []):
            normalized_products.append((yard.get('yard_id'), item))

    exact_yard_name_groups = defaultdict(list)
    for row in registry:
        exact_yard_name_groups[norm_text(row['name'])].append(row['yard_id'])

    alias_to_yards = defaultdict(set)
    for row in registry:
        names = [row['name']] + [x.strip() for x in (row.get('aliases') or '').split(';') if x.strip()]
        for name in names:
            alias_to_yards[norm_text(name)].add(row['yard_id'])

    duplicate_or_overlapping_names = [
        {'normalized_name': name, 'yard_ids': sorted(ids)}
        for name, ids in sorted(alias_to_yards.items()) if name and len(ids) > 1
    ]

    flagship_failures = []
    evidence_by_yard = defaultdict(set)
    for row in evidence:
        if row.get('yard_id'):
            evidence_by_yard[row['yard_id']].add(row['evidence_id'])
    for yard in canonical:
        fid = yard.get('flagship_evidence_id')
        yid = yard.get('yard_id')
        nested_ids = {item.get('evidence_id') for item in yard.get('evidence', [])}
        if blank(fid) or fid not in evidence_id_set or fid not in nested_ids or fid not in evidence_by_yard.get(yid, set()):
            flagship_failures.append({
                'yard_id': yid,
                'flagship_evidence_id': fid,
                'exists_anywhere_in_csv': fid in evidence_id_set if fid else False,
                'exists_for_yard_in_csv': fid in evidence_by_yard.get(yid, set()) if fid else False,
                'exists_in_nested_yard_evidence': fid in nested_ids if fid else False,
            })

    required = manifest['required_display_fields']
    missing_required = {
        field: [r['evidence_id'] for r in evidence if blank(r.get(field))]
        for field in required
    }

    source_domains = Counter()
    law_firm_domains = defaultdict(list)
    law_patterns = (
        'simmonsfirm.com', 'harrismartin.com', 'worthingtoncaron.com',
        'legalnewsline.com', 'insurancejournal.com',
    )
    for row in evidence:
        url = (row.get('citation_url') or '').strip()
        if url:
            domain = urlparse(url).netloc.lower().removeprefix('www.')
            source_domains[domain] += 1
            if any(pattern in domain for pattern in law_patterns):
                law_firm_domains[domain].append(row['evidence_id'])

    type_counts = Counter(r.get('finding_type', '') for r in evidence)
    citation_type_counts = Counter(r.get('citation_type', '') for r in evidence)
    verification_counts = Counter(r.get('verification_status', '') for r in evidence)

    # Historical core vocabularies from BUILD_STATE plus explicitly introduced v1.6 categories.
    expected_finding_types = {
        'cohort_smr', 'cohort_other', 'fiber_measurement', 'gov_survey',
        'litigation_documented', 'environmental_record', 'historical_record',
        'litigation_product_id', 'archival_lead', 'osha_inspection',
    }
    expected_citation_types = {
        'pmid', 'doi', 'gov_report', 'archive_id', 'court_record', 'epa_id',
        'url', 'archival_guide', 'osha_inspection_or_health_data',
    }

    suspicious_finding_types = {
        key: {
            'count': value,
            'evidence_ids': [r['evidence_id'] for r in evidence if r.get('finding_type') == key],
        }
        for key, value in sorted(type_counts.items()) if key not in expected_finding_types
    }
    suspicious_citation_types = {
        key: {
            'count': value,
            'evidence_ids': [r['evidence_id'] for r in evidence if r.get('citation_type') == key],
        }
        for key, value in sorted(citation_type_counts.items()) if key not in expected_citation_types
    }

    no_evidence_yards = sorted(yard_id_set - {r['yard_id'] for r in evidence if r.get('yard_id') in yard_id_set})
    null_tier_yards = [r['yard_id'] for r in registry if blank(r.get('evidence_tier'))]
    orphan_evidence = [r['evidence_id'] for r in evidence if blank(r.get('yard_id'))]
    invalid_yard_fk = [
        {'evidence_id': r['evidence_id'], 'yard_id': r.get('yard_id')}
        for r in evidence if not blank(r.get('yard_id')) and r.get('yard_id') not in yard_id_set
    ]
    registry_name_by_id = {r['yard_id']: r['name'] for r in registry}
    evidence_name_mismatches = [
        {
            'evidence_id': r['evidence_id'],
            'yard_id': r['yard_id'],
            'evidence_yard_name': r.get('yard_name'),
            'registry_name': registry_name_by_id.get(r['yard_id']),
        }
        for r in evidence
        if r.get('yard_id') in registry_name_by_id
        and norm_text(r.get('yard_name')) != norm_text(registry_name_by_id[r['yard_id']])
    ]

    employment_conflicts = []
    emp_groups = defaultdict(list)
    for i, row in enumerate(employment, start=2):
        key = (row.get('yard_id'), row.get('year_or_period'), row.get('estimate_type'), row.get('source_id'))
        emp_groups[key].append({'line': i, 'employment_estimate': row.get('employment_estimate'), 'note': row.get('note')})
    for key, records in emp_groups.items():
        vals = {r['employment_estimate'] for r in records}
        if len(records) > 1 and len(vals) > 1:
            employment_conflicts.append({
                'yard_id': key[0], 'year_or_period': key[1], 'estimate_type': key[2],
                'source_id': key[3], 'records': records,
                'all_notes_blank': all(blank(r['note']) for r in records),
            })

    nested_missing_yard_id = []
    nested_schema_variants = Counter()
    for parent_yard_id, item in nested_evidence:
        nested_schema_variants[tuple(sorted(item.keys()))] += 1
        if blank(item.get('yard_id')):
            nested_missing_yard_id.append({'parent_yard_id': parent_yard_id, 'evidence_id': item.get('evidence_id')})

    registry_superfund_counts = Counter(r.get('superfund_status', '') for r in registry)
    registry_tier_counts = Counter(r.get('evidence_tier', '') for r in registry)
    literal_none_citations = [r['evidence_id'] for r in evidence if norm_text(r.get('citation_value')) == 'none']

    product_ids = [item.get('product_id') for _, item in nested_products if item.get('product_id')]
    normalized_product_ids = [item.get('id') for _, item in normalized_products if item.get('id')]

    ids_with_probable_typo = [
        evidence_id for evidence_id in evidence_ids
        if 'shipyar-' in evidence_id or re.search(r'naval-s-\d+$', evidence_id)
    ]

    output = {
        'provenance': {
            'zenodo_record': '10.5281/zenodo.22313866',
            'dataset_version': '1.6',
            'canonical_json_sha256': hashlib.sha256((DATA / 'shipyards.json').read_bytes()).hexdigest(),
        },
        'counts': {
            'registry_rows': len(registry),
            'evidence_rows': len(evidence),
            'employment_rows': len(employment),
            'canonical_yards': len(canonical),
            'nested_evidence_rows': len(nested_evidence),
            'unique_yard_ids': len(yard_id_set),
            'unique_evidence_ids': len(evidence_id_set),
        },
        'csv_structure': {
            'registry_header': registry_header,
            'evidence_header': evidence_header,
            'employment_header': employment_header,
            'malformed_registry_rows': registry_malformed,
            'malformed_evidence_rows': evidence_malformed,
            'malformed_employment_rows': employment_malformed,
        },
        'key_integrity': {
            'duplicate_yard_ids': [k for k, v in Counter(yard_ids).items() if v > 1],
            'duplicate_evidence_ids': [k for k, v in Counter(evidence_ids).items() if v > 1],
            'orphan_evidence_missing_yard_id': orphan_evidence,
            'invalid_evidence_yard_fk': invalid_yard_fk,
            'evidence_yard_name_mismatches': evidence_name_mismatches,
            'flagship_failures': flagship_failures,
            'no_evidence_yards': no_evidence_yards,
            'null_evidence_tier_yards': null_tier_yards,
            'evidence_tier_counts': dict(sorted(registry_tier_counts.items())),
            'superfund_status_counts': dict(sorted(registry_superfund_counts.items())),
        },
        'required_field_completeness': missing_required,
        'controlled_vocabulary': {
            'finding_type_counts': dict(sorted(type_counts.items())),
            'citation_type_counts': dict(sorted(citation_type_counts.items())),
            'verification_status_counts': dict(sorted(verification_counts.items())),
            'suspicious_finding_types': suspicious_finding_types,
            'suspicious_citation_types': suspicious_citation_types,
        },
        'duplicates': {
            'same_citation_value': summarize_duplicates(evidence, ['citation_value']),
            'same_yard_citation_value': summarize_duplicates(evidence, ['yard_id', 'citation_value']),
            'same_yard_finding_citation': summarize_duplicates(evidence, ['yard_id', 'finding_value', 'citation_value']),
        },
        'entity_quality': {
            'duplicate_exact_names': [
                {'normalized_name': name, 'yard_ids': ids}
                for name, ids in exact_yard_name_groups.items() if len(ids) > 1
            ],
            'cross_record_name_or_alias_overlaps': duplicate_or_overlapping_names,
            'todd_record': next((r for r in registry if r['yard_id'] == 'todd-shipyards'), None),
        },
        'employment_quality': {
            'conflicting_same_period_source_rows': employment_conflicts,
        },
        'source_hygiene': {
            'law_firm_domains': dict(sorted(law_firm_domains.items())),
            'citation_url_missing': missing_required.get('citation_url', []),
            'literal_none_citation_values': literal_none_citations,
            'top_domains': source_domains.most_common(),
        },
        'identifier_quality': {
            'probable_typo_evidence_ids': ids_with_probable_typo,
        },
        'canonical_json_quality': {
            'nested_evidence_missing_yard_id': nested_missing_yard_id,
            'detailed_product_rows': len(nested_products),
            'unique_detailed_product_ids': len(set(product_ids)),
            'missing_detailed_product_ids': [
                {'yard_id': yard_id, 'product_name': item.get('product_name', '')}
                for yard_id, item in nested_products if blank(item.get('product_id'))
            ],
            'duplicate_detailed_product_ids': [key for key, count in Counter(product_ids).items() if count > 1],
            'yards_with_detailed_products': len({yard_id for yard_id, _ in nested_products}),
            'normalized_product_assignments': len(normalized_products),
            'unique_normalized_product_categories': len(set(normalized_product_ids)),
            'nested_evidence_key_shapes': [
                {'keys': list(keys), 'count': count}
                for keys, count in nested_schema_variants.most_common()
            ],
        },
    }

    OUT.write_text(json.dumps(output, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
    print(json.dumps({
        'output': str(OUT),
        'counts': output['counts'],
        'flagship_failures': len(flagship_failures),
        'orphan_evidence': len(orphan_evidence),
        'missing_finding_value': len(missing_required['finding_value']),
        'missing_population': len(missing_required['population']),
        'missing_study_period': len(missing_required['study_period']),
        'missing_citation_url': len(missing_required['citation_url']),
        'missing_limitations_note': len(missing_required['limitations_note']),
        'no_evidence_yards': no_evidence_yards,
        'null_tier_yards': null_tier_yards,
        'employment_conflicts': len(employment_conflicts),
        'malformed_evidence_rows': len(evidence_malformed),
    }, indent=2))


if __name__ == '__main__':
    main()