"""
Analysis for research report EFF-RR-2026-008: malicious packages in open-source registries.

    python3 -I analyse.py <data-dir> <out-dir>              classify records and write results
    python3 -I analyse.py <data-dir> <out-dir> --sample     also query the npm registry for a
                                                            fixed random sample (cached in data-dir)

<data-dir> holds the OSV dumps <Ecosystem>.zip (npm, PyPI, RubyGems, NuGet, crates.io, Go, Maven,
Packagist) and top-pypi.json (most-downloaded PyPI projects). See README.md for how to fetch them.
Only the standard library is used.
"""
import collections
import csv
import json
import random
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import zipfile
from datetime import datetime, timezone
from pathlib import Path

ECOSYSTEMS = ['npm', 'PyPI', 'RubyGems', 'NuGet', 'crates.io', 'Go', 'Maven', 'Packagist']
WINDOW = ('2023-01-01', '2026-11-12')  # OSV records published in this period; the dumps were taken 12 November 2026

# ---------- Classification rules -------------------------------------------------------------
# Applied in this order to the record's summary and details, lower-cased; the first match wins.
CATEGORIES = [
    ('false_positive', 'Marked false positive', r'false positive'),
    ('spam', 'Spam and token farming', r'tea\.xyz|token reward|auto-?publish|flood(ed|ing)? (npm|pypi)|spam'),
    ('compromised', 'Compromised legitimate package', r'was compromised|sha[i1]-hulud|unauthori[sz]ed (new )?versions|maintainer account'),
    ('recon', 'Dependency confusion and reconnaissance', r'dependency confusion|basic information|pentest|security research|proof of concept|\bresearch\b|oastify|canarytoken|interact\.sh|burpcollaborator|experiment'),
    ('crypto', 'Cryptocurrency theft', r'wallet|crypto|private key|seed phrase|mnemonic|solana|ethereum|bitcoin|blockchain|clipper'),
    ('stealer', 'Credential and data theft', r'steal|exfil|credential|secret|password|cookie|infostealer|spyware|keylog|\.npmrc|ssh key|discord|telegram|environment variable'),
    ('dropper', 'Malware delivery and remote access', r'download(s|ing)? (and|&) (run|execut)|execut\w* (a )?(malicious )?(executable|binary|payload)|dropper|reverse ?shell|revshell|backdoor|remote access|\brat\b|second.stage|steganograph|clickfix|malware'),
    ('domain', 'Contacts a known malicious domain', r'domain associated with malicious'),
]
UNSPECIFIED = ('unspecified', 'Malicious, no detail given')
LABEL = {k: v for k, v, _ in CATEGORIES} | {UNSPECIFIED[0]: UNSPECIFIED[1]}
HARMFUL = {'compromised', 'crypto', 'stealer', 'dropper', 'domain', 'unspecified'}


def text_of(r):
    d = r.get('details') or ''
    d = re.sub(r'## Source: [^\n]*\n|_-= Per source details\. Do not edit below this line\.=-_|^---$', ' ', d, flags=re.M)
    return re.sub(r'\s+', ' ', f"{r.get('summary') or ''} {d}").strip().lower()


def classify(r):
    t = text_of(r)
    for key, _, rx in CATEGORIES:
        if re.search(rx, t):
            return key
    return UNSPECIFIED[0]


def sources(r):
    origins = (r.get('database_specific') or {}).get('malicious-packages-origins') or []
    found = sorted({o.get('source') for o in origins if o.get('source')})
    if not found:  # older records: fall back to the credited finder, normalised to the same style
        found = sorted({re.sub(r'[^a-z0-9]+', '-', c['name'].lower()).strip('-') for c in r.get('credits') or [] if c.get('name')})
    return found or ['unknown']


# ---------- PyPI name imitation ---------------------------------------------------------------
norm = lambda n: re.sub(r'[-_.]+', '-', n.lower())


def deletes(s):
    return {s[:i] + s[i + 1:] for i in range(len(s))}


def imitation(name, top_exact, top_index, top_rank):
    """Classifies a PyPI name against the most-downloaded projects: exact, close misspelling, or combined."""
    n = norm(name)
    if n in top_exact:
        return 'exact', n
    # Close misspelling of a top-5,000 project of 5+ characters: the two names become identical after deleting at
    # most one character from each, or after swapping two neighbouring letters in the malicious name (lengths within one).
    cands = {n} | deletes(n) | {n[:i] + n[i + 1] + n[i] + n[i + 2:] for i in range(len(n) - 1)}
    for c in cands:
        for hit in top_index.get(c, ()):
            if hit != n and len(hit) >= 5 and abs(len(hit) - len(n)) <= 1:
                return 'misspelling', hit
    # Combosquat: a top-1000 project name (6+ characters) used as a whole word with extra words.
    parts = n.split('-')
    for p in parts:
        if p in top_rank and top_rank[p] <= 1000 and len(p) >= 6 and p != n:
            return 'combined', p
    return 'none', ''


# ---------- Main --------------------------------------------------------------------------------
def main():
    data, out = Path(sys.argv[1]), Path(sys.argv[2])
    out.mkdir(parents=True, exist_ok=True)
    rows = []
    for eco in ECOSYSTEMS:
        z = zipfile.ZipFile(data / f'{eco}.zip')
        for n in z.namelist():
            if not n.startswith('MAL-'):
                continue
            r = json.loads(z.read(n))
            pub = r['published'][:10]
            if not (WINDOW[0] <= pub <= WINDOW[1]):
                continue
            a = (r.get('affected') or [{}])[0]
            rows.append({
                'id': r['id'], 'ecosystem': eco, 'package': a.get('package', {}).get('name', ''),
                'published': pub, 'year': pub[:4], 'withdrawn': bool(r.get('withdrawn')),
                'category': classify(r), 'sources': '|'.join(sources(r)),
                'versions': len(a.get('versions') or []),
            })

    kept = [r for r in rows if not r['withdrawn'] and r['category'] != 'false_positive']
    C = collections.Counter
    pct = lambda n, d: round(100 * n / d, 1) if d else 0
    years = ['2023', '2024', '2025', '2026']
    res = {
        'window': WINDOW, 'recordsInWindow': len(rows),
        'excluded': {'withdrawn': sum(r['withdrawn'] for r in rows), 'falsePositive': sum(r['category'] == 'false_positive' and not r['withdrawn'] for r in rows)},
        'records': len(kept),
        'byEcosystem': dict(C(r['ecosystem'] for r in kept).most_common()),
        'byEcosystemYear': {e: {y: sum(1 for r in kept if r['ecosystem'] == e and r['year'] == y) for y in years} for e in ECOSYSTEMS},
        'byYear': {y: sum(1 for r in kept if r['year'] == y) for y in years},
        'categories': {k: {'label': LABEL[k], 'n': n, 'pct': pct(n, len(kept))} for k, n in C(r['category'] for r in kept).most_common()},
        'categoriesByEcosystem': {e: dict(C(r['category'] for r in kept if r['ecosystem'] == e).most_common()) for e in ['npm', 'PyPI', 'RubyGems', 'NuGet']},
        'harmful': {
            'n': sum(r['category'] in HARMFUL for r in kept),
            'byEcosystem': dict(C(r['ecosystem'] for r in kept if r['category'] in HARMFUL).most_common()),
            'byYear': {y: sum(1 for r in kept if r['category'] in HARMFUL and r['year'] == y) for y in years},
            'byEcosystemYear': {e: {y: sum(1 for r in kept if r['category'] in HARMFUL and r['ecosystem'] == e and r['year'] == y) for y in years} for e in ['npm', 'PyPI', 'RubyGems', 'NuGet']},
        },
        'spamByMonth2025': dict(sorted(C(r['published'][:7] for r in kept if r['category'] == 'spam').items())),
        'sources': dict(C(s for r in kept for s in r['sources'].split('|')).most_common(12)),
        'multiSourceShare': pct(sum(len(r['sources'].split('|')) > 1 for r in kept), len(kept)),
        'compromisedPackages': sorted({f"{r['ecosystem']}:{r['package']}" for r in kept if r['category'] == 'compromised'}),
        'compromisedByMonth': dict(sorted(C(r['published'][:7] for r in kept if r['category'] == 'compromised').items())),
        'harmfulSources': dict(C(s for r in kept if r['category'] in HARMFUL for s in r['sources'].split('|')).most_common(8)),
        'describedHarmful': {k: n for k, n in C(r['category'] for r in kept if r['category'] in HARMFUL - {'unspecified'}).most_common()},
    }

    # PyPI: which malicious names imitate popular projects?
    top = json.load(open(data / 'top-pypi.json'))
    ranked = [norm(x['project']) for x in top['rows']]
    top_rank = {n: i + 1 for i, n in enumerate(ranked)}
    top_exact = set(ranked)
    top_index = collections.defaultdict(set)
    for n in ranked[:5000]:
        top_index[n].add(n)
        for d in deletes(n):
            top_index[d].add(n)
    pypi = [r for r in kept if r['ecosystem'] == 'PyPI' and r['category'] in HARMFUL | {'recon'}]
    imit = collections.Counter()
    examples = collections.defaultdict(list)
    targets = collections.Counter()
    for r in pypi:
        kind, hit = imitation(r['package'], top_exact, top_index, top_rank)
        r['imitation'], r['imitates'] = kind, hit
        imit[kind] += 1
        if kind != 'none':
            targets[hit] += 1
            if len(examples[kind]) < 6:
                examples[kind].append(f"{r['package']} -> {hit}")
    res['pypiImitation'] = {
        'topListUpdated': top.get('last_update'), 'topListSize': len(ranked), 'n': len(pypi),
        'kinds': {k: {'n': n, 'pct': pct(n, len(pypi))} for k, n in imit.most_common()},
        'mostImitated': targets.most_common(12), 'examples': examples,
    }

    # npm registry sample: how long did malicious packages stay available?
    if '--sample' in sys.argv or (data / 'npm-registry-sample.json').exists():
        res['npmExposure'] = npm_exposure(data, kept, '--sample' in sys.argv)
        with open(out / 'npm-registry-sample.csv', 'w', newline='') as f:
            w = csv.writer(f)
            w.writerow(['group', 'id', 'package', 'osv_published', 'category', 'first_version_time', 'security_placeholder_time', 'unpublished'])
            for g in ('harmful', 'spam'):
                for row in res['npmExposure'].pop(g + 'Rows'):
                    w.writerow([g] + row)

    (out / 'results.json').write_text(json.dumps(res, indent=2, default=list) + '\n')
    # Compact outputs. The full record set is public in OSV, so only counts and the coded samples are published.
    with open(out / 'counts.csv', 'w', newline='') as f:
        w = csv.writer(f)
        w.writerow(['ecosystem', 'year', 'category', 'records'])
        for (e, y, c), n in sorted(C((r['ecosystem'], r['year'], r['category']) for r in kept).items()):
            w.writerow([e, y, c, n])
    with open(out / 'pypi-name-imitation.csv', 'w', newline='') as f:
        w = csv.writer(f)
        w.writerow(['id', 'package', 'published', 'category', 'imitation', 'imitates'])
        for r in pypi:
            if r['imitation'] != 'none':
                w.writerow([r['id'], r['package'], r['published'], r['category'], r['imitation'], r['imitates']])
    print(f"{len(kept)} records kept of {len(rows)} in window; harmful {res['harmful']['n']}")


SAMPLE_PER_GROUP = 300


def npm_exposure(data, kept, fetch):
    """Fixed random sample of npm records published in 2025-2026, queried against the npm registry."""
    cache_file = data / 'npm-registry-sample.json'
    cache = json.loads(cache_file.read_text()) if cache_file.exists() else {}
    pool = [r for r in kept if r['ecosystem'] == 'npm' and r['published'] >= '2025-01-01' and r['category'] != 'compromised']
    rng = random.Random(2026)
    groups = {
        'harmful': rng.sample(sorted((r for r in pool if r['category'] in HARMFUL), key=lambda r: r['id']), SAMPLE_PER_GROUP),
        'spam': rng.sample(sorted((r for r in pool if r['category'] == 'spam'), key=lambda r: r['id']), SAMPLE_PER_GROUP),
    }
    if fetch:
        for g in groups.values():
            for r in g:
                if r['package'] in cache:
                    continue
                url = 'https://registry.npmjs.org/' + urllib.parse.quote(r['package'], safe='@')
                req = urllib.request.Request(url, headers={'User-Agent': 'research-client', 'Accept': 'application/json'})
                try:
                    doc = json.loads(urllib.request.urlopen(req, timeout=30).read())
                    cache[r['package']] = {'status': 200, 'time': doc.get('time', {}), 'versions': list((doc.get('versions') or {}).keys())}
                except urllib.error.HTTPError as e:
                    cache[r['package']] = {'status': e.code}
                time.sleep(0.25)
        cache_file.write_text(json.dumps(cache))

    def ts(s):
        return datetime.fromisoformat(s.replace('Z', '+00:00'))

    out = {'sampleDate': '2026-11-12', 'perGroup': SAMPLE_PER_GROUP}
    for gname, g in groups.items():
        state = collections.Counter()
        hours = []
        single = removed_before_report = timed_with_report = 0
        rows_out = []
        for r in g:
            c = cache.get(r['package'])
            if not c or c['status'] != 200:
                state['not found'] += 1
                continue
            t = c['time']
            sec = sorted(v for k, v in t.items() if k.startswith('0.0.1-security'))
            real = sorted(v for k, v in t.items() if k not in ('created', 'modified', 'unpublished') and not k.startswith('0.0.1-security'))
            rows_out.append([r['id'], r['package'], r['published'], r['category'], (real[0] if real else ''), (sec[0] if sec else ''), 'unpublished' if 'unpublished' in t else ''])
            if sec:
                state['replaced by npm security placeholder'] += 1
                if real:
                    hours.append((ts(sec[0]) - ts(real[0])).total_seconds() / 3600)
                    single += len(real) == 1
                    # OSV gives a publication date only; compare by day.
                    timed_with_report += 1
                    removed_before_report += sec[0][:10] < r['published']
            elif 'unpublished' in t:
                state['unpublished'] += 1
            else:
                state['still available'] += 1
        hours.sort()
        q = lambda p: round(hours[min(len(hours) - 1, int(p * len(hours)))], 1) if hours else None
        out[gname + 'Rows'] = rows_out
        out[gname] = {
            'states': dict(state.most_common()),
            'timed': len(hours),
            'medianHours': q(0.5), 'p25Hours': q(0.25), 'p75Hours': q(0.75), 'p90Hours': q(0.9),
            'within24h': round(100 * sum(h <= 24 for h in hours) / len(hours), 1) if hours else None,
            'over7d': round(100 * sum(h > 168 for h in hours) / len(hours), 1) if hours else None,
            'over30d': round(100 * sum(h > 720 for h in hours) / len(hours), 1) if hours else None,
            'singleVersionPct': round(100 * single / len(hours), 1) if hours else None,
            'removedBeforeOsvRecordPct': round(100 * removed_before_report / timed_with_report, 1) if timed_with_report else None,
        }
    return out


if __name__ == '__main__':
    main()