/**
 * Analysis for research report EFF-RR-2026-001.
 *
 *   node research/kev-2026/analyse.mjs <kev.json> <nvd.json> <out-dir>
 *
 * Inputs (see README.md for how to retrieve them):
 *   kev.json  CISA Known Exploited Vulnerabilities catalogue (JSON feed)
 *   nvd.json  NVD CVE API 2.0 response for ?hasKev (gives each CVE's publication date)
 *
 * Outputs: results.json (every number in the report) and kev-2023-2026.csv (one coded row per entry).
 */
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
import { join } from 'node:path';

const [kevPath, nvdPath, out] = process.argv.slice(2);
if (!out) { console.error('Usage: node analyse.mjs <kev.json> <nvd.json> <out-dir>'); process.exit(1); }

const kev = JSON.parse(readFileSync(kevPath, 'utf8'));
const nvd = JSON.parse(readFileSync(nvdPath, 'utf8'));
const published = new Map(nvd.vulnerabilities.map((v) => [v.cve.id, v.cve.published.slice(0, 10)]));

/* ---------- Coding rules ---------- */

// Product category. Rules are applied in this order to "<vendor> <product>"; the first match wins.
const CATEGORIES = [
  ['edge', 'Network edge and remote access', /firewall|\basa\b|firepower|\bftd\b|\bfmc\b|pan-os|expedition|fortios|fortiproxy|fortiweb|fortimail|fortimanager|fortisandbox|fortinet multiple|netscaler|big-ip|connect secure|policy secure|ivanti sentry|cloud services? appliance|virtual traffic manager|sma ?1000|sma100|sonicos|sslvpn|router|routeros|vigor|sd-wan|velocloud|versa|ios xe|cisco ios|nx-os|junos|screenos|arrayos|loadmaster|email security gateway|secure email gateway|barracuda|check point (multiple|quantum|security gateway)|security gateway|xg firewall|cyberoam|sophos web appliance|firebox|zyxel (dsl|emg|p660|gs1900)|aviatrix|identity services engine|extensible operating system|airlink|ruckus|access point|dsl-2750b|dd-wrt|archer|tl-w|\bunifi\b|dir-\d|brocade|screenconnect|remote support|privileged remote access|simplehelp|looking glass/i],
  ['iot', 'IoT, OT and embedded devices', /camera|\bnvr|nvrmini|viostor|dahua|hikvision|geovision|digiever|\bnas\b|nas devices|terramaster|megarac|\bplc\b|scadabr|rockwell|knx|emerge|meteobridge|solarview|lantronix|fxc|sureline|dante discovery|dcs-2530l|dnr-322l|realtek sdk|sip phones/i],
  ['os', 'Operating systems, mobile and hypervisors', /microsoft (windows|win32k|dwm|cng|streaming service|internet key exchange)|linux kernel|^apple |^android |samsung mobile|pixel|qualcomm|mali|intel ethernet|sudo|polkit|gnu c library|libuser|igel os|esxi|directx|gnu bash|inetutils|vmware esx|vmware tools/i],
  ['client', 'Browsers and desktop applications', /chromium|google (skia|dawn)|firefox|mozilla|internet explorer|microsoft (office|outlook|word|publisher|project|wordpad|visual basic|smartscreen|skype)|acrobat|flash player|winrar|7-zip|daemon tools|wps office|whatsapp|notepad\+\+|freetype|authy|trueconf client|teamt5|defender|imagemagick/i],
];
const OTHER = ['server', 'Enterprise server, web and developer software'];

function category(vendor, product) {
  const s = `${vendor.trim()} ${product.trim()}`;
  for (const [id, , re] of CATEGORIES) if (re.test(s)) return id;
  return OTHER[0];
}
const LABEL = Object.fromEntries([...CATEGORIES.map(([id, label]) => [id, label]), OTHER]);

// Subgroups of server software discussed in the report.
const AI_TOOLING = /langflow|litellm|mlflow|ray-project|gpustack|marimo|n8n/i;
const FILE_TRANSFER = /moveit|cleo|goanywhere|crushftp|wing ftp|ws_ftp|serv-u|aspera|filezen|proself|centrestack|triofox/i;

// Weakness family, from the first CWE in an entry that appears in this table.
const FAMILIES = {
  memory: ['Memory safety', [119, 120, 121, 122, 124, 125, 126, 129, 131, 190, 191, 401, 415, 416, 476, 680, 704, 763, 787, 822, 823, 824, 843]],
  auth: ['Authentication and access control', [250, 266, 267, 269, 276, 284, 285, 287, 288, 290, 294, 302, 303, 304, 305, 306, 307, 345, 346, 347, 425, 521, 522, 639, 640, 648, 732, 798, 862, 863, 1188, 1390]],
  injection: ['Injection and code execution', [74, 77, 78, 79, 88, 89, 91, 93, 94, 95, 96, 611, 917, 1236, 1321, 1336]],
  file: ['Path traversal and file handling', [22, 23, 24, 29, 35, 36, 59, 61, 73, 434, 552]],
  deser: ['Unsafe deserialisation', [502]],
  forgery: ['Request forgery (SSRF, CSRF)', [352, 918]],
};
const familyOf = new Map(Object.entries(FAMILIES).flatMap(([id, [, cwes]]) => cwes.map((c) => [`CWE-${c}`, id])));
function family(cwes) {
  for (const c of cwes || []) if (familyOf.has(c)) return familyOf.get(c);
  return (cwes || []).some((c) => /^CWE-\d+$/.test(c)) ? 'other' : 'none';
}
const FAMILY_LABEL = { ...Object.fromEntries(Object.entries(FAMILIES).map(([id, [l]]) => [id, l])), other: 'Other specified weakness', none: 'No weakness recorded' };

/* ---------- Helpers ---------- */

const days = (a, b) => Math.round((Date.parse(b) - Date.parse(a)) / 86400000);
const pct = (n, d) => (d ? Math.round((n / d) * 1000) / 10 : 0);
const count = (rows, key) => rows.reduce((m, r) => ((m[key(r)] = (m[key(r)] || 0) + 1), m), {});
const median = (xs) => { const s = [...xs].sort((a, b) => a - b); const m = s.length >> 1; return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2; };
const by = (obj) => Object.entries(obj).sort((a, b) => b[1] - a[1]);

/* ---------- Coding ---------- */

const rows = kev.vulnerabilities.map((v) => {
  const pub = published.get(v.cveID) || null;
  const s = `${v.vendorProject} ${v.product}`;
  return {
    cve: v.cveID,
    vendor: v.vendorProject.trim(),
    product: v.product.trim(),
    dateAdded: v.dateAdded,
    yearAdded: v.dateAdded.slice(0, 4),
    nvdPublished: pub,
    daysToKev: pub ? days(pub, v.dateAdded) : null,
    ransomware: v.knownRansomwareCampaignUse === 'Known',
    category: category(v.vendorProject, v.product),
    aiTooling: AI_TOOLING.test(s),
    fileTransfer: FILE_TRANSFER.test(s),
    family: family(v.cwes),
    cwes: (v.cwes || []).join(' '),
  };
});

// Study window: entries added from 1 January 2023. Earlier additions were mostly CISA loading its backlog.
const WINDOW_START = '2023-01-01';
const w = rows.filter((r) => r.dateAdded >= WINDOW_START);
const years = [...new Set(w.map((r) => r.yearAdded))].sort();

/* ---------- Results ---------- */

const timing = (set) => {
  const d = set.filter((r) => r.daysToKev !== null).map((r) => r.daysToKev);
  const band = (lo, hi) => d.filter((x) => x >= lo && x <= hi).length;
  return {
    n: d.length,
    medianDays: median(d),
    atOrBeforePublication: band(-Infinity, 0), within1to7: band(1, 7), within8to30: band(8, 30),
    within31to365: band(31, 365), over1Year: band(366, Infinity), over3Years: band(1096, Infinity),
  };
};
const newCves = (set) => set.filter((r) => r.nvdPublished && r.nvdPublished >= WINDOW_START);

const results = {
  source: { catalogVersion: kev.catalogVersion, dateReleased: kev.dateReleased, totalEntries: rows.length, nvdMatched: rows.filter((r) => r.nvdPublished).length },
  window: { start: WINDOW_START, end: kev.dateReleased.slice(0, 10), entries: w.length },
  addedByYear: count(rows, (r) => r.yearAdded),
  backlogDays: by(count(rows.filter((r) => r.yearAdded <= '2022'), (r) => r.dateAdded)).slice(0, 3),
  categories: Object.fromEntries(by(count(w, (r) => r.category)).map(([k, n]) => [k, { label: LABEL[k], n, pct: pct(n, w.length) }])),
  categoryByYear: Object.fromEntries(years.map((y) => { const s = w.filter((r) => r.yearAdded === y); return [y, { n: s.length, edge: s.filter((r) => r.category === 'edge').length, edgePct: pct(s.filter((r) => r.category === 'edge').length, s.length) }]; })),
  topVendors: by(count(w, (r) => r.vendor)).slice(0, 10).map(([v, n]) => ({ vendor: v, n, pct: pct(n, w.length) })),
  vendorsTotal: Object.keys(count(w, (r) => r.vendor)).length,
  topEdgeVendors: by(count(w.filter((r) => r.category === 'edge'), (r) => r.vendor)).slice(0, 8),
  timingAll: timing(w),
  timingNewCves: timing(newCves(w)),
  timingNewByCategory: Object.fromEntries(Object.keys(LABEL).map((c) => [c, timing(newCves(w).filter((r) => r.category === c))])),
  families: Object.fromEntries(by(count(w, (r) => r.family)).map(([k, n]) => [k, { label: FAMILY_LABEL[k], n, pct: pct(n, w.length) }])),
  familiesByCategory: Object.fromEntries(['edge', 'os', 'client', 'server'].map((c) => { const s = w.filter((r) => r.category === c); return [c, Object.fromEntries(by(count(s, (r) => r.family)).map(([k, n]) => [k, { n, pct: pct(n, s.length) }]))]; })),
  topCwes: by(count(w.flatMap((r) => r.cwes.split(' ').filter(Boolean).map((c) => ({ c }))), (x) => x.c)).slice(0, 10),
  ransomware: {
    windowKnown: w.filter((r) => r.ransomware).length,
    byCategory: Object.fromEntries(Object.keys(LABEL).map((c) => { const s = w.filter((r) => r.category === c); const k = s.filter((r) => r.ransomware).length; return [c, { n: s.length, known: k, pct: pct(k, s.length) }]; })),
    byYearAdded: Object.fromEntries(Object.keys(count(rows, (r) => r.yearAdded)).sort().map((y) => { const s = rows.filter((r) => r.yearAdded === y); const k = s.filter((r) => r.ransomware).length; return [y, { n: s.length, known: k, pct: pct(k, s.length) }]; })),
  },
  aiTooling: w.filter((r) => r.aiTooling).map((r) => ({ cve: r.cve, vendor: r.vendor, product: r.product, dateAdded: r.dateAdded, daysToKev: r.daysToKev })),
  fileTransfer: { n: w.filter((r) => r.fileTransfer).length, ransomwareKnown: w.filter((r) => r.fileTransfer && r.ransomware).length },
};

mkdirSync(out, { recursive: true });
writeFileSync(join(out, 'results.json'), JSON.stringify(results, null, 2) + '\n');

const cols = ['cve', 'vendor', 'product', 'dateAdded', 'nvdPublished', 'daysToKev', 'ransomware', 'category', 'family', 'cwes', 'aiTooling', 'fileTransfer'];
const csvCell = (v) => { const s = v === null ? '' : String(v); return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s; };
writeFileSync(join(out, 'kev-2023-2026.csv'), [cols.join(','), ...w.map((r) => cols.map((c) => csvCell(r[c])).join(','))].join('\n') + '\n');

console.log(`Coded ${w.length} entries (${results.window.start} to ${results.window.end}) from catalogue ${results.source.catalogVersion}.`);