/**
 * Build the compact, reviewable flight-delay snapshot used by EXP-005.
 *
 * Source: US Bureau of Transportation Statistics, Reporting Carrier On-Time
 * Performance (1987-present). The official monthly archives are large; this
 * script keeps only the daily sufficient statistics fixed by the protocol.
 *
 *   npm run data:retrograde
 *   npm run data:retrograde -- --from=2024-01 --to=2026-06 --full
 */

import { createHash } from "node:crypto";
import { inflateRawSync } from "node:zlib";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";

const OUTPUT = resolve(process.cwd(), "src/content/data/retrograde-flight-delays.json");
const DATASET_ID = "bts-reporting-carrier-on-time-performance";
const ARCHIVE_ROOT = "https://transtats.bts.gov/PREZIP";
const TABLE_URL =
  "https://www.transtats.bts.gov/TableInfo.asp?QO_fu146_anzr=b0-gvzr&gnoyr_VQ=FGJ";
const DOWNLOAD_URL =
  "https://www.transtats.bts.gov/DL_SelectFields.aspx?QO_fu146_anzr=%5D&gnoyr_VQ=FGJ";

type Day = {
  date: string;
  flights: number;
  delayed: number;
  cancelled: number;
  diverted: number;
};

type Archive = {
  month: string;
  url: string;
  sha256: string;
  sourceModifiedAt: string | null;
  rows: number;
  flights: number;
};

type Snapshot = {
  schemaVersion: 1;
  dataset: {
    id: string;
    title: string;
    publisher: string;
    tableUrl: string;
    downloadUrl: string;
    eventDefinition: string;
    coverageStart: string;
    coverageEnd: string;
    generatedAt: string;
    licence: string;
  };
  archives: Archive[];
  days: Day[];
};

type Month = { year: number; month: number };

function parseMonth(value: string): Month {
  const match = /^(\d{4})-(0[1-9]|1[0-2])$/.exec(value);
  if (!match) throw new Error(`Invalid month ${value}; expected YYYY-MM`);
  return { year: Number(match[1]), month: Number(match[2]) };
}

function monthKey(value: Month): string {
  return `${value.year}-${String(value.month).padStart(2, "0")}`;
}

function shiftMonth(value: Month, delta: number): Month {
  const date = new Date(Date.UTC(value.year, value.month - 1 + delta, 1));
  return { year: date.getUTCFullYear(), month: date.getUTCMonth() + 1 };
}

function monthsBetween(from: Month, to: Month): Month[] {
  const values: Month[] = [];
  for (let cursor = from; monthKey(cursor) <= monthKey(to); cursor = shiftMonth(cursor, 1)) {
    values.push(cursor);
  }
  return values;
}

function argument(name: string): string | undefined {
  const prefix = `--${name}=`;
  return process.argv.find((value) => value.startsWith(prefix))?.slice(prefix.length);
}

function archiveUrl(month: Month): string {
  return (
    `${ARCHIVE_ROOT}/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_` +
    `${month.year}_${month.month}.zip`
  );
}

function findSignature(buffer: Buffer, signature: number, from: number): number {
  for (let offset = from; offset >= 0; offset--) {
    if (buffer.readUInt32LE(offset) === signature) return offset;
  }
  return -1;
}

/** Extract the first CSV using the ZIP central directory; no shell unzip dependency. */
function extractCsv(zip: Buffer): Buffer {
  const eocd = findSignature(zip, 0x06054b50, Math.max(0, zip.length - 22));
  if (eocd < 0) throw new Error("BTS response is not a readable ZIP archive");

  const entries = zip.readUInt16LE(eocd + 10);
  let cursor = zip.readUInt32LE(eocd + 16);

  for (let entry = 0; entry < entries; entry++) {
    if (zip.readUInt32LE(cursor) !== 0x02014b50) throw new Error("Invalid ZIP central directory");

    const method = zip.readUInt16LE(cursor + 10);
    const compressedSize = zip.readUInt32LE(cursor + 20);
    const fileNameLength = zip.readUInt16LE(cursor + 28);
    const extraLength = zip.readUInt16LE(cursor + 30);
    const commentLength = zip.readUInt16LE(cursor + 32);
    const localOffset = zip.readUInt32LE(cursor + 42);
    const fileName = zip.toString("utf8", cursor + 46, cursor + 46 + fileNameLength);

    if (fileName.toLowerCase().endsWith(".csv")) {
      if (zip.readUInt32LE(localOffset) !== 0x04034b50) throw new Error("Invalid ZIP local header");
      const localNameLength = zip.readUInt16LE(localOffset + 26);
      const localExtraLength = zip.readUInt16LE(localOffset + 28);
      const dataOffset = localOffset + 30 + localNameLength + localExtraLength;
      const compressed = zip.subarray(dataOffset, dataOffset + compressedSize);
      if (method === 0) return Buffer.from(compressed);
      if (method === 8) return inflateRawSync(compressed);
      throw new Error(`Unsupported ZIP compression method ${method}`);
    }

    cursor += 46 + fileNameLength + extraLength + commentLength;
  }

  throw new Error("BTS ZIP archive contains no CSV file");
}

function parseCsvLine(line: string): string[] {
  const values: string[] = [];
  let value = "";
  let quoted = false;

  for (let index = 0; index < line.length; index++) {
    const character = line[index];
    if (character === '"') {
      if (quoted && line[index + 1] === '"') {
        value += '"';
        index++;
      } else {
        quoted = !quoted;
      }
    } else if (character === "," && !quoted) {
      values.push(value);
      value = "";
    } else {
      value += character;
    }
  }
  values.push(value);
  return values;
}

function decodeCell(buffer: Buffer, start: number, end: number): string {
  while (start < end && (buffer[start] === 0x20 || buffer[start] === 0x22)) start++;
  while (end > start && (buffer[end - 1] === 0x0d || buffer[end - 1] === 0x20 || buffer[end - 1] === 0x22)) end--;
  return buffer.toString("utf8", start, end);
}

function selectedCells(
  buffer: Buffer,
  start: number,
  end: number,
  positionByColumn: number[],
  valueCount: number,
): string[] {
  const values = new Array<string>(valueCount).fill("");
  const lastColumn = positionByColumn.length - 1;
  let column = 0;
  let fieldStart = start;
  let quoted = false;

  const capture = (fieldEnd: number) => {
    const position = positionByColumn[column];
    if (position >= 0) values[position] = decodeCell(buffer, fieldStart, fieldEnd);
  };

  for (let offset = start; offset < end; offset++) {
    const byte = buffer[offset];
    if (byte === 0x22) quoted = !quoted;
    if (byte === 0x2c && !quoted) {
      capture(offset);
      column++;
      fieldStart = offset + 1;
      if (column > lastColumn) break;
    }
  }
  if (column <= lastColumn) capture(end);
  return values;
}

function aggregateCsv(csv: Buffer, expectedMonth: string): { days: Day[]; rows: number; flights: number } {
  const headerEnd = csv.indexOf(0x0a);
  if (headerEnd < 0) throw new Error("BTS CSV has no header row");
  const header = parseCsvLine(csv.toString("utf8", 0, headerEnd).replace(/\r$/, ""));
  const names = ["FlightDate", "Flights", "ArrDel15", "Cancelled", "Diverted"];
  const indexes = names.map((name) => header.indexOf(name));
  if (indexes.some((index) => index < 0)) {
    throw new Error(`BTS schema changed; expected fields: ${names.join(", ")}`);
  }

  const lastColumn = Math.max(...indexes);
  const positionByColumn = new Array<number>(lastColumn + 1).fill(-1);
  indexes.forEach((column, position) => (positionByColumn[column] = position));

  const days = new Map<string, Day>();
  let rows = 0;
  let totalFlights = 0;
  let lineStart = headerEnd + 1;

  while (lineStart < csv.length) {
    let lineEnd = csv.indexOf(0x0a, lineStart);
    if (lineEnd < 0) lineEnd = csv.length;
    if (lineEnd > lineStart + 1) {
      const [date, flightsText, delayedText, cancelledText, divertedText] = selectedCells(
        csv,
        lineStart,
        lineEnd,
        positionByColumn,
        names.length,
      );
      if (date) {
        if (!date.startsWith(expectedMonth)) {
          throw new Error(`Archive ${expectedMonth} contains an unexpected date: ${date}`);
        }
        const flights = Number.parseFloat(flightsText) || 0;
        const day = days.get(date) ?? { date, flights: 0, delayed: 0, cancelled: 0, diverted: 0 };
        day.flights += flights;
        if ((Number.parseFloat(delayedText) || 0) >= 1) day.delayed += flights;
        if ((Number.parseFloat(cancelledText) || 0) >= 1) day.cancelled += flights;
        if ((Number.parseFloat(divertedText) || 0) >= 1) day.diverted += flights;
        days.set(date, day);
        rows++;
        totalFlights += flights;
      }
    }
    lineStart = lineEnd + 1;
  }

  const result = [...days.values()].sort((a, b) => a.date.localeCompare(b.date));
  if (result.length < 28 || totalFlights <= 0) throw new Error(`Archive ${expectedMonth} is incomplete`);
  for (const day of result) {
    if (day.delayed > day.flights || day.cancelled > day.flights || day.diverted > day.flights) {
      throw new Error(`Impossible counts on ${day.date}`);
    }
  }
  return { days: result, rows, flights: totalFlights };
}

async function existingSnapshot(): Promise<Snapshot | null> {
  try {
    return JSON.parse(await readFile(OUTPUT, "utf8")) as Snapshot;
  } catch (error) {
    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
    throw error;
  }
}

async function ingest(month: Month): Promise<{ archive: Archive; days: Day[] } | null> {
  const key = monthKey(month);
  const url = archiveUrl(month);
  process.stdout.write(`BTS ${key}: downloading… `);
  const response = await fetch(url, {
    headers: { "user-agent": "occultresearch.org EXP-005 reproducible-ingestion/1.0" },
  });
  if (response.status === 404) {
    console.log("not released");
    return null;
  }
  if (!response.ok) throw new Error(`BTS ${key}: HTTP ${response.status}`);

  const zip = Buffer.from(await response.arrayBuffer());
  const sha256 = createHash("sha256").update(zip).digest("hex");
  process.stdout.write(`${(zip.length / 1_000_000).toFixed(1)} MB; reducing… `);
  const aggregated = aggregateCsv(extractCsv(zip), key);
  console.log(`${aggregated.rows.toLocaleString("en-US")} rows → ${aggregated.days.length} days`);

  return {
    archive: {
      month: key,
      url,
      sha256,
      sourceModifiedAt: response.headers.get("last-modified"),
      rows: aggregated.rows,
      flights: aggregated.flights,
    },
    days: aggregated.days,
  };
}

function comparable(snapshot: Snapshot): string {
  return JSON.stringify({ ...snapshot, dataset: { ...snapshot.dataset, generatedAt: "" } });
}

async function main() {
  const existing = await existingSnapshot();
  const full = process.argv.includes("--full");
  const previousMonth = shiftMonth(
    { year: new Date().getUTCFullYear(), month: new Date().getUTCMonth() + 1 },
    -1,
  );
  const existingLast = existing?.archives.at(-1)?.month;
  const defaultFrom = !full && existingLast ? monthKey(shiftMonth(parseMonth(existingLast), -2)) : "2024-01";
  const from = parseMonth(argument("from") ?? defaultFrom);
  const to = parseMonth(argument("to") ?? monthKey(previousMonth));
  if (monthKey(from) > monthKey(to)) throw new Error("--from must not be later than --to");

  const archiveMap = new Map((existing?.archives ?? []).map((archive) => [archive.month, archive]));
  const dayMap = new Map((existing?.days ?? []).map((day) => [day.date, day]));
  const requestedMonths = monthsBetween(from, to);
  const concurrency = Math.max(1, Math.min(4, Number.parseInt(argument("concurrency") ?? "3", 10) || 3));

  for (let offset = 0; offset < requestedMonths.length; offset += concurrency) {
    const batch = requestedMonths.slice(offset, offset + concurrency);
    const results = await Promise.all(batch.map((month) => ingest(month)));
    for (let index = 0; index < batch.length; index++) {
      const ingested = results[index];
      if (!ingested) continue;
      const key = monthKey(batch[index]);
      archiveMap.set(key, ingested.archive);
      for (const date of [...dayMap.keys()]) if (date.startsWith(key)) dayMap.delete(date);
      for (const day of ingested.days) dayMap.set(day.date, day);
    }
  }

  const archives = [...archiveMap.values()].sort((a, b) => a.month.localeCompare(b.month));
  const days = [...dayMap.values()].sort((a, b) => a.date.localeCompare(b.date));
  if (!archives.length || !days.length) throw new Error("No released BTS archive was ingested");

  const snapshot: Snapshot = {
    schemaVersion: 1,
    dataset: {
      id: DATASET_ID,
      title: "Reporting Carrier On-Time Performance (1987-present)",
      publisher: "US Bureau of Transportation Statistics",
      tableUrl: TABLE_URL,
      downloadUrl: DOWNLOAD_URL,
      eventDefinition: "A scheduled domestic flight arriving at least 15 minutes late (ArrDel15 = 1)",
      coverageStart: days[0].date,
      coverageEnd: days.at(-1)!.date,
      generatedAt: new Date().toISOString(),
      licence: "US federal government public data",
    },
    archives,
    days,
  };

  if (existing && comparable(existing) === comparable(snapshot)) {
    console.log(`Snapshot unchanged through ${snapshot.dataset.coverageEnd}`);
    return;
  }

  await mkdir(dirname(OUTPUT), { recursive: true });
  await writeFile(OUTPUT, `${JSON.stringify(snapshot, null, 2)}\n`, "utf8");
  console.log(`Wrote ${OUTPUT}: ${days.length} days, ${archives.length} source archives`);
}

main().catch((error) => {
  console.error(error instanceof Error ? error.message : error);
  process.exitCode = 1;
});