A · Ham veri ve temizlik

## 3. Annual Paid Leave

Part-time staff get the same number as full-time staff. The HR portal
shows the balance for each person. Questions go to the unit's HR
business partner.

| Years of service (full years) | Annual paid leave (working days) |
|---|---|
| 1 | 16 |
| 2 | 16 |

/**
 * What it does: turns each file of one edition into a document with labels and text.
 *
 * Read one edition of the corpus.
 * @param edition folder name under corpus/, e.g. "2025"
 * @returns one Doc per file: metadata from the front matter, body untouched
 */
export async function loadCorpus(edition: string): Promise<Doc[]> {
  const files = await readEditionFiles(edition);
  return files.map(({ path, raw }) => {
    const { meta, body } = parseFrontMatter(raw);
    const missing = KEYS.filter((k) => !meta[k]);
    if (missing.length) throw new Error(`${path}: front matter is missing ${missing.join(", ")}`);
    return { ...(meta as unknown as DocMeta), path, body };
  });
}

/**
 * What it does: splits a file into its labels on top (id, edition, access…) and its text.
 *
 * Split a file into its front matter (the `---` block on top) and its body.
 * Only `key: value` lines are supported — enough for our metadata, no YAML library needed.
 * @param raw the whole file, line endings already normalised to \n
 * @example parseFrontMatter('---\nid: hr-leave\nedition: "2025"\n---\nbody').meta.edition // "2025"
 */
export function parseFrontMatter(raw: string): { meta: Record<string, string>; body: string } {
  const m = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
  if (!m) return { meta: {}, body: raw };
  const meta: Record<string, string> = {};
  for (const l of m[1]!.split("\n")) {
    const kv = l.match(/^(\w+):\s*(.*)$/);
    if (kv) meta[kv[1]!] = kv[2]!.trim().replace(/^["'](.*)["']$/, "$1");
  }
  return { meta, body: m[2]! };
}

/** Step 1 output: one file from corpus/<edition>/, front matter parsed, body untouched. */
export interface Doc extends DocMeta {
  path: string;
  body: string;
}

  const docs: CleanDoc[] = input.data.map(({ path: _path, body, ...meta }) => ({ ...meta, ...cleanText(body) }));

/**
 * What it does: checks if a line is a heading, a table row, or a blank line.
 *
 * Headings, table rows and blank lines are complete on their own — nothing is joined onto them.
 */
const closed = (l: string) => l.startsWith("#") || l.startsWith("|") || l.trim() === "";
/**
 * What it does: checks if a line must start fresh, not be joined to the line before.
 *
 * Headings, table rows, blank lines and list items always start a new line.
 */
const opens = (l: string) => closed(l) || /^\s*([-*]|\d+\.)\s/.test(l);

/**
 * What it does: drops noise lines, then joins lines that the page width broke apart.
 *
 * Remove noise lines, then re-join prose lines that the page width broke apart.
 * @param body a document body from step 1
 * @returns the clean text and how many noise lines were dropped
 */
export function cleanText(body: string, rules: RegExp[] = NOISE): { text: string; noiseLines: number } {
  const lines = body.split("\n");
  const kept = lines.filter((l) => !isNoise(l, rules));
  const out: string[] = [];
  for (const l of kept) {
    const prev = out[out.length - 1];
    if (prev !== undefined && !closed(prev) && !opens(l)) out[out.length - 1] = `${prev} ${l.trim()}`;
    else out.push(l);
  }
  const text = out.join("\n").replace(/\n{3,}/g, "\n\n").trim();
  return { text, noiseLines: lines.length - kept.length };
}

// ── YOUR TURN ───────────────────────────────────────────────────────────────────────────────────
// The Turkish document (it-security) repeats "GİZLİ – Yalnızca şirket içi kullanım içindir" under the page header.
// No rule removes it yet: find it in data/2-clean.json (copy the İ and the – from there). Write the rule the way
// the English confidentiality rule below is written. Check: npm run step -- 2 → it-security shows −12 noise lines, not −8.
export const turkishConfidential: RegExp | undefined = undefined;

// TOGGLE noise rules — comment one out, re-run step 2, and find what it used to remove
const NOISE: RegExp[] = [
  /^Kraken Air \| .+ \| (Sürüm|Edition) \d{4}$/, // default — page header, repeated on every page
  /^CONFIDENTIAL – .+$/, // default — English confidentiality line, under the page header
  /^Sayfa \d+ \/ \d+$/, // default — Turkish page number
  /^Page \d+ of \d+$/, // default — English page number

  ...(turkishConfidential ? [turkishConfidential] : []),
];