berichtsheftgenerator/web/scripts/extract-phrases.mjs

/**
 * S01 – Extrahiert Rohtext + strukturierte Phrasen aus allen Berichtsheft-.docx.
 * Ausgabe: extracted-all.txt, raw-phrases.json (+ Aggregat für spätere Schritte).
 */
import { readdirSync, readFileSync, writeFileSync, statSync } from "fs";
import { dirname, join, resolve, basename } from "path";
import { fileURLToPath } from "url";
import PizZip from "pizzip";

const __dirname = dirname(fileURLToPath(import.meta.url));
const ROOT = resolve(__dirname, "..", "..");
const YEAR_DIRS = [
  join(ROOT, "1. Ausbildungsjahr"),
  join(ROOT, "2. Ausbildungsjahr"),
];

function listDocx(dir) {
  try {
    return readdirSync(dir)
      .filter((f) => f.toLowerCase().endsWith(".docx") && !f.startsWith("~$"))
      .map((f) => join(dir, f))
      .filter((p) => statSync(p).isFile());
  } catch {
    return [];
  }
}

function docxToLines(path) {
  const zip = new PizZip(readFileSync(path));
  const file = zip.file("word/document.xml");
  if (!file) return [];
  const xml = file.asText();
  return xml
    .split(/<\/w:p>/)
    .map((p) =>
      [...p.matchAll(/<w:t[^>]*>([^<]*)<\/w:t>/g)]
        .map((m) => m[1])
        .join("")
        .replace(/\s+/g, " ")
        .trim(),
    )
    .filter(Boolean);
}

function dedupeConsecutive(lines) {
  /** Content is duplicated (two table columns) – keep first occurrence. */
  const out = [];
  const seenInDoc = new Set();
  for (const line of lines) {
    const key = line.toLowerCase();
    if (seenInDoc.has(key)) continue;
    seenInDoc.add(key);
    out.push(line);
  }
  return out;
}

function sliceBetween(lines, startRe, endRe) {
  const start = lines.findIndex((l) => startRe.test(l));
  if (start < 0) return [];
  const rest = lines.slice(start + 1);
  const end = rest.findIndex((l) => endRe.test(l));
  return end < 0 ? rest : rest.slice(0, end);
}

function decodeEntities(s) {
  return s
    .replace(/&amp;/g, "&")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"')
    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)));
}

function stripLeadingJunk(s) {
  return decodeEntities(s)
    .replace(/^[\d\s]+/, "")
    .replace(/^[\s\-•·]+/, "")
    .replace(/^\d{4,}/, "")
    .replace(/\s+/g, " ")
    .trim()
    .replace(/[,\s]+$/, "")
    .trim();
}

function isHeaderNoise(s) {
  return /^(Abteilung|Ausbildungsnachweis|Für die Zeit|Name|Datum|Zweites|Erstes|Drittes|Betriebliche|Zuordnung|Thema der Woche|Berufsschule|Lfd\.?\s*Nr|Ausbilderin|Ausbilder)/i.test(
    s,
  );
}

const JUNK_RE =
  /^(43180|Nico|Baumann|Ferien|Feiertag|Urlaub|Schultag|Krank|Montag|Dienstag|Mittwoch|Donnerstag|Freitag|Samstag|Sonntag|………………|…………|\.+|-+|_+|Additive,?$|\d{4,}|\d+$)/i;

function isJunk(s) {
  if (!s || s.length < 3) return true;
  if (JUNK_RE.test(s)) return true;
  if (/^[.\-_\s…]+$/.test(s)) return true;
  if (/Unterschrift/i.test(s)) return true;
  if (
    /Ausbildungsnachweis|Für die Zeit|Betriebliche Tätig|Zuordnung|Thema der Woche|Berufsschule|Lfd\.?\s*Nr/i.test(
      s,
    )
  )
    return true;
  if (/^\d{5,}/.test(s)) return true;
  if (/Zwischenprüfung\s+Freitag/i.test(s)) return true;
  if (!/[a-zäöüß]/i.test(s)) return true;
  return false;
}

function cleanPhrase(text) {
  const t = stripLeadingJunk(text);
  if (isJunk(t) || isHeaderNoise(t)) return null;
  if (t.length < 3 || t.length > 180) return null;
  return t;
}

function isFrameworkRef(s) {
  return (
    /^Lfd\.?\s*Nr/i.test(s) ||
    /^\d+(\.\d+)+$/.test(s) ||
    /Ausbildungsrahmenplan/i.test(s) ||
    /^Zuordnung der Tätigkeit/i.test(s)
  );
}

function isPlausibleTheme(s) {
  if (!s || s.length < 3 || s.length > 55) return false;
  if (/[.]{2,}/.test(s)) return false;
  if (/\b(abgeholt|gefahren|gebucht|auffüllen|einsortieren|anbieten|festhalten)\b/i.test(s))
    return false;
  if (/^(bei |zum |was |welche |wenn |mehr |immer )/i.test(s)) return false;
  return true;
}

function parseSections(lines) {
  const unique = dedupeConsecutive(lines);

  const betriebIdx = unique.findIndex((l) => /^Betriebliche Tätigkeiten/i.test(l));
  let activityLines = [];
  if (betriebIdx > 0) {
    let start = 0;
    for (let i = 0; i < betriebIdx; i++) {
      if (/^Name\b/i.test(unique[i]) || /^Datum\b/i.test(unique[i])) start = i + 1;
    }
    activityLines = unique.slice(start, betriebIdx);
  }

  // Rahmenplan-Refs zwischen „Betriebliche Tätigkeiten“ und „Thema der Woche“
  const frameworkRaw = sliceBetween(
    unique,
    /^Betriebliche Tätigkeiten/i,
    /^Thema der Woche/i,
  );

  const themeAfter = sliceBetween(
    unique,
    /^Thema der Woche/i,
    /^(Zuordnung zum Lernziel|Berufsschule)/i,
  );

  const schoolLines = sliceBetween(
    unique,
    /^Berufsschule/i,
    /^(Datum und Unterschrift|\.{3,}|…{2,})/i,
  ).filter((l) => !/^(Datum und Unterschrift|\.{3,}|…)/i.test(l));

  return {
    activityLines,
    frameworkRaw,
    themeLines: themeAfter,
    schoolLines,
  };
}

function addCount(map, text) {
  const t = cleanPhrase(text);
  if (!t) return;
  const key = t.toLowerCase();
  const prev = map.get(key);
  if (prev) prev.count += 1;
  else map.set(key, { text: t, count: 1 });
}

function toSortedList(map, { minLen = 4, maxLen = 180 } = {}) {
  return [...map.values()]
    .filter((x) => x.text.length >= minLen && x.text.length <= maxLen)
    .sort((a, b) => b.count - a.count || a.text.localeCompare(b.text, "de"));
}

function structureHeft(path, lines) {
  const { activityLines, frameworkRaw, themeLines, schoolLines } = parseSections(lines);

  const activities = [];
  for (const a of activityLines) {
    const t = cleanPhrase(a);
    if (t) activities.push(t);
  }

  const frameworkRefs = [];
  for (const f of frameworkRaw) {
    const raw = decodeEntities(f).replace(/\s+/g, " ").trim();
    if (!raw) continue;
    if (isFrameworkRef(raw) || /^Lfd/i.test(raw)) {
      frameworkRefs.push(raw);
      continue;
    }
    // kurze Stichworte vor „Thema der Woche“ oft Rahmenplan-/Themenreste – behalten zum Filtern
    const cleaned = stripLeadingJunk(raw);
    if (
      cleaned &&
      cleaned.length <= 40 &&
      !isJunk(cleaned) &&
      !/^[.\-…]+$/.test(cleaned)
    ) {
      frameworkRefs.push(cleaned);
    }
  }

  const weekThemes = [];
  for (const t of themeLines) {
    const cleaned = stripLeadingJunk(t);
    if (!cleaned || cleaned === "-") continue;
    if (/ferien/i.test(cleaned)) continue;
    if (!isPlausibleTheme(cleaned)) continue;
    if (isJunk(cleaned) || isHeaderNoise(cleaned)) continue;
    weekThemes.push(cleaned);
  }

  const schoolTopics = [];
  for (const s of schoolLines) {
    if (/ferien/i.test(s)) continue;
    const t = cleanPhrase(s);
    if (t) schoolTopics.push(t);
  }

  return {
    sourceFile: basename(path),
    activities,
    weekThemes,
    schoolTopics,
    frameworkRefs,
  };
}

async function main() {
  const files = YEAR_DIRS.flatMap(listDocx).sort((a, b) =>
    basename(a).localeCompare(basename(b), "de"),
  );
  if (!files.length) {
    console.error("Keine .docx in 1./2. Ausbildungsjahr gefunden.");
    process.exit(1);
  }

  const activities = new Map();
  const themes = new Map();
  const school = new Map();
  const dump = [];
  const rawPhrases = [];

  for (const path of files) {
    const lines = docxToLines(path);
    const name = basename(path);
    dump.push(`===== ${name} =====`, "", ...lines, "", "");

    const structured = structureHeft(path, lines);
    rawPhrases.push(structured);

    for (const a of structured.activities) addCount(activities, a);
    for (const t of structured.weekThemes) addCount(themes, t);
    for (const s of structured.schoolTopics) addCount(school, s);
  }

  const actList = toSortedList(activities, { minLen: 6, maxLen: 160 }).filter(
    (a) =>
      !/^\d/.test(a.text) &&
      !/Lfd/i.test(a.text) &&
      /[a-zäöüß]{3,}/i.test(a.text),
  );
  const themeList = toSortedList(themes, { minLen: 3, maxLen: 80 }).filter(
    (t) => !/^[.\-]+$/.test(t.text) && t.text !== "-",
  );
  const schoolList = toSortedList(school, { minLen: 4, maxLen: 160 }).filter(
    (s) => !/^krank$/i.test(s.text) && !/stillarbeit$/i.test(s.text),
  );

  const clean = {
    activities: actList,
    themes: themeList.map((t) => t.text),
    themesWeighted: themeList,
    school: schoolList.map((s) => s.text),
    schoolWeighted: schoolList,
    meta: {
      files: files.length,
      activities: actList.length,
      themes: themeList.length,
      school: schoolList.length,
      generatedAt: new Date().toISOString(),
    },
  };

  const rawOut = {
    meta: {
      files: files.length,
      hefte: rawPhrases.length,
      generatedAt: new Date().toISOString(),
    },
    hefte: rawPhrases,
  };

  writeFileSync(join(__dirname, "extracted-all.txt"), dump.join("\n"), "utf8");
  writeFileSync(join(__dirname, "raw-phrases.json"), JSON.stringify(rawOut, null, 2), "utf8");
  // Aggregat nur als Zwischenstand; saubere Form kommt aus S02 (clean-phrases.mjs)
  writeFileSync(join(__dirname, "parsed-phrases.json"), JSON.stringify(clean, null, 2), "utf8");

  const withActs = rawPhrases.filter((h) => h.activities.length > 0).length;
  const emptyActs = rawPhrases.filter((h) => h.activities.length === 0);
  console.log(
    `Parsed ${files.length} docx → raw-phrases.json (${withActs} mit Activities, ${emptyActs.length} ohne)`,
  );
  console.log(
    `Aggregat: ${actList.length} Tätigkeiten, ${themeList.length} Themen, ${schoolList.length} Schule`,
  );
  if (emptyActs.length) {
    console.log(
      "Ohne Activities:",
      emptyActs.map((h) => h.sourceFile).join(" · "),
    );
  }
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});