gute
This commit is contained in:
@@ -0,0 +1,336 @@
|
||||
/**
|
||||
* S01 – Extrahiert Rohtext + strukturierte Phrasen aus allen Berichtsheft-.docx.
|
||||
* Ausgabe: extracted-all.txt, raw-phrases.json (+ Aggregat für spätere Schritte).
|
||||
*/
|
||||
import { readdirSync, readFileSync, writeFileSync, statSync } from "fs";
|
||||
import { dirname, join, resolve, basename } from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
import PizZip from "pizzip";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = resolve(__dirname, "..", "..");
|
||||
const YEAR_DIRS = [
|
||||
join(ROOT, "1. Ausbildungsjahr"),
|
||||
join(ROOT, "2. Ausbildungsjahr"),
|
||||
];
|
||||
|
||||
function listDocx(dir) {
|
||||
try {
|
||||
return readdirSync(dir)
|
||||
.filter((f) => f.toLowerCase().endsWith(".docx") && !f.startsWith("~$"))
|
||||
.map((f) => join(dir, f))
|
||||
.filter((p) => statSync(p).isFile());
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function docxToLines(path) {
|
||||
const zip = new PizZip(readFileSync(path));
|
||||
const file = zip.file("word/document.xml");
|
||||
if (!file) return [];
|
||||
const xml = file.asText();
|
||||
return xml
|
||||
.split(/<\/w:p>/)
|
||||
.map((p) =>
|
||||
[...p.matchAll(/<w:t[^>]*>([^<]*)<\/w:t>/g)]
|
||||
.map((m) => m[1])
|
||||
.join("")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim(),
|
||||
)
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function dedupeConsecutive(lines) {
|
||||
/** Content is duplicated (two table columns) – keep first occurrence. */
|
||||
const out = [];
|
||||
const seenInDoc = new Set();
|
||||
for (const line of lines) {
|
||||
const key = line.toLowerCase();
|
||||
if (seenInDoc.has(key)) continue;
|
||||
seenInDoc.add(key);
|
||||
out.push(line);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function sliceBetween(lines, startRe, endRe) {
|
||||
const start = lines.findIndex((l) => startRe.test(l));
|
||||
if (start < 0) return [];
|
||||
const rest = lines.slice(start + 1);
|
||||
const end = rest.findIndex((l) => endRe.test(l));
|
||||
return end < 0 ? rest : rest.slice(0, end);
|
||||
}
|
||||
|
||||
function decodeEntities(s) {
|
||||
return s
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)));
|
||||
}
|
||||
|
||||
function stripLeadingJunk(s) {
|
||||
return decodeEntities(s)
|
||||
.replace(/^[\d\s]+/, "")
|
||||
.replace(/^[\s\-•·]+/, "")
|
||||
.replace(/^\d{4,}/, "")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.replace(/[,\s]+$/, "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function isHeaderNoise(s) {
|
||||
return /^(Abteilung|Ausbildungsnachweis|Für die Zeit|Name|Datum|Zweites|Erstes|Drittes|Betriebliche|Zuordnung|Thema der Woche|Berufsschule|Lfd\.?\s*Nr|Ausbilderin|Ausbilder)/i.test(
|
||||
s,
|
||||
);
|
||||
}
|
||||
|
||||
const JUNK_RE =
|
||||
/^(43180|Nico|Baumann|Ferien|Feiertag|Urlaub|Schultag|Krank|Montag|Dienstag|Mittwoch|Donnerstag|Freitag|Samstag|Sonntag|………………|…………|\.+|-+|_+|Additive,?$|\d{4,}|\d+$)/i;
|
||||
|
||||
function isJunk(s) {
|
||||
if (!s || s.length < 3) return true;
|
||||
if (JUNK_RE.test(s)) return true;
|
||||
if (/^[.\-_\s…]+$/.test(s)) return true;
|
||||
if (/Unterschrift/i.test(s)) return true;
|
||||
if (
|
||||
/Ausbildungsnachweis|Für die Zeit|Betriebliche Tätig|Zuordnung|Thema der Woche|Berufsschule|Lfd\.?\s*Nr/i.test(
|
||||
s,
|
||||
)
|
||||
)
|
||||
return true;
|
||||
if (/^\d{5,}/.test(s)) return true;
|
||||
if (/Zwischenprüfung\s+Freitag/i.test(s)) return true;
|
||||
if (!/[a-zäöüß]/i.test(s)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function cleanPhrase(text) {
|
||||
const t = stripLeadingJunk(text);
|
||||
if (isJunk(t) || isHeaderNoise(t)) return null;
|
||||
if (t.length < 3 || t.length > 180) return null;
|
||||
return t;
|
||||
}
|
||||
|
||||
function isFrameworkRef(s) {
|
||||
return (
|
||||
/^Lfd\.?\s*Nr/i.test(s) ||
|
||||
/^\d+(\.\d+)+$/.test(s) ||
|
||||
/Ausbildungsrahmenplan/i.test(s) ||
|
||||
/^Zuordnung der Tätigkeit/i.test(s)
|
||||
);
|
||||
}
|
||||
|
||||
function isPlausibleTheme(s) {
|
||||
if (!s || s.length < 3 || s.length > 55) return false;
|
||||
if (/[.]{2,}/.test(s)) return false;
|
||||
if (/\b(abgeholt|gefahren|gebucht|auffüllen|einsortieren|anbieten|festhalten)\b/i.test(s))
|
||||
return false;
|
||||
if (/^(bei |zum |was |welche |wenn |mehr |immer )/i.test(s)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function parseSections(lines) {
|
||||
const unique = dedupeConsecutive(lines);
|
||||
|
||||
const betriebIdx = unique.findIndex((l) => /^Betriebliche Tätigkeiten/i.test(l));
|
||||
let activityLines = [];
|
||||
if (betriebIdx > 0) {
|
||||
let start = 0;
|
||||
for (let i = 0; i < betriebIdx; i++) {
|
||||
if (/^Name\b/i.test(unique[i]) || /^Datum\b/i.test(unique[i])) start = i + 1;
|
||||
}
|
||||
activityLines = unique.slice(start, betriebIdx);
|
||||
}
|
||||
|
||||
// Rahmenplan-Refs zwischen „Betriebliche Tätigkeiten“ und „Thema der Woche“
|
||||
const frameworkRaw = sliceBetween(
|
||||
unique,
|
||||
/^Betriebliche Tätigkeiten/i,
|
||||
/^Thema der Woche/i,
|
||||
);
|
||||
|
||||
const themeAfter = sliceBetween(
|
||||
unique,
|
||||
/^Thema der Woche/i,
|
||||
/^(Zuordnung zum Lernziel|Berufsschule)/i,
|
||||
);
|
||||
|
||||
const schoolLines = sliceBetween(
|
||||
unique,
|
||||
/^Berufsschule/i,
|
||||
/^(Datum und Unterschrift|\.{3,}|…{2,})/i,
|
||||
).filter((l) => !/^(Datum und Unterschrift|\.{3,}|…)/i.test(l));
|
||||
|
||||
return {
|
||||
activityLines,
|
||||
frameworkRaw,
|
||||
themeLines: themeAfter,
|
||||
schoolLines,
|
||||
};
|
||||
}
|
||||
|
||||
function addCount(map, text) {
|
||||
const t = cleanPhrase(text);
|
||||
if (!t) return;
|
||||
const key = t.toLowerCase();
|
||||
const prev = map.get(key);
|
||||
if (prev) prev.count += 1;
|
||||
else map.set(key, { text: t, count: 1 });
|
||||
}
|
||||
|
||||
function toSortedList(map, { minLen = 4, maxLen = 180 } = {}) {
|
||||
return [...map.values()]
|
||||
.filter((x) => x.text.length >= minLen && x.text.length <= maxLen)
|
||||
.sort((a, b) => b.count - a.count || a.text.localeCompare(b.text, "de"));
|
||||
}
|
||||
|
||||
function structureHeft(path, lines) {
|
||||
const { activityLines, frameworkRaw, themeLines, schoolLines } = parseSections(lines);
|
||||
|
||||
const activities = [];
|
||||
for (const a of activityLines) {
|
||||
const t = cleanPhrase(a);
|
||||
if (t) activities.push(t);
|
||||
}
|
||||
|
||||
const frameworkRefs = [];
|
||||
for (const f of frameworkRaw) {
|
||||
const raw = decodeEntities(f).replace(/\s+/g, " ").trim();
|
||||
if (!raw) continue;
|
||||
if (isFrameworkRef(raw) || /^Lfd/i.test(raw)) {
|
||||
frameworkRefs.push(raw);
|
||||
continue;
|
||||
}
|
||||
// kurze Stichworte vor „Thema der Woche“ oft Rahmenplan-/Themenreste – behalten zum Filtern
|
||||
const cleaned = stripLeadingJunk(raw);
|
||||
if (
|
||||
cleaned &&
|
||||
cleaned.length <= 40 &&
|
||||
!isJunk(cleaned) &&
|
||||
!/^[.\-…]+$/.test(cleaned)
|
||||
) {
|
||||
frameworkRefs.push(cleaned);
|
||||
}
|
||||
}
|
||||
|
||||
const weekThemes = [];
|
||||
for (const t of themeLines) {
|
||||
const cleaned = stripLeadingJunk(t);
|
||||
if (!cleaned || cleaned === "-") continue;
|
||||
if (/ferien/i.test(cleaned)) continue;
|
||||
if (!isPlausibleTheme(cleaned)) continue;
|
||||
if (isJunk(cleaned) || isHeaderNoise(cleaned)) continue;
|
||||
weekThemes.push(cleaned);
|
||||
}
|
||||
|
||||
const schoolTopics = [];
|
||||
for (const s of schoolLines) {
|
||||
if (/ferien/i.test(s)) continue;
|
||||
const t = cleanPhrase(s);
|
||||
if (t) schoolTopics.push(t);
|
||||
}
|
||||
|
||||
return {
|
||||
sourceFile: basename(path),
|
||||
activities,
|
||||
weekThemes,
|
||||
schoolTopics,
|
||||
frameworkRefs,
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const files = YEAR_DIRS.flatMap(listDocx).sort((a, b) =>
|
||||
basename(a).localeCompare(basename(b), "de"),
|
||||
);
|
||||
if (!files.length) {
|
||||
console.error("Keine .docx in 1./2. Ausbildungsjahr gefunden.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const activities = new Map();
|
||||
const themes = new Map();
|
||||
const school = new Map();
|
||||
const dump = [];
|
||||
const rawPhrases = [];
|
||||
|
||||
for (const path of files) {
|
||||
const lines = docxToLines(path);
|
||||
const name = basename(path);
|
||||
dump.push(`===== ${name} =====`, "", ...lines, "", "");
|
||||
|
||||
const structured = structureHeft(path, lines);
|
||||
rawPhrases.push(structured);
|
||||
|
||||
for (const a of structured.activities) addCount(activities, a);
|
||||
for (const t of structured.weekThemes) addCount(themes, t);
|
||||
for (const s of structured.schoolTopics) addCount(school, s);
|
||||
}
|
||||
|
||||
const actList = toSortedList(activities, { minLen: 6, maxLen: 160 }).filter(
|
||||
(a) =>
|
||||
!/^\d/.test(a.text) &&
|
||||
!/Lfd/i.test(a.text) &&
|
||||
/[a-zäöüß]{3,}/i.test(a.text),
|
||||
);
|
||||
const themeList = toSortedList(themes, { minLen: 3, maxLen: 80 }).filter(
|
||||
(t) => !/^[.\-]+$/.test(t.text) && t.text !== "-",
|
||||
);
|
||||
const schoolList = toSortedList(school, { minLen: 4, maxLen: 160 }).filter(
|
||||
(s) => !/^krank$/i.test(s.text) && !/stillarbeit$/i.test(s.text),
|
||||
);
|
||||
|
||||
const clean = {
|
||||
activities: actList,
|
||||
themes: themeList.map((t) => t.text),
|
||||
themesWeighted: themeList,
|
||||
school: schoolList.map((s) => s.text),
|
||||
schoolWeighted: schoolList,
|
||||
meta: {
|
||||
files: files.length,
|
||||
activities: actList.length,
|
||||
themes: themeList.length,
|
||||
school: schoolList.length,
|
||||
generatedAt: new Date().toISOString(),
|
||||
},
|
||||
};
|
||||
|
||||
const rawOut = {
|
||||
meta: {
|
||||
files: files.length,
|
||||
hefte: rawPhrases.length,
|
||||
generatedAt: new Date().toISOString(),
|
||||
},
|
||||
hefte: rawPhrases,
|
||||
};
|
||||
|
||||
writeFileSync(join(__dirname, "extracted-all.txt"), dump.join("\n"), "utf8");
|
||||
writeFileSync(join(__dirname, "raw-phrases.json"), JSON.stringify(rawOut, null, 2), "utf8");
|
||||
// Aggregat nur als Zwischenstand; saubere Form kommt aus S02 (clean-phrases.mjs)
|
||||
writeFileSync(join(__dirname, "parsed-phrases.json"), JSON.stringify(clean, null, 2), "utf8");
|
||||
|
||||
const withActs = rawPhrases.filter((h) => h.activities.length > 0).length;
|
||||
const emptyActs = rawPhrases.filter((h) => h.activities.length === 0);
|
||||
console.log(
|
||||
`Parsed ${files.length} docx → raw-phrases.json (${withActs} mit Activities, ${emptyActs.length} ohne)`,
|
||||
);
|
||||
console.log(
|
||||
`Aggregat: ${actList.length} Tätigkeiten, ${themeList.length} Themen, ${schoolList.length} Schule`,
|
||||
);
|
||||
if (emptyActs.length) {
|
||||
console.log(
|
||||
"Ohne Activities:",
|
||||
emptyActs.map((h) => h.sourceFile).join(" · "),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user