gute
This commit is contained in:
@@ -0,0 +1,433 @@
|
||||
/**
|
||||
* Beste Suggestions aus allen echten Berichtsheften.
|
||||
* - Extrahiert aus 1./2. Ausbildungsjahr
|
||||
* - Wirft Müll / KI-Kunstsätze raus
|
||||
* - Gewichtet nach Häufigkeit
|
||||
* - Max. ~25% stilgleiche Varianten
|
||||
*/
|
||||
import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs";
|
||||
import { dirname, join } from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
import JSZip from "jszip";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(__dirname, "..", "..");
|
||||
const YEAR1 = join(ROOT, "1. Ausbildungsjahr");
|
||||
const YEAR2 = join(ROOT, "2. Ausbildungsjahr");
|
||||
const OUT_TS = join(__dirname, "..", "src", "data", "suggestions.ts");
|
||||
const OUT_JSON = join(__dirname, "clean-phrases.json");
|
||||
|
||||
function listDocx(dir) {
|
||||
if (!existsSync(dir)) return [];
|
||||
return readdirSync(dir)
|
||||
.filter((f) => f.toLowerCase().endsWith(".docx") && !f.startsWith("~$"))
|
||||
.map((f) => join(dir, f));
|
||||
}
|
||||
|
||||
async function extractTextFromDocx(filePath) {
|
||||
try {
|
||||
const buf = readFileSync(filePath);
|
||||
const zip = await JSZip.loadAsync(buf);
|
||||
const file = zip.file("word/document.xml");
|
||||
if (!file) return "";
|
||||
let xml = await file.async("string");
|
||||
xml = xml.replace(/<\/w:p>/g, "\n");
|
||||
xml = xml.replace(/<[^>]+>/g, "");
|
||||
return xml
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/&#\d+;/g, "");
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function normalize(s) {
|
||||
return s
|
||||
.replace(/\u00a0/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.replace(/^[\s\-•–—]+/, "")
|
||||
.replace(/[.]+$/, (m) => (m.length > 1 ? "" : "."))
|
||||
.trim();
|
||||
}
|
||||
|
||||
function normKey(s) {
|
||||
return normalize(s)
|
||||
.toLowerCase()
|
||||
.replace(/[.…]+$/g, "")
|
||||
.replace(/\s+/g, " ");
|
||||
}
|
||||
|
||||
const JUNK =
|
||||
/^(abteilung|ausbildungsnachweis|für die zeit|name|datum|zweites|erstes|drittes|betriebliche|zuordnung|thema der woche|berufsschule|lfd\.?\s*nr|ausbilderin|nico|baumann|ferien|feiertag|urlaub|schultag|montag|dienstag|mittwoch|donnerstag|freitag)(\b|$)/i;
|
||||
|
||||
const JUNK_ANY =
|
||||
/unterschrift|lfd\.?\s*nr|ausbildungsrahmenplan|…………|\.{5,}|^\d{5,}|43180|76835|88900|31115|60325|69850|zuordnung zum lernziel|tägliche tätigkeiten/i;
|
||||
|
||||
const BAD_ARTIFICIAL =
|
||||
/\bNr\.\s*\d{1,4}\b|Rechnung\s+\d{3,}|Auftrag\s+\d{3,}|Unit\s+\d+|Aufgabe\s+\d+|Übungseinheit\s+\d+|Position\s+\d+|Charge\s+\d+|BWL:\s|BFK:\s|Buchhaltung:\s|Englisch:\s|Deutsch:\s|Controlling Kennzahlen|Optimierung der internen|Kommunikationsprozesse|ZUGFeRD|Proformarechnung für Export/i;
|
||||
|
||||
function isSchoolLine(text) {
|
||||
const t = text.toLowerCase();
|
||||
if (
|
||||
/^(bwl|bfk|englisch|deutsch|excel|word|geschichte|politik|religion|textverarbeitung|projekt|introducing|polypol|kritik am|aufgaben zum|prüfungs|stillarbeit|besprechung der|beschaffung|erstellen einer|diagramme|briefe im|nachhaltigkeit|belastung|das-\/dass|optimale bestell|bestellverfahren|beispielrechnung|lagerkennzahlen|formatvorlage|funktion der|demograf|tarifvertrag|arbeitsschutz|listening|klassenarbeit)/i.test(
|
||||
t,
|
||||
)
|
||||
)
|
||||
return true;
|
||||
if (
|
||||
/\b(bwl|englisch|excel|klassenarbeit|textverarbeitung|listening|polypol|projektarbeit|sozialabgaben|ergonomie)\b/i.test(
|
||||
t,
|
||||
) &&
|
||||
!/\b(kunde|rechnung geschrieben|termine|ablage|reifen|kasse|lagerware)\b/i.test(t)
|
||||
)
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function categorizeBetrieb(text) {
|
||||
const t = text.toLowerCase();
|
||||
if (/rechnung|gutschrift|zahlungs|mahnung|avis|skonto|betrag|bar |ec-/.test(t))
|
||||
return "Rechnungswesen";
|
||||
if (/termin|kunde|kundin|telefon|bewertung|mobilit|angebot/.test(t)) return "Kunden";
|
||||
if (/reifen|lager|lieferung|bestell|ware|artikel|wm |retour|einbuch/.test(t))
|
||||
return "Lager";
|
||||
if (/kasse|kassenbuch/.test(t)) return "Kasse";
|
||||
if (/fahrzeug|abmeld|zulassung|versicherung|unfall|schaden|getriebe|mietfahrzeug|tüv|kilometer/.test(t))
|
||||
return "Fahrzeuge";
|
||||
if (/gefahren|gebracht|abgeholt|feuchtwangen|crailsheim|rothenburg|getränke/.test(t))
|
||||
return "Fahrten";
|
||||
if (/ablage|spül|auftrag|büro|cyber|guardey|kalender|archiv|e-mail|emails/.test(t))
|
||||
return "Organisation";
|
||||
return "Betrieb";
|
||||
}
|
||||
|
||||
function categorizeSchool(text) {
|
||||
const t = text.toLowerCase();
|
||||
if (/bwl|kalkulation|beschaffung|lagerkenn|polypol|bruttoinlands|bestellverfahren|sozialabgaben/.test(t))
|
||||
return "BWL";
|
||||
if (/buchhaltung|umsatzsteuer|rechnung bestandteil/.test(t)) return "Buchhaltung";
|
||||
if (/englisch|introducing|listening|dialogues|briefe im englischen|telefonate/.test(t))
|
||||
return "Englisch";
|
||||
if (/deutsch|das-\/dass|stellenanzeige|rechtschreib/.test(t)) return "Deutsch";
|
||||
if (/excel|word|textverarbeitung|formatvorlage|diagramm|ergonomie/.test(t)) return "IT";
|
||||
if (/geschichte|migration|medien|politik|tarif|arbeitsschutz|demograf/.test(t))
|
||||
return "Gesellschaft";
|
||||
if (/projekt|präsentation|kommunikation|meeting|checkliste|e-mail/.test(t)) return "BFK";
|
||||
return "Berufsschule";
|
||||
}
|
||||
|
||||
function acceptActivity(text) {
|
||||
const t = normalize(text);
|
||||
if (t.length < 8 || t.length > 140) return false;
|
||||
if (JUNK.test(t) || JUNK_ANY.test(t) || BAD_ARTIFICIAL.test(t)) return false;
|
||||
if (!/[a-zäöüß]/i.test(t)) return false;
|
||||
if (/^bis\s+\d{2}\.\d{2}/i.test(t)) return false;
|
||||
// drop pure "Ferien" as activity
|
||||
if (/^ferien$/i.test(t)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function acceptSchool(text) {
|
||||
const t = normalize(text);
|
||||
if (t.length < 4 || t.length > 120) return false;
|
||||
if (JUNK.test(t) || JUNK_ANY.test(t) || BAD_ARTIFICIAL.test(t)) return false;
|
||||
if (/^ferien$/i.test(t)) return false;
|
||||
if (!/[a-zäöüß]/i.test(t)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function acceptTheme(text) {
|
||||
const t = normalize(text).replace(/,$/, "");
|
||||
if (t.length < 3 || t.length > 55) return false;
|
||||
if (JUNK.test(t) || JUNK_ANY.test(t) || /^\d/.test(t) || t === "-") return false;
|
||||
if (/^(planung|vorschriften)$/i.test(t) && t.length < 5) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function parseSections(raw) {
|
||||
const lines = raw
|
||||
.split(/\n+/)
|
||||
.map((l) => normalize(l.replace(/^\d{5,}/, "")))
|
||||
.filter(Boolean);
|
||||
|
||||
const activities = [];
|
||||
const themes = [];
|
||||
const school = [];
|
||||
let section = "act";
|
||||
|
||||
for (const line of lines) {
|
||||
if (/thema der woche/i.test(line)) {
|
||||
section = "theme";
|
||||
continue;
|
||||
}
|
||||
if (/berufsschule/i.test(line)) {
|
||||
section = "school";
|
||||
continue;
|
||||
}
|
||||
if (/zuordnung|ausbildungsrahmenplan|betriebliche tätigkeiten/i.test(line)) {
|
||||
section = "fw";
|
||||
continue;
|
||||
}
|
||||
if (/ausbildungsnachweis nr|für die zeit vom/i.test(line)) {
|
||||
section = "act";
|
||||
continue;
|
||||
}
|
||||
if (section === "act" && acceptActivity(line) && !isSchoolLine(line)) {
|
||||
activities.push(normalize(line.replace(/\.$/, "")));
|
||||
} else if (section === "theme" && acceptTheme(line)) {
|
||||
themes.push(normalize(line.replace(/,$/, "")));
|
||||
} else if (section === "school" && acceptSchool(line)) {
|
||||
school.push(normalize(line));
|
||||
}
|
||||
}
|
||||
return { activities, themes, school };
|
||||
}
|
||||
|
||||
function bump(map, text, file) {
|
||||
const key = normKey(text);
|
||||
if (!key) return;
|
||||
const prev = map.get(key) || { text, count: 0, sources: new Set() };
|
||||
// prefer shorter/cleaner display text when similar
|
||||
if (text.length < prev.text.length) prev.text = text;
|
||||
prev.count += 1;
|
||||
prev.sources.add(file);
|
||||
map.set(key, prev);
|
||||
}
|
||||
|
||||
/** Nur stilgleiche Varianten aus echten Mustern */
|
||||
const VARIANT_RULES = [
|
||||
{ ifIncludes: "termine vereinbart", add: ["Termine telefonisch vereinbart", "Termine mit Kunden vereinbart"] },
|
||||
{ ifIncludes: "ablage einsortiert", add: ["Ablage sortiert", "Ablage organisiert"] },
|
||||
{ ifIncludes: "ablage verwaltet", add: ["Ablage archiviert"] },
|
||||
{ ifIncludes: "rechnungen geschrieben", add: ["Rechnungen erstellt", "Rechnungen fertiggestellt"] },
|
||||
{ ifIncludes: "rechnungen erstellt", add: ["Rechnungen geschrieben"] },
|
||||
{ ifIncludes: "getränke geholt", add: ["Getränke für die Belegschaft geholt", "Getränke besorgt"] },
|
||||
{ ifIncludes: "kasse gezählt", add: ["Kassenbuch geführt", "Kassenbuch geführt und die Kasse gezählt"] },
|
||||
{ ifIncludes: "kassenbuch geführt", add: ["Kasse gezählt"] },
|
||||
{ ifIncludes: "reifen bestellt", add: ["Reifen für Kunden bestellt"] },
|
||||
{ ifIncludes: "aufträge vorbereitet", add: ["Aufträge für die kommende Woche vorbereitet", "Aufträge bearbeitet"] },
|
||||
{ ifIncludes: "kunden per telefon betreut", add: ["Kundentelefonate durchgeführt zwecks Termine", "Telefonisch Termine vereinbart"] },
|
||||
{ ifIncludes: "spülmaschine", add: ["Spülmaschine ausgeräumt", "Spülmaschine ein- und ausgeräumt"] },
|
||||
{ ifIncludes: "mobilitätsgarant", add: ["Mobilitätsgarantien abgeschlossen", "Bosch-Mobilitätsgarantie abgeschlossen"] },
|
||||
{ ifIncludes: "bewertungen gesammelt", add: ["Bewertungen für autoservice.com gesammelt"] },
|
||||
{ ifIncludes: "lieferung", add: ["Lieferung angenommen und vorsortiert", "Lieferung überprüft, vorsortiert und eingebucht"] },
|
||||
];
|
||||
|
||||
async function main() {
|
||||
const files = [...listDocx(YEAR1), ...listDocx(YEAR2)];
|
||||
console.log(`Parsing ${files.length} docx…`);
|
||||
|
||||
const actMap = new Map();
|
||||
const themeMap = new Map();
|
||||
const schoolMap = new Map();
|
||||
|
||||
for (const file of files) {
|
||||
const name = file.split(/[/\\]/).pop();
|
||||
const raw = await extractTextFromDocx(file);
|
||||
if (!raw) {
|
||||
console.warn("empty:", name);
|
||||
continue;
|
||||
}
|
||||
const { activities, themes, school } = parseSections(raw);
|
||||
for (const a of activities) bump(actMap, a, name);
|
||||
for (const t of themes) bump(themeMap, t, name);
|
||||
for (const s of school) bump(schoolMap, s, name);
|
||||
}
|
||||
|
||||
// Drop very weak one-offs that look like noise (optional keep count>=1 but filter quality)
|
||||
let activities = [...actMap.values()]
|
||||
.filter((a) => acceptActivity(a.text) && !isSchoolLine(a.text))
|
||||
.filter((a) => !BAD_ARTIFICIAL.test(a.text))
|
||||
.sort((a, b) => b.count - a.count || a.text.localeCompare(b.text, "de"));
|
||||
|
||||
let school = [...schoolMap.values()]
|
||||
.filter((s) => acceptSchool(s.text))
|
||||
.filter((s) => isSchoolLine(s.text) || s.count >= 1)
|
||||
.filter((s) => !BAD_ARTIFICIAL.test(s.text))
|
||||
.sort((a, b) => b.count - a.count || a.text.localeCompare(b.text, "de"));
|
||||
|
||||
// Prefer school lines that look like school; demote misclassified
|
||||
school = school.filter((s) => {
|
||||
// if it looks strongly betrieb, skip
|
||||
if (/termine vereinbart|ablage |reifen bestellt|kasse gezählt|getränke/i.test(s.text))
|
||||
return false;
|
||||
return true;
|
||||
});
|
||||
|
||||
const themes = [...themeMap.values()]
|
||||
.filter((t) => acceptTheme(t.text))
|
||||
.sort((a, b) => b.count - a.count)
|
||||
.map((t) => t.text);
|
||||
|
||||
// Boost multi-year / high-frequency
|
||||
for (const a of activities) {
|
||||
a.weight = a.count + (a.sources.size > 1 ? 1 : 0);
|
||||
if (a.count >= 4) a.weight += 2;
|
||||
if (a.count >= 8) a.weight += 3;
|
||||
}
|
||||
for (const s of school) {
|
||||
s.weight = s.count + (s.sources.size > 1 ? 1 : 0);
|
||||
if (s.count >= 3) s.weight += 2;
|
||||
}
|
||||
|
||||
// Sparse variants from real patterns only
|
||||
const originalKeys = new Set(activities.map((a) => normKey(a.text)));
|
||||
const variants = [];
|
||||
for (const a of activities) {
|
||||
if (a.count < 2) continue;
|
||||
for (const rule of VARIANT_RULES) {
|
||||
if (!normKey(a.text).includes(rule.ifIncludes)) continue;
|
||||
for (const v of rule.add) {
|
||||
const k = normKey(v);
|
||||
if (originalKeys.has(k)) continue;
|
||||
if (variants.some((x) => normKey(x.text) === k)) continue;
|
||||
if (!acceptActivity(v)) continue;
|
||||
variants.push({
|
||||
text: v,
|
||||
count: 0,
|
||||
weight: Math.max(1, a.weight - 2),
|
||||
sources: new Set(["variant"]),
|
||||
variant: true,
|
||||
});
|
||||
originalKeys.add(k);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const maxVariants = Math.floor(activities.length * 0.28);
|
||||
const keptVariants = variants.slice(0, maxVariants);
|
||||
|
||||
const allBetrieb = [...activities, ...keptVariants];
|
||||
const suggestions = [];
|
||||
|
||||
for (const a of allBetrieb) {
|
||||
suggestions.push({
|
||||
text: a.text,
|
||||
type: "betrieb",
|
||||
category: categorizeBetrieb(a.text),
|
||||
weight: Math.max(1, a.weight || 1),
|
||||
original: !a.variant,
|
||||
});
|
||||
}
|
||||
for (const s of school) {
|
||||
suggestions.push({
|
||||
text: s.text,
|
||||
type: "schule",
|
||||
category: categorizeSchool(s.text),
|
||||
weight: Math.max(1, s.weight || 1),
|
||||
original: true,
|
||||
});
|
||||
}
|
||||
|
||||
// Final dedupe
|
||||
const seen = new Set();
|
||||
const unique = [];
|
||||
for (const s of suggestions.sort((a, b) => b.weight - a.weight)) {
|
||||
const k = normKey(s.text) + "|" + s.type;
|
||||
if (seen.has(k)) continue;
|
||||
seen.add(k);
|
||||
unique.push(s);
|
||||
}
|
||||
|
||||
const withIds = unique.map((s, i) => ({
|
||||
id: i + 1,
|
||||
text: s.text,
|
||||
type: s.type,
|
||||
category: s.category,
|
||||
weight: s.weight,
|
||||
}));
|
||||
|
||||
const betriebN = withIds.filter((s) => s.type === "betrieb").length;
|
||||
const schoolN = withIds.filter((s) => s.type === "schule").length;
|
||||
const originals = unique.filter((s) => s.type === "betrieb" && s.original).length;
|
||||
const originalShare = betriebN ? originals / betriebN : 0;
|
||||
|
||||
writeFileSync(
|
||||
OUT_JSON,
|
||||
JSON.stringify(
|
||||
{
|
||||
activities: activities.map((a) => ({
|
||||
text: a.text,
|
||||
count: a.count,
|
||||
weight: a.weight,
|
||||
})),
|
||||
school: school.map((s) => ({ text: s.text, count: s.count, weight: s.weight })),
|
||||
themes,
|
||||
stats: {
|
||||
files: files.length,
|
||||
betrieb: betriebN,
|
||||
school: schoolN,
|
||||
variants: keptVariants.length,
|
||||
originalShareBetrieb: Number(originalShare.toFixed(3)),
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const themeList = [...new Set(themes)].filter(Boolean);
|
||||
|
||||
const ts = `/* AUTO-GENERATED – echte Berichtshefte · ${withIds.length} Vorschläge · ${new Date().toISOString().slice(0, 10)} */
|
||||
export type SuggestionType = "betrieb" | "schule";
|
||||
|
||||
export type Suggestion = {
|
||||
id: number;
|
||||
text: string;
|
||||
type: SuggestionType;
|
||||
category: string;
|
||||
/** Häufigkeit/Qualität aus den Originalheften */
|
||||
weight: number;
|
||||
};
|
||||
|
||||
export const SUGGESTIONS: Suggestion[] = ${JSON.stringify(withIds, null, 2)};
|
||||
|
||||
export const WEEK_THEMES: string[] = ${JSON.stringify(themeList, null, 2)};
|
||||
|
||||
export const CATEGORIES = [...new Set(SUGGESTIONS.map((s) => s.category))].sort();
|
||||
|
||||
export const SUGGESTION_STATS = ${JSON.stringify(
|
||||
{
|
||||
total: withIds.length,
|
||||
betrieb: betriebN,
|
||||
school: schoolN,
|
||||
variants: keptVariants.length,
|
||||
originalShareBetrieb: Number(originalShare.toFixed(3)),
|
||||
sourceFiles: files.length,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)} as const;
|
||||
`;
|
||||
|
||||
writeFileSync(OUT_TS, ts, "utf8");
|
||||
|
||||
console.log("Wrote", OUT_TS);
|
||||
console.log(
|
||||
`total=${withIds.length} betrieb=${betriebN} schule=${schoolN} variants=${keptVariants.length} originalShare=${(originalShare * 100).toFixed(1)}%`,
|
||||
);
|
||||
console.log("Top 15 betrieb:");
|
||||
withIds
|
||||
.filter((s) => s.type === "betrieb")
|
||||
.slice(0, 15)
|
||||
.forEach((s) => console.log(` w${s.weight} ${s.text}`));
|
||||
console.log("Top 10 schule:");
|
||||
withIds
|
||||
.filter((s) => s.type === "schule")
|
||||
.slice(0, 10)
|
||||
.forEach((s) => console.log(` w${s.weight} ${s.text}`));
|
||||
|
||||
if (originalShare < 0.7) {
|
||||
console.warn("WARN: original share < 70%");
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user