feat: import DOCX/PDF/scanned templates via DeepSeek recognition

Backend pipeline: - POST /api/templates/import (multipart, max 25 MB) - extract.ts: DOCX→mammoth, PDF→pdf-parse, fallback to OCR via tesseract+poppler-utils (pdftoppm renders pages to PNG, tesseract reads with rus+eng) - deepseek.ts: chat completions client with strict JSON response_format - recognize.ts: structured prompt that produces simplified DocBody (string text), postprocessor wraps text in TipTap-compatible JSON, validates with zod schema - prompt enforces placeholder substitution: {{customer.*}}, {{executor.*}}, {{contract.number}}, {{contract.date}}, {{today}} - error codes: NO_OCR / NO_DEEPSEEK_KEY / UNSUPPORTED_MIME / INVALID_DOC_BODY Dockerfile: apk add tesseract-ocr (+rus +eng data), poppler-utils, imagemagick Frontend: - Templates page: ⤴ Загрузить документ → file picker (.docx,.pdf,.png,.jpg) - doc type selector (contract/invoice/act/upd) - import-banner with spinner shows uploading→analyzing stages - on success navigates to /templates/:id (TemplateEdit) for review Reuses DEEPSEEK_API_KEY pattern from Hall-planer. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-01 11:40:28 +03:00
parent 624d378bb5
commit e768d30fb6
13 changed files with 1114 additions and 7 deletions
@@ -0,0 +1,114 @@
+import { execFile } from 'node:child_process';
+import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { promisify } from 'node:util';
+import mammoth from 'mammoth';
+import * as pdfParseNs from 'pdf-parse';
+
+// pdf-parse в ESM экспортирует default или сам namespace — нормализуем.
+const pdfParse: (buf: Buffer) => Promise<{ text: string; numpages: number }> =
+  // @ts-expect-error namespace shape varies between cjs/esm
+  (pdfParseNs.default ?? pdfParseNs) as never;
+
+const execFileP = promisify(execFile);
+
+export type ExtractedSource = 'docx' | 'pdf-text' | 'pdf-ocr' | 'image-ocr';
+export type Extracted = { text: string; source: ExtractedSource; pageCount?: number };
+
+const OCR_LANGS = 'rus+eng';
+
+/**
+ * Извлекает читаемый текст из произвольного загруженного документа.
+ * - DOCX → mammoth (без OCR);
+ * - PDF с текстовым слоем → pdf-parse;
+ * - PDF-скан → pdftoppm в PNG → tesseract;
+ * - PNG/JPG → tesseract напрямую.
+ *
+ * Если в системе нет tesseract/poppler — для не-DOCX вернётся ошибка с code=NO_OCR.
+ */
+export async function extractText(buf: Buffer, mime: string, filename: string): Promise<Extracted> {
+  const lower = (mime + ' ' + filename).toLowerCase();
+  if (lower.includes('wordprocessingml') || lower.endsWith('.docx')) {
+    const r = await mammoth.extractRawText({ buffer: buf });
+    return { text: cleanup(r.value), source: 'docx' };
+  }
+  if (mime.includes('pdf') || lower.endsWith('.pdf')) {
+    return extractFromPdf(buf);
+  }
+  if (mime.startsWith('image/') || /\.(png|jpe?g|tiff?|bmp)$/i.test(filename)) {
+    const text = await ocrImageBuffer(buf, '.png');
+    return { text: cleanup(text), source: 'image-ocr' };
+  }
+  throw Object.assign(new Error(`Неподдерживаемый формат: ${mime || filename}`), {
+    code: 'UNSUPPORTED_MIME',
+  });
+}
+
+async function extractFromPdf(buf: Buffer): Promise<Extracted> {
+  const parsed = await pdfParse(buf);
+  const text = cleanup(parsed.text);
+  // Эвристика: если текст-слой даёт меньше 60 символов на страницу — это сканированный PDF.
+  const charsPerPage = parsed.numpages > 0 ? text.length / parsed.numpages : text.length;
+  if (text.length > 200 && charsPerPage > 60) {
+    return { text, source: 'pdf-text', pageCount: parsed.numpages };
+  }
+  // OCR pipeline
+  const ocr = await ocrPdfBuffer(buf);
+  return { text: cleanup(ocr), source: 'pdf-ocr', pageCount: parsed.numpages };
+}
+
+async function ocrImageBuffer(buf: Buffer, ext: string): Promise<string> {
+  const dir = await mkdtemp(join(tmpdir(), 'docmgr-ocr-'));
+  try {
+    const inFile = join(dir, `in${ext}`);
+    await writeFile(inFile, buf);
+    const { stdout } = await execFileP('tesseract', [inFile, 'stdout', '-l', OCR_LANGS], {
+      maxBuffer: 32 * 1024 * 1024,
+    });
+    return stdout;
+  } catch (e) {
+    if ((e as { code?: string }).code === 'ENOENT') {
+      throw Object.assign(new Error('Tesseract не установлен в окружении. PDF-сканы обработать не получится.'), {
+        code: 'NO_OCR',
+      });
+    }
+    throw e;
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+}
+
+async function ocrPdfBuffer(buf: Buffer): Promise<string> {
+  const dir = await mkdtemp(join(tmpdir(), 'docmgr-pdf-'));
+  try {
+    const inFile = join(dir, 'in.pdf');
+    await writeFile(inFile, buf);
+    // pdftoppm input.pdf out -png -r 200 → out-1.png, out-2.png ...
+    try {
+      await execFileP('pdftoppm', [inFile, join(dir, 'out'), '-png', '-r', '200']);
+    } catch (e) {
+      if ((e as { code?: string }).code === 'ENOENT') {
+        throw Object.assign(new Error('pdftoppm (poppler-utils) не установлен.'), { code: 'NO_OCR' });
+      }
+      throw e;
+    }
+    const files = (await readdir(dir)).filter((f) => f.startsWith('out-') && f.endsWith('.png')).sort();
+    const parts: string[] = [];
+    for (const f of files) {
+      const pageBuf = await readFile(join(dir, f));
+      parts.push(await ocrImageBuffer(pageBuf, '.png'));
+    }
+    return parts.join('\n\n');
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+}
+
+function cleanup(s: string): string {
+  return s
+    .replace(//g, '') // soft hyphens
+    .replace(/[ \t]+\n/g, '\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}