outlook-mcp/utils/bodyParser.js

'use strict';

/**
 * bodyParser.js
 * Heuristic cleaner for email body text returned by Microsoft Graph API.
 * Strips boilerplate noise without touching actual message content.
 * All rules are based on observed real-world patterns from this mailbox.
 */

// ---------------------------------------------------------------------------
// Pattern library — ordered by application sequence
// ---------------------------------------------------------------------------

// 1. HTML entities produced by Graph API's text/plain conversion
const HTML_ENTITIES = [
  [/ /gi, ' '],
  [/&/gi, '&'],
  [/>/gi, '>'],
  [/&lt;/gi, '<'],
  [/&quot;/gi, '"'],
  [/&#39;/gi, "'"],
];

// 2. External email caution banners — appear at start of body or inline
// Covers known Proofpoint sentinel tokens and human-readable CAUTION blocks.
const CAUTION_BANNERS = [
  // Proofpoint Essentials sentinel style: NkdkJdXPPEBannerStart ... NkdkJdXPPEBannerEnd
  /[A-Za-z0-9]{8,}BannerStart[\s\S]*?[A-Za-z0-9]{8,}BannerEnd/g,
  // Full two-sentence form with bold/marker text
  /CAUTION[\s\-–]*EXTERNAL\s+EMAIL\s*:.*?(?:content is safe\.?)/gis,
  // Short form
  /CAUTION\s*:?\s*This email originated from outside.*?(?:content is safe\.?)/gis,
  // Generic external sender marker lines
  /External Sender[\s\-–]*:?[\s\S]*?This message came from outside[\s\S]*?Learn More/gi,
];

// 3. Legal boilerplate blocks — DISCLAIMER and CONFIDENTIALITY NOTICE
// These repeat on every reply in a chain. Match greedy to end of block.
const LEGAL_BLOCKS = [
  // DISCLAIMER block (YMCA, others)
  /DISCLAIMER\s*:.*?(?=DISCLAIMER\s*:|CONFIDENTIALITY\s*NOTICE\s*:|$)/gis,
  // CONFIDENTIALITY NOTICE block (Prime HHCC, Farber & Lindley, others)
  /CONFIDENTIALITY\s*NOTICE\s*:.*?(?=DISCLAIMER\s*:|CONFIDENTIALITY\s*NOTICE\s*:|$)/gis,
  // Generic confidentiality footer
  /This message \(including any attachments\) may contain confidential,[\s\S]*?scanned for spam and viruses by Proofpoint Essentials[\s\S]*?$/gim,
];

// 3b. Non-delivery report and automated response noise
const AUTO_NOISE_BLOCKS = [
  // Microsoft NDR "Original Message Details" and onward
  /Original Message Details[\s\S]*$/gi,
  // Generic delivery failure explanation blocks
  /Action Required[\s\S]*How to Fix It[\s\S]*$/gi,
  // Message hops / headers dump inside NDRs
  /^Message Hops[\s\S]*$/gim,
  // Auto-reply / out-of-office markers
  /^\s*Auto-?generated by.*$/gim,
];

// 4. Signature block delimiters — everything from a recognized sig opener onward
// Only applied when stripping signatures is explicitly requested (see exports).
const SIGNATURE_DELIMITERS = [
  // Standard triple-dash separator
  /^---\s*$/m,
  // deRenzy signature pattern: name on one line, then IT@/Seton@ email lines
  /^Seton Carmichael\s*\n(?:IT@|Seton@)/m,
  /^Richard Priest\s*\n/m,
];

// ---------------------------------------------------------------------------
// HTML → plain text conversion
// ---------------------------------------------------------------------------

/**
 * Convert HTML email body to plain text, preserving line structure.
 * Two-pass: structural tags → newlines first, then strip remaining tags.
 * @param {string} html
 * @returns {string}
 */
function htmlToText(html) {
  let text = html;
  // Mark Outlook signature boundaries before conversion.
  // Outlook wraps signatures in id="Signature" (or id="x_Signature" on nested messages).
  // We replace the opening tag with a sentinel [SIG] so callers can decide whether to
  // keep or strip the signature after HTML→text conversion.
  text = text.replace(/<div[^>]*\bid=["'][^"']*[Ss]ignature["'][^>]*>/gi, '\n[SIG]\n');
  // Block-level tags that should become newlines
  text = text.replace(/<br\s*\/?>/gi, '\n');
  text = text.replace(/<\/(?:div|p|tr|li|blockquote|h[1-6])>/gi, '\n');
  // Strip all remaining tags
  text = text.replace(/<[^>]+>/g, '');
  return text;
}

// ---------------------------------------------------------------------------
// Core cleaner
// ---------------------------------------------------------------------------

/**
 * Clean a single email body string. Handles both HTML and plain text input.
 * @param {string} body - Raw body text or HTML from Graph API
 * @param {object} [opts]
 * @param {boolean} [opts.stripSignature=false] - Also strip trailing signature block
 * @returns {string} Cleaned plain text
 */
function cleanBody(body, opts = {}) {
  if (!body || typeof body !== 'string') return body || '';

  let text = body;

  // Step 1: Convert HTML to plain text if needed
  if (/<html[\s>]/i.test(text) || /<body[\s>]/i.test(text) || /<div[\s>]/i.test(text)) {
    text = htmlToText(text);
  }

  // Step 2: Normalize HTML entities (may remain after tag stripping)
  for (const [pattern, replacement] of HTML_ENTITIES) {
    text = text.replace(pattern, replacement);
  }

  // Step 2: Strip external caution banners
  for (const pattern of CAUTION_BANNERS) {
    text = text.replace(pattern, '');
  }

  // Step 3: Strip legal boilerplate blocks
  for (const pattern of LEGAL_BLOCKS) {
    text = text.replace(pattern, '');
  }

  // Step 3b: Strip automated noise (NDRs, hops, auto-replies)
  for (const pattern of AUTO_NOISE_BLOCKS) {
    text = text.replace(pattern, '');
  }

  // Step 3c: Remove all lines that look like raw SMTP/X-MS headers inside NDRs
  // These are the long colon-heavy strings dumped by Microsoft delivery failures.
  text = text.replace(/^([A-Z][a-zA-Z0-9\-]*|X-[A-Za-z\-]+):.*$/gm, '');

  // Step 4: Handle [SIG] sentinel (injected by htmlToText for id="Signature" divs)
  if (opts.stripSignature) {
    // Remove from [SIG] marker onward
    text = text.replace(/\n?\[SIG\][\s\S]*/g, '');
  } else {
    // Keep signature content, just remove the marker itself
    text = text.replace(/\[SIG\]\n?/g, '');
  }

  // Step 4b: Text-based signature delimiters (fallback for non-HTML emails)
  if (opts.stripSignature) {
    for (const delimiter of SIGNATURE_DELIMITERS) {
      const match = text.search(delimiter);
      if (match !== -1) {
        text = text.slice(0, match);
        break;
      }
    }
  }

  // Step 5: Collapse runs of 3+ blank lines down to 2, trim edges
  text = text.replace(/\n{3,}/g, '\n\n');
  text = text.trim();

  return text;
}

/**
 * Clean the body field of a Graph API email message object in-place.
 * Returns the same object with body.content cleaned.
 * Also cleans bodyPreview if present.
 * @param {object} message - Graph API message object
 * @param {object} [opts] - Same options as cleanBody
 * @returns {object} The same message object, mutated
 */
function cleanMessage(message, opts = {}) {
  if (!message) return message;

  if (message.body && message.body.content) {
    message.body.content = cleanBody(message.body.content, opts);
  }
  // bodyPreview is a short excerpt — just do entity decode and trim
  if (message.bodyPreview) {
    message.bodyPreview = cleanBody(message.bodyPreview, { stripSignature: false });
  }

  return message;
}

/**
 * Clean an array of message objects.
 * @param {object[]} messages
 * @param {object} [opts]
 * @returns {object[]}
 */
function cleanMessages(messages, opts = {}) {
  if (!Array.isArray(messages)) return messages;
  return messages.map(m => cleanMessage(m, opts));
}

module.exports = { cleanBody, cleanMessage, cleanMessages };