Hidden JSON Errors That Silently Break Your App — Duplicate Keys, BOM, Precision Loss, and More

// Full duplicate key validator — works before parsing
function validateNoDuplicateKeys(jsonText) {
  const errors = [];
  const keyRe = /"((?:[^"\\]|\\.)*)"s*:/g;
  const occurrences = new Map();
  let match;

  while ((match = keyRe.exec(jsonText)) !== null) {
    const key = match[1];
    const line = (jsonText.slice(0, match.index).match(/\n/g) || []).length + 1;
    if (!occurrences.has(key)) {
      occurrences.set(key, []);
    }
    occurrences.get(key).push(line);
  }

  for (const [key, lines] of occurrences) {
    if (lines.length > 1) {
      errors.push({
        key,
        lines,
        message: `Key "${key}" appears ${lines.length}× on lines: ${lines.join(', ')}. All values except the last will be silently discarded.`,
      });
    }
  }

  return errors;
}

// Usage:
const dupes = validateNoDuplicateKeys(jsonString);
if (dupes.length > 0) {
  console.warn('Duplicate keys detected:', dupes);
  // Handle: throw, warn, or show to the user
}

// Python alternative — detect with a custom object_pairs_hook:
import json

def detect_duplicates(pairs):
    keys = [k for k, _ in pairs]
    dupes = {k for k in keys if keys.count(k) > 1}
    if dupes:
        raise ValueError(f"Duplicate JSON keys found: {dupes}")
    return dict(pairs)

json.loads(json_string, object_pairs_hook=detect_duplicates)
# → raises ValueError if any key appears more than once

// Robust JSON file loader — handles BOM, encoding, empty files
async function loadJsonFile(filePath) {
  const { readFile } = await import('fs/promises');

  let raw;
  try {
    raw = await readFile(filePath, 'utf8');
  } catch (e) {
    throw new Error(`Cannot read file ${filePath}: ${e.message}`);
  }

  if (!raw.trim()) {
    throw new Error(`File ${filePath} is empty`);
  }

  // Detect and strip BOM
  const hasBom = raw.charCodeAt(0) === 0xFEFF;
  if (hasBom) {
    console.warn(`Warning: ${filePath} has a UTF-8 BOM — stripping before parse`);
    raw = raw.slice(1);
  }

  try {
    return JSON.parse(raw);
  } catch (e) {
    throw new Error(`JSON parse error in ${filePath}: ${e.message}`);
  }
}

// Detecting BOM in HTTP response:
async function fetchJsonRobust(url) {
  const response = await fetch(url);
  if (!response.ok) throw new Error(`HTTP ${response.status}`);
  const text = await response.text();
  const clean = text.charCodeAt(0) === 0xFEFF ? text.slice(1) : text;
  return JSON.parse(clean);
}

// Detect if a JSON number string will lose precision after parsing
function detectPrecisionLoss(jsonText) {
  const numberRe = /:s*(-?d{16,}(?:.d+)?)/g;
  const warnings = [];
  let match;

  while ((match = numberRe.exec(jsonText)) !== null) {
    const numStr = match[1];
    const parsed = parseFloat(numStr);
    const reparsed = String(parsed);

    if (reparsed !== numStr && !numStr.includes('.')) {
      const line = (jsonText.slice(0, match.index).match(/\n/g) || []).length + 1;
      warnings.push({
        original: numStr,
        parsed: reparsed,
        line,
        message: `Number ${numStr} will parse as ${reparsed} — precision loss!`,
      });
    }
  }

  return warnings;
}

// Check API responses before using them:
const json = await response.text();
const precisionWarnings = detectPrecisionLoss(json);
if (precisionWarnings.length > 0) {
  console.warn('Precision loss detected in JSON:', precisionWarnings);
}

// Node.js: use a streaming parser that supports BigInt
// npm install json-bigint
import JSONBig from 'json-bigint';
const parsed = JSONBig.parse(json);  // large integers become BigInt automatically

// Detect control characters in JSON string values
function findControlChars(jsonText) {
  const controlRe = /[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g;
  const issues = [];
  let match;

  while ((match = controlRe.exec(jsonText)) !== null) {
    const charCode = match[0].charCodeAt(0);
    const line = (jsonText.slice(0, match.index).match(/\n/g) || []).length + 1;
    issues.push({
      char: `\\u${charCode.toString(16).padStart(4, '0')}`,
      code: charCode,
      line,
      context: jsonText.slice(Math.max(0, match.index - 20), match.index + 20),
    });
  }

  return issues;
}

// Sanitize string values after parsing
function sanitizeControlChars(value) {
  if (typeof value === 'string') {
    // Remove null bytes (most dangerous), escape other control chars
    return value
      .replace(/\x00/g, '')              // strip null bytes
      .replace(/[\x01-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ''); // strip other controls
  }
  if (Array.isArray(value)) return value.map(sanitizeControlChars);
  if (typeof value === 'object' && value !== null) {
    return Object.fromEntries(
      Object.entries(value).map(([k, v]) => [k, sanitizeControlChars(v)])
    );
  }
  return value;
}

// Full safe parse with control character sanitization:
function safeParseJson(jsonText) {
  const controls = findControlChars(jsonText);
  if (controls.length > 0) {
    console.warn('Control characters in JSON:', controls);
  }
  const parsed = JSON.parse(jsonText); // parse even with control chars
  return sanitizeControlChars(parsed); // then sanitize
}

// For user input — sanitize BEFORE stringifying:
function sanitizeForJson(userInput) {
  if (typeof userInput !== 'string') return userInput;
  return userInput
    .replace(/\x00/g, '')
    .replace(/[\x01-\x1F\x7F]/g, ' '); // replace with space instead of removing
}

const safePayload = JSON.stringify({
  name: sanitizeForJson(req.body.name),
  bio: sanitizeForJson(req.body.bio),
});

// Measure maximum nesting depth before parsing
function measureNestingDepth(jsonText) {
  let depth = 0, maxDepth = 0;
  let inString = false;

  for (let i = 0; i < jsonText.length; i++) {
    const char = jsonText[i];
    if (char === '\\' && inString) { i++; continue; }
    if (char === '"') { inString = !inString; continue; }
    if (inString) continue;

    if (char === '{' || char === '[') { depth++; maxDepth = Math.max(maxDepth, depth); }
    else if (char === '}' || char === ']') depth--;
  }

  return maxDepth;
}

// Safe parse with depth limit
const MAX_NESTING_DEPTH = 50; // reasonable limit for API data

function safeParseWithDepthLimit(jsonText, maxDepth = MAX_NESTING_DEPTH) {
  const depth = measureNestingDepth(jsonText);
  if (depth > maxDepth) {
    throw new Error(`JSON nesting depth ${depth} exceeds limit of ${maxDepth}. Possible JSON bomb.`);
  }
  return JSON.parse(jsonText);
}

// In an Express API — validate user-submitted JSON:
app.use(express.json({
  limit: '10mb',          // size limit
  // Add depth checking in a middleware:
}));

app.use((req, res, next) => {
  if (req.headers['content-type']?.includes('application/json')) {
    try {
      const depth = measureNestingDepth(req.body_raw || '');
      if (depth > MAX_NESTING_DEPTH) {
        return res.status(400).json({ error: 'JSON nesting too deep' });
      }
    } catch {}
  }
  next();
});

// Detect encoding issues after parse — look for replacement characters
function validateEncoding(parsedObject) {
  const issues = [];

  function scan(value, path) {
    if (typeof value === 'string') {
      if (value.includes('\uFFFD')) {
        issues.push({ path, value: value.slice(0, 100), issue: 'Contains replacement character (encoding error)' });
      }
      // Check for HTML entities that should have been decoded
      if (/&(?:amp|lt|gt|quot|apos);/.test(value)) {
        issues.push({ path, issue: 'Contains HTML entities — may be double-encoded' });
      }
    } else if (Array.isArray(value)) {
      value.forEach((item, i) => scan(item, `${path}[${i}]`));
    } else if (typeof value === 'object' && value !== null) {
      Object.entries(value).forEach(([k, v]) => scan(v, path ? `${path}.${k}` : k));
    }
  }

  scan(parsedObject, '');
  return issues;
}

// Node.js — force UTF-8 when reading files
import { readFileSync } from 'fs';
const content = readFileSync('data.json', { encoding: 'utf8' });

// If the file might be in a different encoding, use iconv-lite:
import iconv from 'iconv-lite';
const raw = readFileSync('legacy-data.json'); // Buffer
const utf8 = iconv.decode(raw, 'win1252');    // convert from Windows-1252 to UTF-8
const data = JSON.parse(utf8);