JSON Data Redaction

JSON has become the universal format for data exchange—APIs return JSON, applications store configuration in JSON, analytics platforms export JSON, and modern databases use JSON documents. This prevalence means PII inevitably flows through JSON data structures: user profiles in API responses, customer records in data exports, personal information in log files, and sensitive details in configuration. Protecting this data requires understanding both JSON's structural nature and the patterns of PII within it.

Our JSON processing combines structural awareness with content analysis. We parse JSON hierarchically, understanding the relationship between fields and their values. Field names provide context—a field named "email" likely contains an email address. Value patterns confirm PII presence—a string matching email format is redacted regardless of field name. This dual approach ensures comprehensive protection for JSON data regardless of schema design or naming conventions.

JSON Structure Analysis

Understanding JSON structure enables intelligent redaction:

// Input JSON
{
  "user": {
    "id": 12345,
    "name": "John Smith",
    "email": "[email protected]",
    "phone": "+1-555-123-4567",
    "address": {
      "street": "123 Main St",
      "city": "New York",
      "state": "NY",
      "zip": "10001"
    },
    "orders": [
      {
        "id": "ORD-001",
        "items": ["Widget A", "Widget B"],
        "notes": "Call customer at 555-987-6543"
      }
    ]
  }
}

// Redacted output
{
  "user": {
    "id": 12345,
    "name": "[NAME]",
    "email": "[EMAIL]",
    "phone": "[PHONE]",
    "address": {
      "street": "[ADDRESS]",
      "city": "[CITY]",
      "state": "NY",
      "zip": "[ZIP]"
    },
    "orders": [
      {
        "id": "ORD-001",
        "items": ["Widget A", "Widget B"],
        "notes": "Call customer at [PHONE]"
      }
    ]
  }
}

Field Name Detection

Field names indicate likely PII content:

// High-confidence PII field names
{
  "email": "...",
  "emailAddress": "...",
  "email_address": "...",
  "phone": "...",
  "phoneNumber": "...",
  "phone_number": "...",
  "mobile": "...",
  "ssn": "...",
  "socialSecurityNumber": "...",
  "social_security_number": "...",
  "name": "...",
  "firstName": "...",
  "lastName": "...",
  "fullName": "...",
  "address": "...",
  "streetAddress": "...",
  "creditCard": "...",
  "dob": "...",
  "dateOfBirth": "...",
  "birthDate": "..."
}

// Field name matching is case-insensitive
// Supports camelCase, snake_case, and variations
// Configured aliases for industry-specific terms

Configurable Field Lists:

{
  "fieldDetection": {
    "alwaysRedact": [
      "$.user.ssn",
      "$.customer.*.credit_card",
      "$.employees[*].salary"
    ],
    "neverRedact": [
      "$.metadata.created_by",
      "$.system.service_account"
    ],
    "patterns": [
      "*_email",
      "*_phone",
      "*.pii.*"
    ]
  }
}

Value Pattern Detection

Detect PII by value patterns regardless of field name:

// PII detected in any string field
{
  "description": "Contact [email protected] for details",
  "notes": "Customer SSN: 123-45-6789",
  "comments": "Call back at (555) 123-4567",
  "reference": "Credit card ending in 4242",
  "blob": "John Smith, 123 Main St, New York NY 10001"
}

// All PII detected and redacted
{
  "description": "Contact [EMAIL] for details",
  "notes": "Customer SSN: [SSN]",
  "comments": "Call back at [PHONE]",
  "reference": "Credit card ending in [CREDIT_CARD]",
  "blob": "[NAME], [ADDRESS], [CITY] [STATE] [ZIP]"
}

Nested Object Processing

Handle complex nested structures:

// Deeply nested data
{
  "company": {
    "departments": [
      {
        "name": "Engineering",
        "teams": [
          {
            "name": "Backend",
            "members": [
              {
                "employee": {
                  "personal": {
                    "name": "John Smith",
                    "contact": {
                      "email": "[email protected]",
                      "phone": "555-1234"
                    }
                  }
                }
              }
            ]
          }
        ]
      }
    ]
  }
}

// Field paths tracked:
// $.company.departments[0].teams[0].members[0].employee.personal.name
// $.company.departments[0].teams[0].members[0].employee.personal.contact.email
// $.company.departments[0].teams[0].members[0].employee.personal.contact.phone

Array Handling

Process arrays of values and objects:

// Array of strings
{
  "emails": ["[email protected]", "[email protected]", "[email protected]"]
}
// → ["[EMAIL]", "[EMAIL]", "[EMAIL]"]

// Array of objects
{
  "contacts": [
    {"name": "John", "email": "[email protected]"},
    {"name": "Jane", "email": "[email protected]"}
  ]
}
// Each object processed independently

// Mixed arrays
{
  "data": [
    "[email protected]",
    {"type": "phone", "value": "555-1234"},
    123,
    null
  ]
}
// Each element handled by type

Streaming and Large Files

Efficiently process large JSON data:

Standard JSON Streaming:

// For large single JSON objects/arrays
// Uses streaming parser (not full DOM load)

const stream = fs.createReadStream('large-data.json');
const parser = new StreamingJsonParser();

parser.on('value', async (path, value) => {
  if (shouldRedact(path, value)) {
    const redacted = await redact(value);
    outputStream.write(path, redacted);
  } else {
    outputStream.write(path, value);
  }
});

stream.pipe(parser);

NDJSON Processing:

// Newline-delimited JSON (JSON Lines)
// One JSON object per line

// Input: logs.ndjson
{"timestamp": "2024-01-15", "user": "[email protected]", "action": "login"}
{"timestamp": "2024-01-15", "user": "[email protected]", "action": "logout"}
{"timestamp": "2024-01-15", "user": "[email protected]", "action": "purchase"}

// Processing
const readline = require('readline');
const rl = readline.createInterface({
  input: fs.createReadStream('logs.ndjson')
});

for await (const line of rl) {
  const obj = JSON.parse(line);
  const redacted = await redactJson(obj);
  outputStream.write(JSON.stringify(redacted) + '\n');
}

Batch API for NDJSON:

POST /v1/redact/ndjson
Content-Type: application/x-ndjson

{"user": {"email": "[email protected]"}}
{"user": {"email": "[email protected]"}}
{"user": {"email": "[email protected]"}}

Response (streaming):
{"user": {"email": "[EMAIL]"}}
{"user": {"email": "[EMAIL]"}}
{"user": {"email": "[EMAIL]"}}

Data Type Preservation

Maintain JSON data types after redaction:

// Input with various types
{
  "string_field": "[email protected]",
  "number_field": 123456789,
  "boolean_field": true,
  "null_field": null,
  "array_field": ["a", "b", "c"],
  "object_field": {"key": "value"}
}

// Redaction options for type handling
{
  "typePreservation": {
    "strings": "redact_inline",     // "[EMAIL]"
    "numbers": "null",              // null (can't redact inline)
    "preserveStructure": true       // keep object/array structure
  }
}

// Output
{
  "string_field": "[EMAIL]",
  "number_field": null,             // Number couldn't be preserved
  "boolean_field": true,            // Unchanged (not PII)
  "null_field": null,               // Unchanged
  "array_field": ["a", "b", "c"],   // Unchanged (no PII)
  "object_field": {"key": "value"}  // Unchanged (no PII)
}

Schema-Based Configuration

Configure redaction based on known schemas:

// JSON Schema with PII annotations
{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "type": "object",
  "properties": {
    "user": {
      "type": "object",
      "properties": {
        "id": {
          "type": "integer",
          "x-pii": false
        },
        "email": {
          "type": "string",
          "format": "email",
          "x-pii": true,
          "x-pii-type": "email"
        },
        "metadata": {
          "type": "object",
          "x-pii-scan": false  // Skip scanning this subtree
        }
      }
    }
  }
}

// Use schema for precise field-level control
{
  "schema": "https://example.com/schemas/user.json",
  "useSchemaAnnotations": true
}

JSONPath Expressions

Use JSONPath for precise field selection:

// JSONPath examples
{
  "redactPaths": [
    "$.user.email",                    // Specific field
    "$.users[*].email",                // All user emails
    "$.orders[*].customer.*",          // All customer fields in orders
    "$..email",                        // All email fields anywhere
    "$.data[?(@.type=='personal')]",   // Conditional selection
    "$.users[0:10].ssn"                // Array slice
  ],
  "excludePaths": [
    "$.metadata.*",                    // Skip metadata
    "$.system.service_account"         // Skip service accounts
  ]
}

Tokenization for JSON

Replace PII with consistent tokens for analytics:

// Tokenization preserves referential integrity
{
  "users": [
    {"id": 1, "email": "[email protected]", "manager_email": "[email protected]"},
    {"id": 2, "email": "[email protected]", "manager_email": "[email protected]"}
  ]
}

// With tokenization
{
  "users": [
    {"id": 1, "email": "tok_abc123", "manager_email": "tok_def456"},
    {"id": 2, "email": "tok_def456", "manager_email": "tok_ghi789"}
  ]
}

// Note: [email protected] → tok_def456 consistently
// Enables: JOIN operations, relationship analysis
// Prevents: PII exposure

API Integration

// Redact JSON data
POST /v1/redact
Content-Type: application/json

{
  "json": {
    "customer": {
      "name": "John Smith",
      "email": "[email protected]",
      "orders": [
        {"id": "123", "amount": 99.99}
      ]
    }
  },
  "options": {
    "format": "json",
    "fieldDetection": true,
    "valueDetection": true,
    "preserveTypes": true,
    "outputFormat": "json"
  }
}

Response:
{
  "redacted_json": {
    "customer": {
      "name": "[NAME]",
      "email": "[EMAIL]",
      "orders": [
        {"id": "123", "amount": 99.99}
      ]
    }
  },
  "detections": [
    {"path": "$.customer.name", "type": "name", "confidence": 0.95},
    {"path": "$.customer.email", "type": "email", "confidence": 0.99}
  ]
}

Common Use Cases

API Response Logging:

// Redact before logging API responses
app.use(async (req, res, next) => {
  const originalJson = res.json.bind(res);

  res.json = async (data) => {
    // Log redacted version
    const redacted = await redactionClient.redactJson(data);
    logger.info('API Response', { body: redacted });

    // Send original to client
    return originalJson(data);
  };

  next();
});

Data Export Sanitization:

// Process data exports before distribution
async function exportUserData(userId) {
  const userData = await db.getFullUserRecord(userId);

  // Redact for external report
  const redactedData = await redactionClient.redactJson(userData, {
    outputFormat: 'json',
    includeReplacementValues: true
  });

  return redactedData;
}

Analytics Data Preparation:

// Prepare JSON for analytics pipeline
const pipeline = [
  readJsonStream('events.ndjson'),
  redactStream({ types: ['pii'] }),
  writeToAnalytics('redacted_events')
];

await runPipeline(pipeline);