Privacy in Experience Layer

Experience respects privacy at both ingest and read paths. Modes are aligned with orchestration privacy policy.

Modes

• allow: raw texts may be stored and served in short summaries
• redact: PII is removed at ingest; summaries must not leak PII
• block: do not store raw texts; only IDs/metadata; retrieval returns unit_id, refs, score (no summary)

PII flow

1. Detect PII at ingest (heuristics + patterns)
2. If redact: scrub inputs/outputs before persistence; mark privacy.mode=redact
3. If block: store metadata only; skip text indexing; unit privacy.mode=block
4. Summarization operates on redacted text only

Adapter coupling

• HTTP adapter forwards privacy_mode from request; L4 must not exceed requested exposure
• For block, HTTP reply omits summary text; headers may include X-Warnings when content was blocked

KV policy and privacy

• pin only allow/redact units; never pin block if it risks exposure
• compress/evict policies must never down-grade privacy guarantees

Comprehensive Privacy Testing Framework

PII Detection & Redaction Testing

describe('L4 Privacy Protection Tests', () => {
  describe('PII Detection Accuracy', () => {
    test('detects common PII patterns', async () => {
      const test_cases = [
        { 
          input: 'User john.doe@company.com needs help with API setup', 
          expected_pii: ['email'],
          privacy_mode: 'redact'
        },
        { 
          input: 'Database connection: postgresql://user:password123@localhost:5432/db', 
          expected_pii: ['connection_string'],
          privacy_mode: 'block'
        },
        { 
          input: 'API key sk-1234567890abcdef for authentication', 
          expected_pii: ['api_key'],
          privacy_mode: 'block'
        }
      ];
      
      for (const test_case of test_cases) {
        const detection_result = await privacyEngine.detectPII(test_case.input);
        
        expect(detection_result.pii_detected).toBe(true);
        expect(detection_result.pii_types).toEqual(
          expect.arrayContaining(test_case.expected_pii)
        );
      }
    });
    
    test('handles redaction mode correctly', async () => {
      const pii_event = {
        input: { text: 'Contact john.doe@company.com for support' },
        privacy: { mode: 'redact', pii: true }
      };
      
      const processed = await privacyEngine.processEvent(pii_event);
      
      expect(processed.input.text).not.toContain('john.doe@company.com');
      expect(processed.input.text).toContain('[EMAIL_REDACTED]');
      expect(processed.privacy.redaction_applied).toBe(true);
      expect(processed.privacy.original_pii_detected).toBe(true);
    });
    
    test('enforces block mode for sensitive content', async () => {
      const sensitive_event = {
        output: { text: 'Here\'s your API key: sk-secret123456789' },
        privacy: { mode: 'block', pii: true }
      };
      
      const processed = await privacyEngine.processEvent(sensitive_event);
      
      expect(processed.blocked).toBe(true);
      expect(processed.output).toBeUndefined();
      expect(processed.metadata).toBeDefined();
      expect(processed.privacy.block_reason).toBe('sensitive_credentials');
    });
  });
  
  describe('Privacy Mode Enforcement', () => {
    test('prevents privacy mode downgrade', async () => {
      const block_event = {
        privacy: { mode: 'block', pii: true },
        content: 'sensitive data'
      };
      
      // Attempt to retrieve with less restrictive mode
      const retrieval_attempt = async () => {
        return await experienceRetrieval.retrieve({
          unit_id: 'test_unit',
          privacy_mode: 'allow' // Less restrictive than original 'block'
        });
      };
      
      await expect(retrieval_attempt).rejects.toThrow('Privacy mode downgrade not allowed');
    });
    
    test('maintains privacy guarantees across pipeline', async () => {
      const redacted_event = {
        input: { text: 'User email: user@example.com' },
        privacy: { mode: 'redact' }
      };
      
      // Process through entire pipeline
      const ingested = await l4Pipeline.ingest(redacted_event);
      const summarized = await l4Pipeline.waitForSummarization(ingested.id);
      const retrieved = await l4Pipeline.retrieve({ unit_id: summarized.unit_id });
      
      // Verify PII is redacted at every stage
      expect(ingested.privacy.redaction_applied).toBe(true);
      expect(summarized.summary).not.toContain('user@example.com');
      expect(retrieved.summary).not.toContain('user@example.com');
      
      // Verify redaction markers are consistent
      expect(retrieved.summary).toContain('[EMAIL_REDACTED]');
    });
  });
});

Privacy Compliance Monitoring

privacy_compliance_monitoring:
  metrics:
    # Core privacy metrics
    - name: l4_pii_detection_accuracy
      type: gauge
      description: "Accuracy of PII detection algorithm"
      target: 0.99
      
    - name: l4_privacy_leak_incidents
      type: counter
      description: "Count of privacy leak incidents"
      labels: [leak_type, severity]
      target: 0  # Zero tolerance
      
    - name: l4_privacy_mode_distribution
      type: gauge
      description: "Distribution of privacy modes used"
      labels: [mode]
      
    - name: l4_redaction_effectiveness
      type: gauge
      description: "Effectiveness of redaction process"
      labels: [pii_type]
      target: 1.0
      
  alerts:
    - name: L4PrivacyLeakDetected
      condition: l4_privacy_leak_incidents > 0
      severity: critical
      immediate_action: true
      
    - name: L4PIIDetectionAccuracyLow
      condition: l4_pii_detection_accuracy < 0.95
      severity: warning
      duration: 5m
      
  compliance_auditing:
    gdpr_compliance:
      data_retention: 30_days_max
      right_to_erasure: implemented
      data_minimization: enforced
      
    security_controls:
      encryption_at_rest: required
      encryption_in_transit: required
      access_logging: comprehensive

Advanced Privacy Protection

interface PrivacyProtectionEngine {
  // Advanced PII detection with context awareness
  detectContextualPII(text: string, context: PrivacyContext): PIIDetectionResult;
  
  // Differential privacy for aggregated insights
  applyDifferentialPrivacy(data: AggregatedData, epsilon: number): PrivateData;
  
  // Zero-knowledge proof generation for privacy-preserving verification
  generatePrivacyProof(data: SensitiveData): ZKProof;
  
  // Homomorphic encryption for computation on encrypted data
  encryptForComputation(data: PersonalData): HomomorphicCiphertext;
}

class EnhancedPrivacyEngine implements PrivacyProtectionEngine {
  detectContextualPII(text: string, context: PrivacyContext): PIIDetectionResult {
    const patterns = {
      email: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g,
      phone: /\b\d{3}[-.]?\d{3}[-.]?\d{4}\b/g,
      ssn: /\b\d{3}-\d{2}-\d{4}\b/g,
      api_key: /\b(sk|pk|api)[-_]?[a-zA-Z0-9]{20,}\b/gi,
      connection_string: /(?:mongodb|postgresql|mysql):\/\/[^\s]+/gi,
      credit_card: /\b\d{4}[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{4}\b/g
    };
    
    const detected_pii = [];
    for (const [type, pattern] of Object.entries(patterns)) {
      const matches = text.match(pattern);
      if (matches) {
        detected_pii.push({ type, instances: matches.length, severity: this.getSeverity(type, context) });
      }
    }
    
    return {
      pii_detected: detected_pii.length > 0,
      pii_types: detected_pii.map(p => p.type),
      severity_level: Math.max(...detected_pii.map(p => p.severity)),
      recommended_action: this.getRecommendedAction(detected_pii, context)
    };
  }
  
  private getSeverity(pii_type: string, context: PrivacyContext): number {
    const severity_map = {
      email: context.environment === 'production' ? 0.7 : 0.5,
      api_key: 0.9,
      connection_string: 0.95,
      credit_card: 1.0,
      ssn: 1.0
    };
    return severity_map[pii_type] || 0.5;
  }
  
  private getRecommendedAction(pii: PIIDetection[], context: PrivacyContext): 'allow' | 'redact' | 'block' {
    const max_severity = Math.max(...pii.map(p => p.severity));
    if (max_severity >= 0.9) return 'block';
    if (max_severity >= 0.6) return 'redact';
    return 'allow';
  }
}

GDPR & Compliance Framework

interface GDPRComplianceManager {
  // Right to erasure implementation
  handleErasureRequest(user_id: string): Promise<ErasureResult>;
  
  // Data portability support  
  exportUserData(user_id: string): Promise<UserDataExport>;
  
  // Consent management
  updateConsentPreferences(user_id: string, preferences: ConsentPreferences): Promise<void>;
  
  // Audit trail generation
  generatePrivacyAuditTrail(time_range: TimeRange): Promise<AuditTrail>;
}

const gdpr_compliance_implementation = {
  data_retention: {
    experience_events: '90_days',
    experience_units: '1_year',
    privacy_logs: '7_years', // Regulatory requirement
    audit_trails: '7_years'
  },
  
  user_rights: {
    right_to_access: 'automated_export',
    right_to_rectification: 'user_portal',
    right_to_erasure: 'automated_deletion',
    right_to_portability: 'structured_export',
    right_to_restrict_processing: 'privacy_mode_controls'
  },
  
  privacy_by_design: {
    data_minimization: 'collect_only_necessary',
    purpose_limitation: 'experience_enhancement_only',
    storage_limitation: 'automated_expiry',
    security: 'encryption_at_rest_and_transit'
  }
};

See also:

• ./contracts.md
• ../orchestration/privacy.md
• ../evaluation/privacy-metrics.md