ML Engineer Journey: Real-Time Fraud Detection System

{
  "source": {
    "type": "kafka",
    "name": "payment-gateway-stream",
    "connection": {
      "bootstrap_servers": "kafka.meridian.internal:9092",
      "topic": "payments.transactions.authorized",
      "consumer_group": "matih-fraud-ingestion",
      "format": "avro",
      "schema_registry": "http://schema-registry.meridian.internal:8081"
    },
    "sync_mode": "streaming",
    "processing_guarantees": "exactly_once",
    "batch_size": 1000,
    "poll_interval_ms": 100
  }
}

┌─────────────────────────────────────────────────────────────┐
│           Transaction Ingestion Health                       │
├─────────────┬──────────┬────────────┬───────────────────────┤
│ Source      │ Lag      │ Throughput │ Status                │
├─────────────┼──────────┼────────────┼───────────────────────┤
│ Kafka       │ 1.2s     │ 10,247/min │ Healthy               │
│ Core Banking│ 8m 14s   │ 2,100/sync │ Healthy               │
│ Fraud Cases │ 22m      │ 48/sync    │ Healthy               │
│ IP Repute   │ 6h 12m   │ 1.2M/sync  │ Healthy (daily batch) │
└─────────────┴──────────┴────────────┴───────────────────────┘

-- Existing velocity features found in catalog
SELECT table_name, column_name, description, last_updated
FROM catalog.column_metadata
WHERE schema_name = 'fraud_features'
ORDER BY last_updated DESC;

-- Real-time fraud feature computation
-- Executed per-transaction at scoring time
-- Target: < 10ms execution
 
SELECT
    t.txn_id,
    t.account_id,
    t.amount,
    t.merchant_category,
    t.channel,
    -- Velocity features (pre-computed, lookup only)
    v1.txn_count_1h,
    v1.amount_sum_1h,
    v1.unique_merchants_1h,
    v24.txn_count_24h,
    v24.amount_sum_24h,
    v24.amount_max_24h,
    v7.unique_countries_7d,
    -- Amount anomaly features (computed inline)
    t.amount / NULLIF(a.avg_txn_amount_90d, 0)
        AS amount_ratio_to_avg,
    CASE WHEN t.amount > a.p95_txn_amount_90d THEN 1 ELSE 0 END
        AS exceeds_p95,
    -- Merchant category pattern
    CASE WHEN mc.fraud_rate > 0.005 THEN 1 ELSE 0 END
        AS high_risk_merchant,
    mc.fraud_rate AS merchant_category_fraud_rate,
    -- Temporal features
    EXTRACT(HOUR FROM t.timestamp) AS txn_hour,
    CASE WHEN EXTRACT(DOW FROM t.timestamp) IN (0, 6) THEN 1 ELSE 0 END
        AS is_weekend,
    -- Device and location
    COALESCE(df.trust_score, 0.5) AS device_trust_score,
    COALESCE(ip.risk_score, 0.5) AS ip_risk_score,
    -- Account profile
    a.account_age_days,
    a.total_txn_count,
    a.has_fraud_history
FROM transactions t
JOIN accounts a ON t.account_id = a.account_id
LEFT JOIN fraud_features.txn_velocity_1h v1 ON t.account_id = v1.account_id
LEFT JOIN fraud_features.txn_velocity_24h v24 ON t.account_id = v24.account_id
LEFT JOIN fraud_features.txn_velocity_7d v7 ON t.account_id = v7.account_id
LEFT JOIN reference.merchant_category_risk mc ON t.merchant_category = mc.category
LEFT JOIN device_fingerprints df ON t.device_id = df.device_id
LEFT JOIN ip_reputation ip ON t.ip_hash = ip.ip_hash
WHERE t.txn_id = :current_txn_id;

-- Analyze fraud patterns by merchant category over time
-- Data source: S3/DuckDB (historical transaction archive)
 
SELECT
    merchant_category,
    DATE_TRUNC('month', timestamp) AS month,
    COUNT(*) AS total_txns,
    SUM(CASE WHEN is_fraud THEN 1 ELSE 0 END) AS fraud_count,
    ROUND(100.0 * SUM(CASE WHEN is_fraud THEN 1 ELSE 0 END)
        / COUNT(*), 4) AS fraud_rate_pct,
    AVG(CASE WHEN is_fraud THEN amount ELSE NULL END) AS avg_fraud_amount,
    PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY
        CASE WHEN is_fraud THEN amount ELSE NULL END) AS median_fraud_amount
FROM read_parquet('s3://meridian-data-lake/transactions/year=*/month=*/*.parquet')
WHERE timestamp >= '2025-01-01'
GROUP BY merchant_category, DATE_TRUNC('month', timestamp)
HAVING fraud_count > 10
ORDER BY fraud_rate_pct DESC;

┌──────────┐   ┌──────────────┐   ┌──────────────┐   ┌───────────┐   ┌──────────┐
│  Kafka   │──▶│   Feature    │──▶│    Model     │──▶│  Decision │──▶│  Alert   │
│  Topic   │   │  Computation │   │   Scoring    │   │  Engine   │   │  Router  │
│          │   │              │   │              │   │           │   │          │
│ 10K/min  │   │ Velocity     │   │ Ensemble     │   │ Score >   │   │ Queue /  │
│          │   │ lookups,     │   │ inference    │   │ 0.85 →    │   │ Block /  │
│          │   │ enrichment   │   │ (Ray Serve)  │   │ block     │   │ Allow    │
└──────────┘   └──────────────┘   └──────────────┘   └───────────┘   └──────────┘
    1ms             8ms                12ms               2ms             1ms
                                                                    Total: ~24ms

{
  "pipeline": {
    "name": "fraud-model-retraining",
    "schedule": "0 2 * * 0",
    "owner": "kenji@meridian.bank",
    "tags": ["fraud", "ml-training", "weekly"],
    "stages": [
      {
        "name": "extract_training_data",
        "type": "sql_transform",
        "query": "SELECT * FROM fraud_features.training_dataset WHERE label_date >= CURRENT_DATE - INTERVAL '90 days'",
        "output": "ml_staging.fraud_training_latest"
      },
      {
        "name": "validate_labels",
        "type": "data_quality",
        "checks": [
          {
            "table": "ml_staging.fraud_training_latest",
            "expectation": "expect_column_values_to_be_in_set",
            "column": "is_fraud",
            "value_set": [true, false],
            "severity": "critical"
          },
          {
            "table": "ml_staging.fraud_training_latest",
            "expectation": "expect_table_row_count_to_be_between",
            "min_value": 1000000,
            "severity": "critical"
          },
          {
            "table": "ml_staging.fraud_training_latest",
            "expectation": "expect_column_proportion_of_unique_values_to_be_between",
            "column": "is_fraud",
            "min_value": 0.0005,
            "max_value": 0.01,
            "severity": "warning"
          }
        ],
        "on_failure": "halt_pipeline"
      },
      {
        "name": "train_model",
        "type": "ml_training",
        "experiment": "fraud-detection-weekly",
        "model_config": {
          "ensemble": [
            {"type": "xgboost", "weight": 0.6},
            {"type": "neural_network", "weight": 0.4}
          ]
        },
        "depends_on": ["validate_labels"]
      },
      {
        "name": "evaluate_model",
        "type": "ml_evaluation",
        "metrics": ["auc", "precision_at_1pct_fpr", "recall_at_2pct_fpr"],
        "promotion_criteria": {
          "auc": "> 0.95",
          "precision_at_1pct_fpr": "> 0.60"
        },
        "depends_on": ["train_model"]
      },
      {
        "name": "refresh_feature_store",
        "type": "feature_store_ingest",
        "source_table": "fraud_features.txn_velocity_1h",
        "feature_group": "fraud_velocity",
        "depends_on": ["evaluate_model"]
      }
    ]
  }
}

{
  "processing_guarantees": {
    "dedup_key": "txn_id",
    "dedup_window_minutes": 60,
    "checkpoint_interval_ms": 5000,
    "state_backend": "redis",
    "redis_key_prefix": "fraud:dedup:",
    "on_duplicate": "skip_and_log"
  }
}

-- Validate fraud labels against investigation outcomes
SELECT
    detection_method,
    COUNT(*) AS cases,
    SUM(CASE WHEN resolution = 'confirmed_fraud' THEN 1 ELSE 0 END) AS true_positives,
    SUM(CASE WHEN resolution = 'false_alarm' THEN 1 ELSE 0 END) AS false_positives,
    ROUND(100.0 * SUM(CASE WHEN resolution = 'confirmed_fraud' THEN 1 ELSE 0 END)
        / COUNT(*), 1) AS precision_pct
FROM fraud_cases
WHERE investigation_date >= '2025-07-01'
GROUP BY detection_method
ORDER BY cases DESC;

{
  "deployment": {
    "name": "fraud-detection-ensemble-v3",
    "serving_engine": "ray_serve",
    "mode": "real_time",
    "models": [
      {
        "name": "fraud-xgboost-v3",
        "weight": 0.6,
        "framework": "xgboost",
        "resources": {"num_cpus": 2, "memory_gb": 4}
      },
      {
        "name": "fraud-nn-v3",
        "weight": 0.4,
        "framework": "pytorch",
        "resources": {"num_cpus": 2, "memory_gb": 4}
      }
    ],
    "sla": {
      "p50_latency_ms": 15,
      "p99_latency_ms": 50,
      "availability": 0.9999,
      "throughput_per_second": 200
    },
    "scaling": {
      "min_replicas": 3,
      "max_replicas": 12,
      "target_ongoing_requests": 50,
      "scale_up_threshold_seconds": 5,
      "availability_zones": ["us-east-1a", "us-east-1b", "us-east-1c"]
    },
    "fallback": {
      "enabled": true,
      "type": "rules_engine",
      "trigger": "model_latency > 100ms OR model_error",
      "rules": [
        {"condition": "amount > 5000 AND channel = 'online'", "action": "flag"},
        {"condition": "txn_count_1h > 20", "action": "flag"},
        {"condition": "country != account_country", "action": "flag"}
      ]
    }
  }
}

┌─────────────────────────────────────────────────────────────────────┐
│                 FRAUD DETECTION OPERATIONS                          │
│                 Last updated: 2026-02-28 14:32:17 UTC               │
├─────────────────────┬──────────────────────┬────────────────────────┤
│  Scoring Latency    │  Model Confidence    │  Fraud Rate (24h)      │
│                     │                      │                        │
│  P50:  11ms         │  Mean:   0.12        │  Detected:    142      │
│  P95:  28ms         │  Median: 0.04        │  Missed (est): 8       │
│  P99:  38ms         │  > 0.85:  0.09%      │  False Pos:    31      │
│  Max:  67ms         │  > 0.50:  0.41%      │  Catch Rate: 94.7%     │
├─────────────────────┼──────────────────────┼────────────────────────┤
│  Throughput         │  Fallback Events     │  Alerts (24h)          │
│                     │                      │                        │
│  Current: 168/sec   │  Today:      0       │  Latency:        0     │
│  Peak:    412/sec   │  This Week:  2       │  Confidence:     1     │
│  Capacity: 847/sec  │  Reason: timeout     │  Drift:          0     │
│  Util:    19.8%     │                      │  Fraud Spike:    0     │
└─────────────────────┴──────────────────────┴────────────────────────┘

{
  "alerts": [
    {
      "name": "scoring-latency-spike",
      "condition": "p99_latency_5min > 100",
      "severity": "critical",
      "channels": ["pagerduty", "slack:#fraud-ops"],
      "action": "Consider scaling replicas or activating fallback"
    },
    {
      "name": "model-confidence-drop",
      "condition": "avg_confidence_1h < 0.08 OR avg_confidence_1h > 0.25",
      "severity": "warning",
      "channels": ["slack:#fraud-ops"],
      "action": "Investigate input distribution shift"
    },
    {
      "name": "fraud-rate-anomaly",
      "condition": "fraud_rate_1h > 3 * fraud_rate_7d_avg",
      "severity": "critical",
      "channels": ["pagerduty", "slack:#fraud-ops", "email:fraud-team"],
      "action": "Potential fraud attack -- escalate to fraud investigation"
    },
    {
      "name": "feature-drift-detected",
      "condition": "any_feature_psi > 0.10",
      "severity": "warning",
      "channels": ["slack:#fraud-ops"],
      "action": "Schedule model retrain if sustained > 48h"
    }
  ]
}

{
  "experiment": {
    "name": "device-fingerprint-feature-test",
    "type": "shadow",
    "production_model": "fraud-detection-ensemble-v3",
    "shadow_model": "fraud-detection-ensemble-v4-candidate",
    "shadow_features_added": [
      "device_fingerprint_match_score",
      "behavioral_biometric_score",
      "typing_pattern_anomaly"
    ],
    "duration_days": 14,
    "evaluation": {
      "metrics": ["auc", "precision_at_1pct_fpr", "recall_at_2pct_fpr"],
      "comparison_window": "daily"
    }
  }
}

┌─────────────────────────────────────────────────┐
│            Transaction Flow                      │
└────────────────────┬────────────────────────────┘
                     │
          ┌──────────┴──────────┐
          │   Traffic Splitter   │
          │   (by account_id     │
          │    hash, stable)     │
          └──────────┬──────────┘
               ┌─────┴─────┐
         ┌─────▼─────┐ ┌───▼─────────┐
         │ Control   │ │ Treatment   │
         │ (95%)     │ │ (5%)        │
         │           │ │             │
         │ v3        │ │ v4 cand.    │
         │ Ensemble  │ │ + device    │
         │           │ │   features  │
         └─────┬─────┘ └──────┬──────┘
               │              │
         ┌─────▼──────────────▼──────┐
         │  Both make real decisions  │
         │  Results tracked per group │
         └───────────────────────────┘