samples/complex_lineage_demo.json

{
  "title": "E-Commerce Analytics Platform - Complete Data Lineage",
  "description": "A comprehensive demonstration of data lineage tracking across an entire e-commerce analytics platform, showcasing multi-source ingestion, transformation layers, and cross-system dependencies.",
  "version": "1.0",
  "generated_at": "2025-11-20T15:00:00Z",
  "lineage_graph": {
    "nodes": [
      {
        "id": "shopify_orders",
        "name": "Shopify Orders API",
        "type": "source",
        "category": "external_api",
        "description": "Order data from Shopify e-commerce platform",
        "metadata": {
          "platform": "Shopify",
          "refresh_frequency": "real-time webhook",
          "data_volume": "~50K orders/day"
        }
      },
      {
        "id": "shopify_products",
        "name": "Shopify Products API",
        "type": "source",
        "category": "external_api"
      },
      {
        "id": "shopify_customers",
        "name": "Shopify Customers API",
        "type": "source",
        "category": "external_api"
      },
      {
        "id": "stripe_payments",
        "name": "Stripe Payments",
        "type": "source",
        "category": "external_api",
        "description": "Payment transaction data from Stripe"
      },
      {
        "id": "stripe_subscriptions",
        "name": "Stripe Subscriptions",
        "type": "source",
        "category": "external_api"
      },
      {
        "id": "ga4_events",
        "name": "Google Analytics 4",
        "type": "source",
        "category": "analytics",
        "description": "Website behavior and conversion events"
      },
      {
        "id": "fb_ads",
        "name": "Facebook Ads",
        "type": "source",
        "category": "marketing"
      },
      {
        "id": "google_ads",
        "name": "Google Ads",
        "type": "source",
        "category": "marketing"
      },
      {
        "id": "zendesk_tickets",
        "name": "Zendesk Support",
        "type": "source",
        "category": "support",
        "description": "Customer support ticket data"
      },
      {
        "id": "raw_orders",
        "name": "raw.orders",
        "type": "table",
        "category": "raw_layer",
        "schema": "raw",
        "database": "analytics_dw"
      },
      {
        "id": "raw_products",
        "name": "raw.products",
        "type": "table",
        "category": "raw_layer"
      },
      {
        "id": "raw_customers",
        "name": "raw.customers",
        "type": "table",
        "category": "raw_layer"
      },
      {
        "id": "raw_payments",
        "name": "raw.payments",
        "type": "table",
        "category": "raw_layer"
      },
      {
        "id": "raw_subscriptions",
        "name": "raw.subscriptions",
        "type": "table",
        "category": "raw_layer"
      },
      {
        "id": "raw_web_events",
        "name": "raw.web_events",
        "type": "table",
        "category": "raw_layer"
      },
      {
        "id": "raw_ad_spend",
        "name": "raw.ad_spend",
        "type": "table",
        "category": "raw_layer"
      },
      {
        "id": "raw_support_tickets",
        "name": "raw.support_tickets",
        "type": "table",
        "category": "raw_layer"
      },
      {
        "id": "stg_orders",
        "name": "staging.stg_orders",
        "type": "model",
        "category": "staging_layer",
        "transformation": "Clean, dedupe, add calculated fields"
      },
      {
        "id": "stg_order_items",
        "name": "staging.stg_order_items",
        "type": "model",
        "category": "staging_layer"
      },
      {
        "id": "stg_products",
        "name": "staging.stg_products",
        "type": "model",
        "category": "staging_layer"
      },
      {
        "id": "stg_customers",
        "name": "staging.stg_customers",
        "type": "model",
        "category": "staging_layer"
      },
      {
        "id": "stg_payments",
        "name": "staging.stg_payments",
        "type": "model",
        "category": "staging_layer"
      },
      {
        "id": "stg_subscriptions",
        "name": "staging.stg_subscriptions",
        "type": "model",
        "category": "staging_layer"
      },
      {
        "id": "stg_web_sessions",
        "name": "staging.stg_web_sessions",
        "type": "model",
        "category": "staging_layer",
        "transformation": "Sessionize events, calculate engagement"
      },
      {
        "id": "stg_ad_campaigns",
        "name": "staging.stg_ad_campaigns",
        "type": "model",
        "category": "staging_layer"
      },
      {
        "id": "stg_support_cases",
        "name": "staging.stg_support_cases",
        "type": "model",
        "category": "staging_layer"
      },
      {
        "id": "int_customer_orders",
        "name": "intermediate.int_customer_orders",
        "type": "model",
        "category": "intermediate_layer",
        "transformation": "Join customers with order history"
      },
      {
        "id": "int_order_payments",
        "name": "intermediate.int_order_payments",
        "type": "model",
        "category": "intermediate_layer",
        "transformation": "Match orders with payments"
      },
      {
        "id": "int_customer_attribution",
        "name": "intermediate.int_customer_attribution",
        "type": "model",
        "category": "intermediate_layer",
        "transformation": "Multi-touch attribution model"
      },
      {
        "id": "int_product_performance",
        "name": "intermediate.int_product_performance",
        "type": "model",
        "category": "intermediate_layer"
      },
      {
        "id": "int_customer_support_history",
        "name": "intermediate.int_customer_support_history",
        "type": "model",
        "category": "intermediate_layer"
      },
      {
        "id": "dim_customers",
        "name": "marts.dim_customers",
        "type": "dimension",
        "category": "marts_layer",
        "description": "Customer dimension with lifetime metrics",
        "grain": "customer"
      },
      {
        "id": "dim_products",
        "name": "marts.dim_products",
        "type": "dimension",
        "category": "marts_layer",
        "grain": "product"
      },
      {
        "id": "dim_date",
        "name": "marts.dim_date",
        "type": "dimension",
        "category": "marts_layer",
        "grain": "day"
      },
      {
        "id": "fct_orders",
        "name": "marts.fct_orders",
        "type": "fact",
        "category": "marts_layer",
        "grain": "order"
      },
      {
        "id": "fct_order_items",
        "name": "marts.fct_order_items",
        "type": "fact",
        "category": "marts_layer",
        "grain": "order_item"
      },
      {
        "id": "fct_web_sessions",
        "name": "marts.fct_web_sessions",
        "type": "fact",
        "category": "marts_layer"
      },
      {
        "id": "fct_marketing_spend",
        "name": "marts.fct_marketing_spend",
        "type": "fact",
        "category": "marts_layer"
      },
      {
        "id": "fct_support_tickets",
        "name": "marts.fct_support_tickets",
        "type": "fact",
        "category": "marts_layer"
      },
      {
        "id": "rpt_daily_sales",
        "name": "reporting.rpt_daily_sales",
        "type": "report",
        "category": "reporting_layer",
        "description": "Daily sales summary by channel and category"
      },
      {
        "id": "rpt_customer_ltv",
        "name": "reporting.rpt_customer_ltv",
        "type": "report",
        "category": "reporting_layer",
        "description": "Customer lifetime value analysis"
      },
      {
        "id": "rpt_marketing_roi",
        "name": "reporting.rpt_marketing_roi",
        "type": "report",
        "category": "reporting_layer",
        "description": "Marketing ROI by channel and campaign"
      },
      {
        "id": "rpt_product_analytics",
        "name": "reporting.rpt_product_analytics",
        "type": "report",
        "category": "reporting_layer"
      },
      {
        "id": "rpt_customer_health",
        "name": "reporting.rpt_customer_health",
        "type": "report",
        "category": "reporting_layer",
        "description": "Customer health score combining all signals"
      },
      {
        "id": "ml_churn_features",
        "name": "features.churn_prediction",
        "type": "feature_set",
        "category": "ml_features",
        "description": "Features for churn prediction model"
      },
      {
        "id": "ml_ltv_features",
        "name": "features.ltv_prediction",
        "type": "feature_set",
        "category": "ml_features"
      },
      {
        "id": "looker_model",
        "name": "Looker Semantic Layer",
        "type": "semantic_model",
        "category": "bi_layer"
      },
      {
        "id": "tableau_extract",
        "name": "Tableau Extract",
        "type": "extract",
        "category": "bi_layer"
      },
      {
        "id": "salesforce_sync",
        "name": "Salesforce Account Sync",
        "type": "destination",
        "category": "reverse_etl"
      },
      {
        "id": "hubspot_sync",
        "name": "HubSpot Contact Sync",
        "type": "destination",
        "category": "reverse_etl"
      }
    ],
    "edges": [
      {"from": "shopify_orders", "to": "raw_orders", "type": "ingest"},
      {"from": "shopify_products", "to": "raw_products", "type": "ingest"},
      {"from": "shopify_customers", "to": "raw_customers", "type": "ingest"},
      {"from": "stripe_payments", "to": "raw_payments", "type": "ingest"},
      {"from": "stripe_subscriptions", "to": "raw_subscriptions", "type": "ingest"},
      {"from": "ga4_events", "to": "raw_web_events", "type": "ingest"},
      {"from": "fb_ads", "to": "raw_ad_spend", "type": "ingest"},
      {"from": "google_ads", "to": "raw_ad_spend", "type": "ingest"},
      {"from": "zendesk_tickets", "to": "raw_support_tickets", "type": "ingest"},

      {"from": "raw_orders", "to": "stg_orders", "type": "transform"},
      {"from": "raw_orders", "to": "stg_order_items", "type": "transform"},
      {"from": "raw_products", "to": "stg_products", "type": "transform"},
      {"from": "raw_customers", "to": "stg_customers", "type": "transform"},
      {"from": "raw_payments", "to": "stg_payments", "type": "transform"},
      {"from": "raw_subscriptions", "to": "stg_subscriptions", "type": "transform"},
      {"from": "raw_web_events", "to": "stg_web_sessions", "type": "transform"},
      {"from": "raw_ad_spend", "to": "stg_ad_campaigns", "type": "transform"},
      {"from": "raw_support_tickets", "to": "stg_support_cases", "type": "transform"},

      {"from": "stg_customers", "to": "int_customer_orders", "type": "join"},
      {"from": "stg_orders", "to": "int_customer_orders", "type": "join"},
      {"from": "stg_orders", "to": "int_order_payments", "type": "join"},
      {"from": "stg_payments", "to": "int_order_payments", "type": "join"},
      {"from": "stg_customers", "to": "int_customer_attribution", "type": "join"},
      {"from": "stg_web_sessions", "to": "int_customer_attribution", "type": "join"},
      {"from": "stg_ad_campaigns", "to": "int_customer_attribution", "type": "join"},
      {"from": "stg_products", "to": "int_product_performance", "type": "join"},
      {"from": "stg_order_items", "to": "int_product_performance", "type": "join"},
      {"from": "stg_customers", "to": "int_customer_support_history", "type": "join"},
      {"from": "stg_support_cases", "to": "int_customer_support_history", "type": "join"},

      {"from": "int_customer_orders", "to": "dim_customers", "type": "model"},
      {"from": "int_customer_attribution", "to": "dim_customers", "type": "model"},
      {"from": "int_customer_support_history", "to": "dim_customers", "type": "model"},
      {"from": "stg_products", "to": "dim_products", "type": "model"},
      {"from": "int_product_performance", "to": "dim_products", "type": "model"},

      {"from": "int_order_payments", "to": "fct_orders", "type": "model"},
      {"from": "dim_customers", "to": "fct_orders", "type": "reference"},
      {"from": "stg_order_items", "to": "fct_order_items", "type": "model"},
      {"from": "dim_products", "to": "fct_order_items", "type": "reference"},
      {"from": "fct_orders", "to": "fct_order_items", "type": "reference"},
      {"from": "stg_web_sessions", "to": "fct_web_sessions", "type": "model"},
      {"from": "dim_customers", "to": "fct_web_sessions", "type": "reference"},
      {"from": "stg_ad_campaigns", "to": "fct_marketing_spend", "type": "model"},
      {"from": "int_customer_attribution", "to": "fct_marketing_spend", "type": "reference"},
      {"from": "stg_support_cases", "to": "fct_support_tickets", "type": "model"},
      {"from": "dim_customers", "to": "fct_support_tickets", "type": "reference"},

      {"from": "fct_orders", "to": "rpt_daily_sales", "type": "aggregate"},
      {"from": "fct_order_items", "to": "rpt_daily_sales", "type": "aggregate"},
      {"from": "dim_products", "to": "rpt_daily_sales", "type": "reference"},
      {"from": "dim_customers", "to": "rpt_customer_ltv", "type": "aggregate"},
      {"from": "fct_orders", "to": "rpt_customer_ltv", "type": "aggregate"},
      {"from": "fct_marketing_spend", "to": "rpt_marketing_roi", "type": "aggregate"},
      {"from": "fct_orders", "to": "rpt_marketing_roi", "type": "aggregate"},
      {"from": "int_customer_attribution", "to": "rpt_marketing_roi", "type": "reference"},
      {"from": "dim_products", "to": "rpt_product_analytics", "type": "aggregate"},
      {"from": "fct_order_items", "to": "rpt_product_analytics", "type": "aggregate"},
      {"from": "dim_customers", "to": "rpt_customer_health", "type": "aggregate"},
      {"from": "fct_orders", "to": "rpt_customer_health", "type": "aggregate"},
      {"from": "fct_web_sessions", "to": "rpt_customer_health", "type": "aggregate"},
      {"from": "fct_support_tickets", "to": "rpt_customer_health", "type": "aggregate"},

      {"from": "dim_customers", "to": "ml_churn_features", "type": "export"},
      {"from": "fct_orders", "to": "ml_churn_features", "type": "export"},
      {"from": "fct_web_sessions", "to": "ml_churn_features", "type": "export"},
      {"from": "fct_support_tickets", "to": "ml_churn_features", "type": "export"},
      {"from": "dim_customers", "to": "ml_ltv_features", "type": "export"},
      {"from": "fct_orders", "to": "ml_ltv_features", "type": "export"},

      {"from": "rpt_daily_sales", "to": "looker_model", "type": "publish"},
      {"from": "rpt_customer_ltv", "to": "looker_model", "type": "publish"},
      {"from": "rpt_marketing_roi", "to": "looker_model", "type": "publish"},
      {"from": "rpt_product_analytics", "to": "looker_model", "type": "publish"},
      {"from": "rpt_customer_health", "to": "looker_model", "type": "publish"},
      {"from": "rpt_daily_sales", "to": "tableau_extract", "type": "export"},

      {"from": "rpt_customer_ltv", "to": "salesforce_sync", "type": "reverse_etl"},
      {"from": "rpt_customer_health", "to": "salesforce_sync", "type": "reverse_etl"},
      {"from": "rpt_customer_ltv", "to": "hubspot_sync", "type": "reverse_etl"}
    ]
  },
  "expected_visualization": {
    "mermaid": "graph LR\n    subgraph Sources\n        shopify_orders[Shopify Orders]\n        shopify_products[Shopify Products]\n        shopify_customers[Shopify Customers]\n        stripe_payments[Stripe Payments]\n        ga4_events[GA4 Events]\n        fb_ads[Facebook Ads]\n        zendesk_tickets[Zendesk]\n    end\n    \n    subgraph Raw\n        raw_orders[raw.orders]\n        raw_products[raw.products]\n        raw_customers[raw.customers]\n        raw_payments[raw.payments]\n        raw_web_events[raw.web_events]\n    end\n    \n    subgraph Staging\n        stg_orders[staging.stg_orders]\n        stg_customers[staging.stg_customers]\n        stg_products[staging.stg_products]\n    end\n    \n    subgraph Marts\n        dim_customers[marts.dim_customers]\n        dim_products[marts.dim_products]\n        fct_orders[marts.fct_orders]\n    end\n    \n    subgraph Reporting\n        rpt_daily_sales[reporting.rpt_daily_sales]\n        rpt_customer_ltv[reporting.rpt_customer_ltv]\n    end\n    \n    shopify_orders --> raw_orders\n    raw_orders --> stg_orders\n    stg_orders --> fct_orders\n    fct_orders --> rpt_daily_sales"
  },
  "notes": "This comprehensive demo showcases a real-world e-commerce analytics platform with 50+ nodes and 80+ edges across multiple data layers, from source systems through to BI tools and reverse ETL destinations."
}