# Diagram-as-code for the DuckLake + RustFS lakehouse.
# Ingestion lands raw bytes/files in RustFS from any data source (dataset hubs,
# streaming, or batch). DuckDB is the query+transform engine that reads RustFS via
# the DuckLake catalog and builds silver and gold; gold is published to Hugging Face.
# Render with:  c4svg lakehouse.c4.yaml lakehouse.svg
canvas: { width: 900, height: 600 }

boxes:
  - { id: SRC, x: 60,  y: 24, w: 300, h: 70, title: "Data sources",
      eqs: ["\\text{dataset hubs (Hugging Face, Kaggle),}", "\\text{streaming, batch}"] }
  - { id: HF,  x: 560, y: 24, w: 250, h: 70, title: "Hugging Face Hub", eqs: ["\\text{published gold datasets}"] }
  - { id: DUCK, x: 110, y: 160, w: 640, h: 76, title: "DuckDB: query and transform engine",
      eqs: ["\\text{reads RustFS, builds silver and gold}"] }
  - { id: LAKE, x: 110, y: 290, w: 640, h: 64, title: "DuckLake: catalog",
      eqs: ["\\text{schemas, snapshots, fragment index, time travel}"] }
  - { id: RUST, x: 110, y: 394, w: 640, h: 40, title: "RustFS (S3): object storage" }
  - { id: RAW,    x: 150, y: 474, w: 160, h: 60, title: "raw",    eqs: ["\\text{objects + URIs}"] }
  - { id: SILVER, x: 350, y: 474, w: 160, h: 60, title: "silver", eqs: ["\\text{cleaned, typed}"] }
  - { id: GOLD,   x: 550, y: 474, w: 160, h: 60, title: "gold",   eqs: ["\\text{ML-ready}"] }

junctions:
  - { id: jL1, x: 50,  y: 59,  dot: false }   # ingest: out of the sources' left ...
  - { id: jL2, x: 50,  y: 504, dot: false }   # ... down and into the raw layer
  - { id: jR1, x: 840, y: 504, dot: false }   # publish: out of gold's right ...
  - { id: jR2, x: 840, y: 59,  dot: false }   # ... up and into Hugging Face

edges:
  # ingest: data sources -> RustFS raw (object storage)
  - { from: SRC, to: jL1, head: none }
  - { from: jL1, to: jL2, head: none, label: "\\text{ingest}" }
  - { from: jL2, to: RAW, enter: left }
  # publish: gold -> Hugging Face Hub
  - { from: GOLD, to: jR1, head: none }
  - { from: jR1,  to: jR2, head: none, label: "\\text{push\\_to\\_hub}" }
  - { from: jR2,  to: HF,  enter: right }
  # engine over catalog over storage
  - { from: DUCK, to: LAKE, enter: top, label: "\\text{ATTACH (SQL)}" }
  - { from: LAKE, to: RUST, enter: top, label: "\\text{data files}" }
  - { from: RUST, to: SILVER, enter: top, head: none }
  # medallion progression, performed by DuckDB
  - { from: RAW,    to: SILVER, enter: left, label: "\\text{clean}" }
  - { from: SILVER, to: GOLD,   enter: left, label: "\\text{aggregate}" }