From 5fe000d144726480adfb9dcaa8a9c3d7106f6fa8 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Wed, 3 Jun 2026 23:14:34 +0800 Subject: [PATCH 01/18] Add Trino JDBC platform --- demo-trino.sh | 6 + trino-setup/README.md | 121 ++++++++++ trino-setup/demo.sh | 111 +++++++++ trino-setup/docker-compose.yml | 126 ++++++++++ trino-setup/pom.xml | 77 ++++++ trino-setup/scripts/init.sql | 50 ++++ trino-setup/scripts/run-init.sh | 23 ++ .../wayang/trino/TrinoIntegrationTest.java | 215 +++++++++++++++++ trino-setup/trino/catalog/iceberg.properties | 16 ++ trino-setup/trino/catalog/tpch.properties | 4 + trino-setup/trino/config.properties | 4 + wayang-platforms/pom.xml | 1 + .../wayang/jdbc/execution/JdbcExecutor.java | 22 +- .../jdbc/operators/JdbcTableSource.java | 3 +- .../jdbc/execution/JdbcExecutorTest.java | 8 +- .../JdbcGlobalReduceOperatorTest.java | 17 +- .../jdbc/operators/JdbcJoinOperatorTest.java | 4 +- .../operators/JdbcReduceByOperatorTest.java | 2 +- .../jdbc/operators/JdbcSortOperatorTest.java | 2 +- wayang-platforms/wayang-trino/pom.xml | 74 ++++++ .../java/org/apache/wayang/trino/Trino.java | 63 +++++ .../org/apache/wayang/trino/TrinoDemo.java | 224 ++++++++++++++++++ .../trino/channels/ChannelConversions.java | 54 +++++ .../wayang/trino/mapping/FilterMapping.java | 63 +++++ .../wayang/trino/mapping/JoinMapping.java | 76 ++++++ .../apache/wayang/trino/mapping/Mappings.java | 37 +++ .../trino/mapping/ProjectionMapping.java | 66 ++++++ .../operators/TrinoExecutionOperator.java | 31 +++ .../trino/operators/TrinoFilterOperator.java | 52 ++++ .../trino/operators/TrinoJoinOperator.java | 49 ++++ .../operators/TrinoProjectionOperator.java | 49 ++++ .../trino/operators/TrinoTableSource.java | 57 +++++ .../wayang/trino/platform/TrinoPlatform.java | 51 ++++ .../trino/plugin/TrinoConversionsPlugin.java | 58 +++++ .../wayang/trino/plugin/TrinoPlugin.java | 58 +++++ .../wayang-trino-defaults.properties | 105 ++++++++ 36 files changed, 1950 insertions(+), 29 deletions(-) create mode 100644 demo-trino.sh create mode 100644 trino-setup/README.md create mode 100644 trino-setup/demo.sh create mode 100644 trino-setup/docker-compose.yml create mode 100644 trino-setup/pom.xml create mode 100644 trino-setup/scripts/init.sql create mode 100644 trino-setup/scripts/run-init.sh create mode 100644 trino-setup/src/test/java/org/apache/wayang/trino/TrinoIntegrationTest.java create mode 100644 trino-setup/trino/catalog/iceberg.properties create mode 100644 trino-setup/trino/catalog/tpch.properties create mode 100644 trino-setup/trino/config.properties create mode 100644 wayang-platforms/wayang-trino/pom.xml create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/Trino.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/TrinoDemo.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/channels/ChannelConversions.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/FilterMapping.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/JoinMapping.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/ProjectionMapping.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoExecutionOperator.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoFilterOperator.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoJoinOperator.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoProjectionOperator.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoTableSource.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/platform/TrinoPlatform.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/plugin/TrinoConversionsPlugin.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/plugin/TrinoPlugin.java create mode 100644 wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties diff --git a/demo-trino.sh b/demo-trino.sh new file mode 100644 index 000000000..d41783b30 --- /dev/null +++ b/demo-trino.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +WAYANG_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "$WAYANG_ROOT/trino-setup/demo.sh" "$@" diff --git a/trino-setup/README.md b/trino-setup/README.md new file mode 100644 index 000000000..960c53fec --- /dev/null +++ b/trino-setup/README.md @@ -0,0 +1,121 @@ +# Trino Local Setup + +Local Trino environment backed by an **Iceberg** data lake, completely containerised. + +## Stack + +| Component | Image | Port | Role | +|-----------|-------|------|------| +| **Trino** | `trinodb/trino:435` | 8080 | SQL query engine | +| **Hive Metastore** | `naushadh/hive-metastore:latest` | 9083 | Iceberg table catalog (Thrift) | +| **PostgreSQL** | `postgres:15-alpine` | 5432 | HMS metadata backing store | +| **MinIO** | `minio/minio:latest` | 9000 / 9001 | S3-compatible object storage | + +HMS is the battle-tested Iceberg catalog for Trino. Parquet data files are written by Trino directly to MinIO; HMS only stores schema/table metadata. + +## Directory Layout + +``` +trino-setup/ +├── docker-compose.yml # Full stack definition +├── trino/ +│ ├── config.properties # Trino node config +│ └── catalog/ +│ ├── iceberg.properties # Iceberg via HMS + MinIO +│ └── tpch.properties # Built-in TPC-H (no storage needed) +├── scripts/ +│ ├── init.sql # Creates iceberg.sales.orders + sample rows +│ └── run-init.sh # Helper: waits for Trino then runs init.sql +├── pom.xml # Standalone Maven project (Java 17) +└── src/test/java/.../ + └── TrinoIntegrationTest.java # JUnit 5 integration tests +``` + +## Quick Start + +### 1. Start the stack + +```bash +cd trino-setup +docker-compose up -d +``` + +Wait ~30 seconds for all services to become healthy. Check with: + +```bash +docker-compose ps +# or watch the Trino UI at http://localhost:8080 +``` + +### 2. Load sample Iceberg data + +```bash +./scripts/run-init.sh +``` + +This creates the schema `iceberg.sales` and inserts 10 sample orders into +`iceberg.sales.orders` (Parquet files on MinIO). + +### 3. Run the integration tests + +```bash +mvn test -Pintegration +``` + +Tests are skipped by default (no `-Pintegration`) to avoid requiring Docker in CI. + +### 4. Manual exploration + +Open the **Trino UI**: http://localhost:8080 + +Or connect via the Trino CLI inside the container: + +```bash +docker exec -it trino trino --catalog iceberg --schema sales +``` + +```sql +-- TPC-H built-in data (no init.sql needed) +SELECT * FROM tpch.tiny.orders LIMIT 5; + +-- Iceberg table +SELECT region, SUM(amount) FROM iceberg.sales.orders GROUP BY region; + +-- Iceberg file metadata +SELECT * FROM iceberg.sales."orders$files"; + +-- Iceberg history +SELECT * FROM iceberg.sales."orders$history"; +``` + +**MinIO console**: http://localhost:9001 (login: `minioadmin` / `minioadmin`) +Look for Parquet files under `warehouse/sales/orders/`. + +### 5. Tear down + +```bash +docker-compose down -v # -v removes volumes (clears MinIO data) +``` + +## Test Coverage + +| Test | What it checks | +|------|----------------| +| `testConnectivity` | `SELECT 1` — JDBC connection works | +| `testTpchConnector` | TPC-H built-in connector, no storage needed | +| `testTpchTopOrders` | ORDER BY + LIMIT on TPC-H | +| `testIcebergSchemaVisible` | Schema created by `init.sql` is visible | +| `testIcebergSelectAll` | Full table scan, 10 rows | +| `testIcebergFilterByRegion` | WHERE pushdown on string column | +| `testIcebergAggregate` | GROUP BY + SUM aggregation | +| `testIcebergFilterByAmount` | WHERE pushdown on double column | +| `testIcebergProjection` | SELECT subset of columns | +| `testIcebergFilesMetadata` | `$files` system table, confirms Parquet on MinIO | + +## Environment Variables + +Override defaults if running Trino on a different host/port: + +```bash +TRINO_HOST=my-trino-host TRINO_PORT=8080 mvn test -Pintegration +``` diff --git a/trino-setup/demo.sh b/trino-setup/demo.sh new file mode 100644 index 000000000..3ae3e42e1 --- /dev/null +++ b/trino-setup/demo.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WAYANG_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +TRINO_SETUP="$SCRIPT_DIR" +TRINO_CONTAINER="trino" +MAVEN_FLAGS="-Pskip-prerequisite-check -Drat.skip=true -Dmaven.javadoc.skip=true" + +banner() { + echo + echo "============================================================" + printf " %s\n" "$*" + echo "============================================================" + echo +} + +step() { + echo + echo "-- $*" + echo +} + +pause() { + if [[ "${WAYANG_DEMO_AUTO:-false}" != "true" ]]; then + echo + read -rp "Press ENTER to continue..." _ || true + echo + fi +} + +run_wayang_demo() { + "$WAYANG_ROOT/mvnw" exec:java -pl wayang-platforms/wayang-trino \ + -Dexec.mainClass="org.apache.wayang.trino.TrinoDemo" \ + ${MAVEN_FLAGS} +} + +banner "ACT 1: Start Trino + Iceberg via Docker" + +step "1a. Starting the stack" +cd "$TRINO_SETUP" +docker compose up -d + +step "1b. Containers running" +docker ps --format "table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}" \ + | grep -E "NAMES|trino|minio|metastore|postgres" + +step "1c. Waiting for Trino to be ready" +MAX_WAIT=90 +ELAPSED=0 +until docker exec "$TRINO_CONTAINER" \ + trino --execute "SELECT 1" --output-format ALIGNED >/dev/null 2>&1; do + if [[ "$ELAPSED" -ge "$MAX_WAIT" ]]; then + echo "Timed out waiting for Trino after ${MAX_WAIT}s" + exit 1 + fi + printf ". waiting (%ds elapsed)\r" "$ELAPSED" + sleep 3 + ELAPSED=$((ELAPSED + 3)) +done +echo "Trino is ready at http://localhost:8080" + +step "1d. Initialising Iceberg tables" +docker exec -i "$TRINO_CONTAINER" trino < "$TRINO_SETUP/scripts/init.sql" 2>&1 \ + | grep -v "^WARNING\|jline\|org.jline" || true +echo "iceberg.sales.orders seeded" + +step "1e. Table schema" +docker exec "$TRINO_CONTAINER" \ + trino --execute "DESCRIBE iceberg.sales.orders" \ + --output-format ALIGNED 2>/dev/null + +pause + +banner "ACT 2: Query Iceberg directly via Trino CLI" + +step "2a. Full table scan" +echo "SQL: SELECT * FROM iceberg.sales.orders" +docker exec "$TRINO_CONTAINER" \ + trino --execute "SELECT * FROM iceberg.sales.orders ORDER BY order_id" \ + --output-format ALIGNED 2>/dev/null + +step "2b. Filter: region = 'AMER'" +echo "SQL: SELECT * FROM iceberg.sales.orders WHERE region = 'AMER'" +docker exec "$TRINO_CONTAINER" \ + trino --execute "SELECT * FROM iceberg.sales.orders WHERE region = 'AMER' ORDER BY order_id" \ + --output-format ALIGNED 2>/dev/null + +step "2c. Projection with filter" +echo "SQL: SELECT region, product, amount FROM iceberg.sales.orders WHERE region = 'AMER'" +docker exec "$TRINO_CONTAINER" \ + trino --execute \ + "SELECT region, product, amount + FROM iceberg.sales.orders + WHERE region = 'AMER' + ORDER BY order_id" \ + --output-format ALIGNED 2>/dev/null + +pause + +banner "ACT 3: Wayang API filter + projection pushdown" +cd "$WAYANG_ROOT" +run_wayang_demo + +banner "Demo complete" +echo "Trino UI: http://localhost:8080" +echo "MinIO UI: http://localhost:9001 (minioadmin / minioadmin)" +echo +echo "To stop the stack:" +echo " cd trino-setup && docker compose down" diff --git a/trino-setup/docker-compose.yml b/trino-setup/docker-compose.yml new file mode 100644 index 000000000..d05a39d15 --- /dev/null +++ b/trino-setup/docker-compose.yml @@ -0,0 +1,126 @@ +--- +# Stack: Trino + Hive Metastore + MinIO (S3 storage) +# +# This is the battle-tested Trino + Iceberg local setup. +# Hive Metastore (HMS) stores Iceberg table metadata over Thrift on port 9083. +# MinIO provides S3-compatible object storage for Parquet data files. +# Trino's Iceberg connector uses HMS as catalog and writes Parquet to MinIO. +# +# Ports: +# Trino: http://localhost:8080 (UI + JDBC) +# MinIO S3: http://localhost:9000 +# MinIO UI: http://localhost:9001 (minioadmin / minioadmin) +# HMS: localhost:9083 (Thrift, internal) +# Postgres: localhost:5432 (HMS backing store) + +services: + + # ── PostgreSQL (Hive Metastore backing database) ─────────────────────────── + postgres: + image: postgres:15-alpine + container_name: trino-postgres + environment: + POSTGRES_DB: metastore + POSTGRES_USER: hive + POSTGRES_PASSWORD: hive + ports: + - "5432:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U hive -d metastore"] + interval: 10s + timeout: 5s + retries: 5 + + # ── MinIO (S3-compatible object storage) ────────────────────────────────── + minio: + image: minio/minio:latest + container_name: trino-minio + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + ports: + - "9000:9000" + - "9001:9001" + command: server /data --console-address ":9001" + volumes: + - minio-data:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 10s + timeout: 5s + retries: 5 + + # Create the warehouse bucket before HMS starts + minio-init: + image: minio/mc:latest + container_name: trino-minio-init + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c " + mc alias set local http://minio:9000 minioadmin minioadmin; + mc mb local/warehouse --ignore-existing; + echo 'bucket warehouse ready'; + exit 0; + " + + # ── Hive Metastore ──────────────────────────────────────────────────────── + # naushadh/hive-metastore is a minimal, pre-configured HMS image + # that supports S3-compatible storage via env vars. + metastore: + image: naushadh/hive-metastore:latest + container_name: trino-metastore + depends_on: + postgres: + condition: service_healthy + minio: + condition: service_healthy + minio-init: + condition: service_completed_successfully + ports: + - "9083:9083" + environment: + DATABASE_HOST: postgres + DATABASE_DB: metastore + DATABASE_USER: hive + DATABASE_PASSWORD: hive + # S3 / MinIO + S3_ENDPOINT_URL: http://minio:9000 + S3_BUCKET: warehouse + S3_PREFIX: / + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + REGION: us-east-1 + # No nc/curl in this image; use bash's /dev/tcp built-in + healthcheck: + test: ["CMD", "/bin/bash", "-c", "exec 3<>/dev/tcp/localhost/9083 2>/dev/null && exit 0 || exit 1"] + interval: 15s + timeout: 10s + retries: 15 + + # ── Trino ───────────────────────────────────────────────────────────────── + trino: + image: trinodb/trino:435 + container_name: trino + depends_on: + metastore: + condition: service_healthy + minio: + condition: service_healthy + ports: + - "8080:8080" + volumes: + - ./trino/catalog:/etc/trino/catalog + - ./trino/config.properties:/etc/trino/config.properties + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/v1/info"] + interval: 15s + timeout: 10s + retries: 10 + +volumes: + postgres-data: + minio-data: diff --git a/trino-setup/pom.xml b/trino-setup/pom.xml new file mode 100644 index 000000000..645bb3241 --- /dev/null +++ b/trino-setup/pom.xml @@ -0,0 +1,77 @@ + + + 4.0.0 + + org.apache.wayang + trino-setup + 1.0-SNAPSHOT + jar + + Trino Local Setup — Integration Tests + + Standalone integration tests for a local Trino stack + (Trino + Nessie Iceberg catalog + MinIO S3 storage). + Independent of the Wayang codebase. + + + + 17 + 17 + UTF-8 + 435 + 5.10.2 + + + + + + io.trino + trino-jdbc + ${trino.version} + test + + + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + + org.slf4j + slf4j-simple + 2.0.12 + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + + ${skipIntegrationTests} + + + + + + + + + integration + + false + + + + diff --git a/trino-setup/scripts/init.sql b/trino-setup/scripts/init.sql new file mode 100644 index 000000000..31a99d020 --- /dev/null +++ b/trino-setup/scripts/init.sql @@ -0,0 +1,50 @@ +-- Run this after the stack is up to create sample Iceberg tables. +-- Usage: ./scripts/run-init.sh +-- Or manually: docker exec -it trino trino < /scripts/init.sql + +-- ── Schema ──────────────────────────────────────────────────────────────── +CREATE SCHEMA IF NOT EXISTS iceberg.sales; + +-- ── Orders table (Iceberg / Parquet on MinIO) ───────────────────────────── +CREATE TABLE IF NOT EXISTS iceberg.sales.orders ( + order_id BIGINT, + region VARCHAR, + product VARCHAR, + amount DOUBLE, + order_date DATE +) +WITH (format = 'PARQUET'); + +-- ── Idempotent seed: clear before inserting so re-runs don't duplicate rows ─ +DELETE FROM iceberg.sales.orders; + +-- ── Sample data: 20 rows, 4 regions (AMER/APAC/EMEA/LATAM), 5 products ──── +-- AMER rows: 3, 6, 9, 12, 16 → 5 rows for filter demo +-- Projection demo selects only: region, product, amount +INSERT INTO iceberg.sales.orders VALUES + (1, 'APAC', 'Widget A', 1500.00, DATE '2024-01-15'), + (2, 'EMEA', 'Widget B', 800.50, DATE '2024-01-16'), + (3, 'AMER', 'Widget A', 2200.00, DATE '2024-01-17'), + (4, 'APAC', 'Widget C', 350.75, DATE '2024-01-18'), + (5, 'EMEA', 'Widget A', 1100.00, DATE '2024-01-19'), + (6, 'AMER', 'Widget B', 950.25, DATE '2024-01-20'), + (7, 'APAC', 'Widget B', 1750.00, DATE '2024-01-21'), + (8, 'EMEA', 'Widget C', 420.00, DATE '2024-01-22'), + (9, 'AMER', 'Widget C', 680.50, DATE '2024-01-23'), + (10, 'APAC', 'Widget A', 3000.00, DATE '2024-01-24'), + (11, 'LATAM', 'Widget D', 560.00, DATE '2024-01-25'), + (12, 'AMER', 'Widget D', 1320.75, DATE '2024-01-26'), + (13, 'EMEA', 'Widget D', 990.00, DATE '2024-01-27'), + (14, 'LATAM', 'Widget E', 2100.50, DATE '2024-01-28'), + (15, 'APAC', 'Widget E', 4500.00, DATE '2024-01-29'), + (16, 'AMER', 'Widget E', 3750.00, DATE '2024-01-30'), + (17, 'EMEA', 'Widget E', 1250.00, DATE '2024-01-31'), + (18, 'LATAM', 'Widget A', 870.25, DATE '2024-02-01'), + (19, 'APAC', 'Widget D', 1680.00, DATE '2024-02-02'), + (20, 'LATAM', 'Widget B', 440.50, DATE '2024-02-03'); + +-- ── Verify ──────────────────────────────────────────────────────────────── +SELECT region, COUNT(*) AS order_count, SUM(amount) AS total_amount +FROM iceberg.sales.orders +GROUP BY region +ORDER BY total_amount DESC; diff --git a/trino-setup/scripts/run-init.sh b/trino-setup/scripts/run-init.sh new file mode 100644 index 000000000..c7b423d04 --- /dev/null +++ b/trino-setup/scripts/run-init.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Runs init.sql against the local Trino instance. +# The stack must be fully up before running this. + +set -e + +TRINO_HOST=${TRINO_HOST:-localhost} +TRINO_PORT=${TRINO_PORT:-8080} + +echo "Waiting for Trino to be ready..." +until curl -sf "http://${TRINO_HOST}:${TRINO_PORT}/v1/info" | grep -q '"starting":false'; do + echo " Trino not ready yet, retrying in 5s..." + sleep 5 +done +echo "Trino is ready." + +echo "Running init.sql..." +docker exec -i trino trino \ + --server "http://${TRINO_HOST}:${TRINO_PORT}" \ + --user admin \ + < "$(dirname "$0")/init.sql" + +echo "Done. Sample Iceberg data loaded into iceberg.sales.orders" diff --git a/trino-setup/src/test/java/org/apache/wayang/trino/TrinoIntegrationTest.java b/trino-setup/src/test/java/org/apache/wayang/trino/TrinoIntegrationTest.java new file mode 100644 index 000000000..6adef3c15 --- /dev/null +++ b/trino-setup/src/test/java/org/apache/wayang/trino/TrinoIntegrationTest.java @@ -0,0 +1,215 @@ +package org.apache.wayang.trino; + +import org.junit.jupiter.api.*; + +import java.sql.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Integration tests for the local Trino stack. + * + * Prerequisites: run `docker-compose up -d` and `./scripts/run-init.sh` first. + * + * Run tests: + * mvn test -Pintegration + * + * Or skip infrastructure setup and run with a custom host: + * TRINO_HOST=localhost TRINO_PORT=8080 mvn test -Pintegration + */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class TrinoIntegrationTest { + + private static final String TRINO_HOST = System.getenv().getOrDefault("TRINO_HOST", "localhost"); + private static final int TRINO_PORT = Integer.parseInt(System.getenv().getOrDefault("TRINO_PORT", "8080")); + private static final String JDBC_URL = String.format("jdbc:trino://%s:%d", TRINO_HOST, TRINO_PORT); + + private static Connection connection; + + // ── Lifecycle ───────────────────────────────────────────────────────── + + @BeforeAll + static void openConnection() throws Exception { + Properties props = new Properties(); + props.setProperty("user", "admin"); // Trino requires a non-empty user + connection = DriverManager.getConnection(JDBC_URL, props); + System.out.printf("Connected to Trino at %s%n", JDBC_URL); + } + + @AfterAll + static void closeConnection() throws Exception { + if (connection != null && !connection.isClosed()) { + connection.close(); + } + } + + // ── Helper ──────────────────────────────────────────────────────────── + + private List> query(String sql) throws SQLException { + List> rows = new ArrayList<>(); + try (Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery(sql)) { + int cols = rs.getMetaData().getColumnCount(); + while (rs.next()) { + List row = new ArrayList<>(); + for (int i = 1; i <= cols; i++) row.add(rs.getObject(i)); + rows.add(row); + } + } + return rows; + } + + // ── Test 1: Basic connectivity ──────────────────────────────────────── + + @Test + @Order(1) + @DisplayName("Trino responds to a simple SELECT 1") + void testConnectivity() throws SQLException { + List> rows = query("SELECT 1"); + assertEquals(1, rows.size()); + assertEquals(1L, ((Number) rows.get(0).get(0)).longValue()); + System.out.println("[PASS] Basic connectivity OK"); + } + + // ── Test 2: TPC-H built-in connector ───────────────────────────────── + + @Test + @Order(2) + @DisplayName("TPC-H tiny catalog: count orders") + void testTpchConnector() throws SQLException { + List> rows = query("SELECT COUNT(*) FROM tpch.tiny.orders"); + long count = ((Number) rows.get(0).get(0)).longValue(); + assertTrue(count > 0, "tpch.tiny.orders should have rows"); + System.out.printf("[PASS] TPC-H tiny.orders has %,d rows%n", count); + } + + @Test + @Order(3) + @DisplayName("TPC-H tiny catalog: top 5 orders by total price") + void testTpchTopOrders() throws SQLException { + List> rows = query(""" + SELECT orderkey, totalprice + FROM tpch.tiny.orders + ORDER BY totalprice DESC + LIMIT 5 + """); + assertEquals(5, rows.size(), "Expected exactly 5 rows"); + System.out.println("[PASS] TPC-H top 5 orders:"); + rows.forEach(r -> System.out.printf(" orderkey=%s totalprice=%s%n", r.get(0), r.get(1))); + } + + // ── Test 4: Iceberg — schema exists ────────────────────────────────── + + @Test + @Order(4) + @DisplayName("Iceberg catalog: schema 'sales' is visible") + void testIcebergSchemaVisible() throws SQLException { + List> rows = query("SHOW SCHEMAS IN iceberg LIKE 'sales'"); + assertFalse(rows.isEmpty(), "Schema 'sales' should exist in iceberg catalog. " + + "Did you run scripts/run-init.sh?"); + System.out.println("[PASS] Iceberg schema 'sales' is visible"); + } + + // ── Test 5: Iceberg — full table scan ──────────────────────────────── + + @Test + @Order(5) + @DisplayName("Iceberg table: select all orders") + void testIcebergSelectAll() throws SQLException { + List> rows = query("SELECT * FROM iceberg.sales.orders ORDER BY order_id"); + assertEquals(20, rows.size(), "Expected 20 rows inserted by init.sql"); + System.out.println("[PASS] Iceberg full scan: 20 rows"); + rows.forEach(r -> System.out.printf(" %s%n", r)); + } + + // ── Test 6: Iceberg — pushdown filter ──────────────────────────────── + + @Test + @Order(6) + @DisplayName("Iceberg table: filter by region = APAC") + void testIcebergFilterByRegion() throws SQLException { + List> rows = query(""" + SELECT order_id, region, amount + FROM iceberg.sales.orders + WHERE region = 'APAC' + ORDER BY order_id + """); + assertFalse(rows.isEmpty(), "Should have APAC orders"); + rows.forEach(r -> assertEquals("APAC", r.get(1), "All rows must be APAC")); + System.out.printf("[PASS] Filter pushdown: %d APAC rows%n", rows.size()); + } + + // ── Test 7: Iceberg — aggregation ──────────────────────────────────── + + @Test + @Order(7) + @DisplayName("Iceberg table: aggregate total_amount by region") + void testIcebergAggregate() throws SQLException { + List> rows = query(""" + SELECT region, COUNT(*) AS order_count, SUM(amount) AS total_amount + FROM iceberg.sales.orders + GROUP BY region + ORDER BY total_amount DESC + """); + assertFalse(rows.isEmpty(), "Aggregation should return rows"); + System.out.println("[PASS] Aggregation by region:"); + rows.forEach(r -> System.out.printf(" region=%-5s count=%s total=%.2f%n", + r.get(0), r.get(1), ((Number) r.get(2)).doubleValue())); + } + + // ── Test 8: Iceberg — amount threshold filter ───────────────────────── + + @Test + @Order(8) + @DisplayName("Iceberg table: filter orders with amount > 1000") + void testIcebergFilterByAmount() throws SQLException { + List> rows = query(""" + SELECT order_id, amount + FROM iceberg.sales.orders + WHERE amount > 1000.0 + ORDER BY amount DESC + """); + rows.forEach(r -> assertTrue( + ((Number) r.get(1)).doubleValue() > 1000.0, + "All rows must have amount > 1000" + )); + System.out.printf("[PASS] Amount filter: %d rows with amount > 1000%n", rows.size()); + } + + // ── Test 9: Iceberg — projection (select subset of columns) ─────────── + + @Test + @Order(9) + @DisplayName("Iceberg table: project only region and product columns") + void testIcebergProjection() throws SQLException { + List> rows = query(""" + SELECT region, product + FROM iceberg.sales.orders + LIMIT 5 + """); + assertEquals(5, rows.size()); + rows.forEach(r -> { + assertNotNull(r.get(0), "region should not be null"); + assertNotNull(r.get(1), "product should not be null"); + }); + System.out.println("[PASS] Projection (region, product): 5 rows returned"); + } + + // ── Test 10: Iceberg metadata — table files ────────────────────────── + + @Test + @Order(10) + @DisplayName("Iceberg metadata: $files table lists at least one Parquet file") + void testIcebergFilesMetadata() throws SQLException { + List> rows = query(""" + SELECT file_path, record_count + FROM iceberg.sales."orders$files" + """); + assertFalse(rows.isEmpty(), "There should be at least one data file after inserts"); + System.out.println("[PASS] Iceberg $files metadata:"); + rows.forEach(r -> System.out.printf(" %s (records=%s)%n", r.get(0), r.get(1))); + } +} diff --git a/trino-setup/trino/catalog/iceberg.properties b/trino-setup/trino/catalog/iceberg.properties new file mode 100644 index 000000000..409449431 --- /dev/null +++ b/trino-setup/trino/catalog/iceberg.properties @@ -0,0 +1,16 @@ +# Iceberg catalog backed by Hive Metastore (HMS) + MinIO (S3 storage). +# HMS stores Iceberg table metadata; Trino writes Parquet files to MinIO via S3. +connector.name=iceberg +iceberg.catalog.type=hive_metastore +hive.metastore.uri=thrift://metastore:9083 + +# Native S3 filesystem — handles both s3:// and s3a:// (which HMS uses internally). +# This avoids the "No FileSystem for scheme s3" error when HMS assigns locations. +fs.native-s3.enabled=true +s3.endpoint=http://minio:9000 +s3.path-style-access=true +s3.aws-access-key=minioadmin +s3.aws-secret-key=minioadmin +s3.region=us-east-1 + +iceberg.file-format=PARQUET diff --git a/trino-setup/trino/catalog/tpch.properties b/trino-setup/trino/catalog/tpch.properties new file mode 100644 index 000000000..d402bf8d7 --- /dev/null +++ b/trino-setup/trino/catalog/tpch.properties @@ -0,0 +1,4 @@ +# Built-in TPC-H connector — no external dependencies. +# Useful for testing basic Trino connectivity without any storage setup. +# Usage: SELECT * FROM tpch.tiny.orders LIMIT 10; +connector.name=tpch diff --git a/trino-setup/trino/config.properties b/trino-setup/trino/config.properties new file mode 100644 index 000000000..0b4b617ce --- /dev/null +++ b/trino-setup/trino/config.properties @@ -0,0 +1,4 @@ +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +discovery.uri=http://localhost:8080 diff --git a/wayang-platforms/pom.xml b/wayang-platforms/pom.xml index 6a852c165..826f42404 100644 --- a/wayang-platforms/pom.xml +++ b/wayang-platforms/pom.xml @@ -43,6 +43,7 @@ wayang-giraph wayang-flink wayang-generic-jdbc + wayang-trino wayang-tensorflow diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java index 4f0d196b8..0339fb03b 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java @@ -116,7 +116,9 @@ public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, fin )); } - sb.append(';'); + // Intentionally no trailing ';'. A trailing semicolon is unnecessary for a + // single-statement JDBC executeQuery and is rejected by strict SQL parsers + // such as Trino and BigQuery. Postgres/SQLite/HSQLDB accept its absence. return sb; } @@ -157,13 +159,16 @@ protected static Tuple2 createSqlQuery(final E } else if (operator instanceof JdbcProjectionOperator) { assert projectionTask == null; // Allow one projection operator per stage for now. projectionTask = (JdbcProjectionOperator) operator; - } else if (operator instanceof final JdbcGlobalReduceOperator globalReduce) { + } else if (operator instanceof JdbcGlobalReduceOperator) { + final JdbcGlobalReduceOperator globalReduce = (JdbcGlobalReduceOperator) operator; assert globalReduceTask == null; // Allow one projection operator per stage for now. globalReduceTask = globalReduce; - } else if (operator instanceof final JdbcReduceByOperator reduceBy) { + } else if (operator instanceof JdbcReduceByOperator) { + final JdbcReduceByOperator reduceBy = (JdbcReduceByOperator) operator; assert reduceByTask == null; // Allow one projection operator per stage for now. reduceByTask = reduceBy; - } else if (operator instanceof final JdbcSortOperator sort) { + } else if (operator instanceof JdbcSortOperator) { + final JdbcSortOperator sort = (JdbcSortOperator) operator; assert sortTask == null; // Allow one projection operator per stage for now. sortTask = sort; } else if (operator instanceof JoinOperator || (operator instanceof SpatialJoinOperator)) { @@ -219,12 +224,15 @@ private static void executeSinkStage(final ExecutionStage stage, final Optimizat // Walk through intermediate operators, stopping at the sink ExecutionTask nextTask = JdbcExecutor.findJdbcExecutionOperatorTaskInStage(startTask, stage); while (nextTask != null && !(nextTask.getOperator() instanceof JdbcTableSinkOperator)) { - if (nextTask.getOperator() instanceof final JdbcFilterOperator filterOperator) { + if (nextTask.getOperator() instanceof JdbcFilterOperator) { + final JdbcFilterOperator filterOperator = (JdbcFilterOperator) nextTask.getOperator(); filterTasks.add(filterOperator); - } else if (nextTask.getOperator() instanceof final JdbcProjectionOperator projectionOperator) { + } else if (nextTask.getOperator() instanceof JdbcProjectionOperator) { + final JdbcProjectionOperator projectionOperator = (JdbcProjectionOperator) nextTask.getOperator(); assert projectionTask == null; projectionTask = projectionOperator; - } else if (nextTask.getOperator() instanceof final JdbcJoinOperator joinOperator) { + } else if (nextTask.getOperator() instanceof JdbcJoinOperator) { + final JdbcJoinOperator joinOperator = (JdbcJoinOperator) nextTask.getOperator(); joinTasks.add(joinOperator); } else { throw new WayangException(String.format("Unsupported JDBC execution task %s", nextTask.toString())); diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcTableSource.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcTableSource.java index 2d546deec..4d7096649 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcTableSource.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcTableSource.java @@ -81,7 +81,8 @@ public CardinalityEstimate estimate(OptimizationContext optimizationContext, Car .createJdbcConnection()) { // Query the table cardinality. - final String sql = String.format("SELECT count(*) FROM %s;", JdbcTableSource.this.getTableName()); + // No trailing ';' — strict parsers (Trino, BigQuery) reject it in executeQuery. + final String sql = String.format("SELECT count(*) FROM %s", JdbcTableSource.this.getTableName()); final ResultSet resultSet = connection.createStatement().executeQuery(sql); if (!resultSet.next()) { throw new SQLException("No query result for \"" + sql + "\"."); diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java index 0dfd8b698..8f7b3d8a2 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java @@ -81,7 +81,7 @@ void testExecuteWithPlainTableSource() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT * FROM customer;", + "SELECT * FROM customer", sqlQueryChannelInstance.getSqlQuery() ); } @@ -130,7 +130,7 @@ void testExecuteWithFilter() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT * FROM customer WHERE age >= 18;", + "SELECT * FROM customer WHERE age >= 18", sqlQueryChannelInstance.getSqlQuery() ); } @@ -172,7 +172,7 @@ void testExecuteWithProjection() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT name, age FROM customer;", + "SELECT name, age FROM customer", sqlQueryChannelInstance.getSqlQuery() ); } @@ -240,7 +240,7 @@ void testExecuteWithProjectionAndFilters() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT name, age FROM customer WHERE age >= 18 AND name IS NOT NULL;", + "SELECT name, age FROM customer WHERE age >= 18 AND name IS NOT NULL", sqlQueryChannelInstance.getSqlQuery() ); } diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcGlobalReduceOperatorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcGlobalReduceOperatorTest.java index b5ccb0848..8ea9c6656 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcGlobalReduceOperatorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcGlobalReduceOperatorTest.java @@ -62,7 +62,7 @@ void testWithHsqldb() throws SQLException { final ExecutionStage sqlStage = mock(ExecutionStage.class); final JdbcTableSource tableSourceA = new HsqldbTableSource("testA"); - + final ExecutionTask tableSourceATask = new ExecutionTask(tableSourceA); tableSourceATask.setOutputChannel(0, new SqlQueryChannel(sqlChannelDescriptor, tableSourceA.getOutput(0))); tableSourceATask.setStage(sqlStage); @@ -86,23 +86,14 @@ void testWithHsqldb() throws SQLException { globalReduceTask.getOutputChannel(0).addConsumer(sqlToStreamTask, 0); sqlToStreamTask.setStage(nextStage); - - final HsqldbPlatform hsqldbPlatform = new HsqldbPlatform(); - - try (Connection jdbcConnection = hsqldbPlatform.createDatabaseDescriptor(configuration).createJdbcConnection()) { - final Statement statement = jdbcConnection.createStatement(); - statement.execute("CREATE TABLE IF NOT EXISTS testA (a INT, b VARCHAR(6));"); - statement.execute("INSERT INTO testA VALUES (0, 'zero');"); - statement.execute("CREATE TABLE IF NOT EXISTS testB (a INT, b INT);"); - statement.execute("INSERT INTO testB VALUES (0, 100);"); - } - final JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); executor.execute(sqlStage, new DefaultOptimizationContext(job), job.getCrossPlatformExecutor()); final SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor() .getChannelInstance(sqlToStreamTask.getInputChannel(0)); + final HsqldbPlatform hsqldbPlatform = new HsqldbPlatform(); + try (Connection jdbcConnection = hsqldbPlatform.createDatabaseDescriptor(configuration).createJdbcConnection()) { final Statement statement = jdbcConnection.createStatement(); final java.sql.ResultSet resultSet = statement.executeQuery(sqlQueryChannelInstance.getSqlQuery()); @@ -112,6 +103,6 @@ void testWithHsqldb() throws SQLException { assertTrue(count > 0); } - assertEquals("SELECT COUNT(*) FROM testA;", sqlQueryChannelInstance.getSqlQuery()); + assertEquals("SELECT COUNT(*) FROM testA", sqlQueryChannelInstance.getSqlQuery()); } } diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java index ab984a751..57d181791 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java @@ -119,7 +119,7 @@ void testWithHsqldb() throws SQLException { System.out.println(); assertEquals( - "SELECT * FROM testA JOIN testB ON testB.a=testA.a;", + "SELECT * FROM testA JOIN testB ON testB.a=testA.a", sqlQueryChannelInstance.getSqlQuery() ); } @@ -197,7 +197,7 @@ void testMultiConditionJoinWithHsqldb() throws SQLException { String generatedSql = sqlQueryChannelInstance.getSqlQuery(); assertEquals( - "SELECT * FROM orders JOIN shipments ON orders.order_id=shipments.order_id AND orders.customer_id=shipments.customer_id;", + "SELECT * FROM orders JOIN shipments ON orders.order_id=shipments.order_id AND orders.customer_id=shipments.customer_id", generatedSql ); diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcReduceByOperatorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcReduceByOperatorTest.java index f00f4020e..556224027 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcReduceByOperatorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcReduceByOperatorTest.java @@ -91,6 +91,6 @@ void testWithHsqldb() throws SQLException { final SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor() .getChannelInstance(sqlToStreamTask.getInputChannel(0)); - assertEquals("SELECT col0,COUNT(*) FROM testA GROUP BY col0;", sqlQueryChannelInstance.getSqlQuery()); + assertEquals("SELECT col0,COUNT(*) FROM testA GROUP BY col0", sqlQueryChannelInstance.getSqlQuery()); } } diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcSortOperatorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcSortOperatorTest.java index 118fb7efa..1dc2fe12f 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcSortOperatorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcSortOperatorTest.java @@ -86,6 +86,6 @@ void testWithHsqldb() throws SQLException { final SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor() .getChannelInstance(sqlToStreamTask.getInputChannel(0)); - assertEquals("SELECT * FROM testA ORDER BY col0 DESC;", sqlQueryChannelInstance.getSqlQuery()); + assertEquals("SELECT * FROM testA ORDER BY col0 DESC", sqlQueryChannelInstance.getSqlQuery()); } } diff --git a/wayang-platforms/wayang-trino/pom.xml b/wayang-platforms/wayang-trino/pom.xml new file mode 100644 index 000000000..7f4d44695 --- /dev/null +++ b/wayang-platforms/wayang-trino/pom.xml @@ -0,0 +1,74 @@ + + + + 4.0.0 + + + wayang-platforms + org.apache.wayang + 1.1.2-SNAPSHOT + + + wayang-trino + + Wayang Platform Trino + + Wayang implementation of the operators to be working with the platform "Trino" + + + + org.apache.wayang.platform.trino + 435 + + + + + io.trino + trino-jdbc + ${trino.version} + + + org.apache.wayang + wayang-basic + 1.1.2-SNAPSHOT + + + org.apache.wayang + wayang-jdbc-template + 1.1.2-SNAPSHOT + + + org.apache.wayang + wayang-spark + 1.1.2-SNAPSHOT + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + + + diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/Trino.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/Trino.java new file mode 100644 index 000000000..46ab325b9 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/Trino.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino; + + +import org.apache.wayang.trino.platform.TrinoPlatform; +import org.apache.wayang.trino.plugin.TrinoConversionsPlugin; +import org.apache.wayang.trino.plugin.TrinoPlugin; + +/** + * Register for relevant components of this module. + */ +public class Trino { + + private final static TrinoPlugin PLUGIN = new TrinoPlugin(); + + private final static TrinoConversionsPlugin CONVERSIONS_PLUGIN = new TrinoConversionsPlugin(); + + /** + * Retrieve the {@link TrinoPlugin}. + * + * @return the {@link TrinoPlugin} + */ + public static TrinoPlugin plugin() { + return PLUGIN; + } + + /** + * Retrieve the {@link TrinoConversionsPlugin}. + * + * @return the {@link TrinoConversionsPlugin} + */ + public static TrinoConversionsPlugin conversionPlugin() { + return CONVERSIONS_PLUGIN; + } + + + /** + * Retrieve the {@link TrinoPlatform}. + * + * @return the {@link TrinoPlatform} + */ + public static TrinoPlatform platform() { + return TrinoPlatform.getInstance(); + } + +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/TrinoDemo.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/TrinoDemo.java new file mode 100644 index 000000000..9becd061d --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/TrinoDemo.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.LocalCallbackSink; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.basic.types.RecordType; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.java.Java; +import org.apache.wayang.trino.operators.TrinoTableSource; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +/** + * Standalone demo for the Wayang Trino connector. + * + *

Demonstrates two Trino operator types: + *

    + *
  1. Seg 3 — Filter pushdown: WHERE region = 'AMER'.
    2. + *
  2. Seg 4 — Projection + Filter pushdown: + * SELECT region, product, amount ... WHERE region = 'AMER'.
    3. + *
+ * + *

Run with: + * 

+ *   cd /path/to/wayang
+ *   mvn exec:java -pl wayang-platforms/wayang-trino \
+ *     -Dexec.mainClass=org.apache.wayang.trino.TrinoDemo \
+ *     -Pskip-prerequisite-check -Drat.skip=true \
+ *     [-Dtrino.url=jdbc:trino://localhost:8080] [-Dtrino.user=admin]
+ *
+ */ +public class TrinoDemo { + + private static final String JDBC_URL = System.getProperty("trino.url", "jdbc:trino://localhost:8080"); + private static final String JDBC_USER = System.getProperty("trino.user", "admin"); + + public static void main(String[] args) throws Exception { + seg3Filter(); + seg4Projection(); + } + + // ── Seg 3 — Filter pushdown ─────────────────────────────────────────────── + + static void seg3Filter() throws Exception { + System.out.println("══════════════════════════════════════════════════════"); + System.out.println(" Seg 3 — Filter Operator Pushdown"); + System.out.println("══════════════════════════════════════════════════════"); + System.out.println(); + System.out.println(" Operator: FilterOperator -> TrinoFilterOperator"); + System.out.println(" SQL sent: SELECT * FROM iceberg.sales.orders"); + System.out.println(" WHERE region = 'AMER'"); + System.out.println(); + + Configuration config = buildConfig(); + WayangContext wayang = buildWayang(config); + + List results = new ArrayList<>(); + TrinoTableSource source = new TrinoTableSource( + "iceberg.sales.orders", "order_id", "region", "product", "amount", "order_date" + ); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + r -> "AMER".equals(r.getField(1)), Record.class + ).withSqlImplementation("region = 'AMER'") + ); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + source.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + + wayang.execute("Trino-Filter-Demo", new WayangPlan(sink)); + + System.out.println(" Results returned by Wayang:"); + System.out.printf(" %-10s %-6s %-10s %10s %-12s%n", + "order_id", "region", "product", "amount", "order_date"); + System.out.println(" " + repeat('-', 54)); + for (Record r : results) { + System.out.printf(" %-10s %-6s %-10s %10s %-12s%n", + r.getField(0), r.getField(1), r.getField(2), r.getField(3), r.getField(4)); + } + System.out.println(); + System.out.printf(" ✓ %d AMER rows — filter pushed to Trino as SQL WHERE clause%n", results.size()); + + verifyInQueryHistory("iceberg.sales.orders"); + + System.out.println("══════════════════════════════════════════════════════"); + System.out.println(); + } + + // ── Seg 4 — Projection + Filter pushdown ───────────────────────────────── + + static void seg4Projection() throws Exception { + System.out.println("══════════════════════════════════════════════════════"); + System.out.println(" Seg 4 — Projection Operator Pushdown"); + System.out.println("══════════════════════════════════════════════════════"); + System.out.println(); + System.out.println(" Operators: FilterOperator -> TrinoFilterOperator"); + System.out.println(" MapOperator -> TrinoProjectionOperator"); + System.out.println(" SQL sent: SELECT region, product, amount"); + System.out.println(" FROM iceberg.sales.orders"); + System.out.println(" WHERE region = 'AMER'"); + System.out.println(); + System.out.println(" Both operators get pushed into a single SQL query —"); + System.out.println(" no unnecessary columns are transferred over the network."); + System.out.println(); + + Configuration config = buildConfig(); + WayangContext wayang = buildWayang(config); + + List results = new ArrayList<>(); + TrinoTableSource source = new TrinoTableSource( + "iceberg.sales.orders", "order_id", "region", "product", "amount", "order_date" + ); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + r -> "AMER".equals(r.getField(1)), Record.class + ).withSqlImplementation("region = 'AMER'") + ); + // Use the Record-aware projection (multi-field). The plain + // ProjectionDescriptor(Class, Class, fields...) builds a POJO projection + // whose Java implementation only supports a single field; createForRecords + // yields a multi-field Record implementation that also works if Wayang + // executes the projection on the Java side instead of pushing it to Trino. + MapOperator projection = new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType("order_id", "region", "product", "amount", "order_date"), + "region", "product", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class) + ); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + source.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + + wayang.execute("Trino-Projection-Demo", new WayangPlan(sink)); + + System.out.println(" Results returned by Wayang (projected columns only):"); + System.out.printf(" %-6s %-10s %10s%n", "region", "product", "amount"); + System.out.println(" " + repeat('-', 30)); + for (Record r : results) { + System.out.printf(" %-6s %-10s %10s%n", r.getField(0), r.getField(1), r.getField(2)); + } + System.out.println(); + System.out.printf(" ✓ %d AMER rows — only 3 of 5 columns fetched (projection pushed to SQL)%n", + results.size()); + + verifyInQueryHistory("iceberg.sales.orders"); + + System.out.println("══════════════════════════════════════════════════════"); + System.out.println(); + } + + // ── Shared helpers ──────────────────────────────────────────────────────── + + private static Configuration buildConfig() { + Configuration config = new Configuration(); + config.setProperty("wayang.trino.jdbc.url", JDBC_URL); + config.setProperty("wayang.trino.jdbc.user", JDBC_USER); + config.setProperty("wayang.trino.jdbc.password", ""); + return config; + } + + private static WayangContext buildWayang(Configuration config) { + return new WayangContext(config) + .withPlugin(Java.basicPlugin()) + .withPlugin(Trino.plugin()); + } + + private static void verifyInQueryHistory(String tableHint) throws Exception { + System.out.println(); + System.out.println(" Checking Trino's system.runtime.queries for proof..."); + Properties props = new Properties(); + props.setProperty("user", JDBC_USER); + try (Connection conn = DriverManager.getConnection(JDBC_URL, props)) { + ResultSet rs = conn.createStatement().executeQuery( + "SELECT query FROM system.runtime.queries " + + "WHERE state = 'FINISHED' AND query LIKE '%" + tableHint + "%' " + + "ORDER BY created DESC LIMIT 2" + ); + System.out.println(); + System.out.println(" Last SQL Trino executed:"); + while (rs.next()) { + System.out.println(" > " + rs.getString(1).replaceAll("\\s+", " ")); + } + } + System.out.println(); + System.out.println(" ✓ Wayang-assembled SQL confirmed in Trino query history."); + } + + private static String repeat(char c, int n) { + StringBuilder sb = new StringBuilder(n); + for (int i = 0; i < n; i++) sb.append(c); + return sb.toString(); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/channels/ChannelConversions.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/channels/ChannelConversions.java new file mode 100644 index 000000000..ed8299cad --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/channels/ChannelConversions.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.channels; + +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.optimizer.channels.DefaultChannelConversion; +import org.apache.wayang.java.channels.StreamChannel; +import org.apache.wayang.jdbc.operators.SqlToRddOperator; +import org.apache.wayang.jdbc.operators.SqlToStreamOperator; +import org.apache.wayang.spark.channels.RddChannel; +import org.apache.wayang.trino.platform.TrinoPlatform; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Register for the {@link ChannelConversion}s supported for this platform. + */ +public class ChannelConversions { + + public static final ChannelConversion SQL_TO_STREAM_CONVERSION = new DefaultChannelConversion( + TrinoPlatform.getInstance().getSqlQueryChannelDescriptor(), + StreamChannel.DESCRIPTOR, + () -> new SqlToStreamOperator(TrinoPlatform.getInstance()) + ); + + public static final ChannelConversion SQL_TO_UNCACHED_RDD_CONVERSION = new DefaultChannelConversion( + TrinoPlatform.getInstance().getSqlQueryChannelDescriptor(), + RddChannel.UNCACHED_DESCRIPTOR, + () -> new SqlToRddOperator(TrinoPlatform.getInstance()) + ); + + public static final Collection ALL = Arrays.asList( + SQL_TO_STREAM_CONVERSION, + SQL_TO_UNCACHED_RDD_CONVERSION + ); + +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/FilterMapping.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/FilterMapping.java new file mode 100644 index 000000000..0b95a7335 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/FilterMapping.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.trino.operators.TrinoFilterOperator; +import org.apache.wayang.trino.platform.TrinoPlatform; + +import java.util.Collection; +import java.util.Collections; + + +/** + * Mapping from {@link FilterOperator} to {@link TrinoFilterOperator}. + */ +@SuppressWarnings("unchecked") +public class FilterMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + TrinoPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "filter", new FilterOperator<>(null, DataSetType.createDefault(Record.class)), false + ).withAdditionalTest(op -> op.getPredicateDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new TrinoFilterOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/JoinMapping.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/JoinMapping.java new file mode 100644 index 000000000..16e406fca --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/JoinMapping.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.trino.operators.TrinoJoinOperator; +import org.apache.wayang.trino.platform.TrinoPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link JoinOperator} to {@link TrinoJoinOperator}. + */ +@SuppressWarnings("unchecked") +public class JoinMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + TrinoPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + OperatorPattern> operatorPattern = new OperatorPattern<>( + "join", + new JoinOperator( + null, + null, + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class) + ), + false + ) + .withAdditionalTest(op -> op.getKeyDescriptor0() instanceof TransformationDescriptor) + .withAdditionalTest(op -> op.getKeyDescriptor1() instanceof TransformationDescriptor) + .withAdditionalTest(op -> op.getKeyDescriptor0().getSqlImplementation() != null) + .withAdditionalTest(op -> op.getKeyDescriptor1().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> { + return new TrinoJoinOperator(matchedOperator).at(epoch); + } + ); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java new file mode 100644 index 000000000..1ffd69491 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.mapping; + +import org.apache.wayang.core.mapping.Mapping; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Register for the {@link Mapping}s supported for this platform. + */ +public class Mappings { + + public static final Collection ALL = Arrays.asList( + new FilterMapping(), + new JoinMapping(), + new ProjectionMapping() + ); + +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/ProjectionMapping.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/ProjectionMapping.java new file mode 100644 index 000000000..95e2338af --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/ProjectionMapping.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.trino.operators.TrinoProjectionOperator; +import org.apache.wayang.trino.platform.TrinoPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link MapOperator} to {@link TrinoProjectionOperator}. + */ +public class ProjectionMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + TrinoPlatform.getInstance())); + } + + private SubplanPattern createSubplanPattern() { + OperatorPattern> operatorPattern = new OperatorPattern<>( + "projection", + new MapOperator<>( + null, + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(op -> op.getFunctionDescriptor() instanceof ProjectionDescriptor) + .withAdditionalTest(op -> op.getNumInputs() == 1); // No broadcasts. + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new TrinoProjectionOperator(matchedOperator).at(epoch)); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoExecutionOperator.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoExecutionOperator.java new file mode 100644 index 000000000..2e2f27b21 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoExecutionOperator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.operators; + +import org.apache.wayang.jdbc.operators.JdbcExecutionOperator; +import org.apache.wayang.trino.platform.TrinoPlatform; + +public interface TrinoExecutionOperator extends JdbcExecutionOperator { + + @Override + default TrinoPlatform getPlatform() { + return TrinoPlatform.getInstance(); + } + +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoFilterOperator.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoFilterOperator.java new file mode 100644 index 000000000..48c0ea4a7 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoFilterOperator.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.jdbc.operators.JdbcFilterOperator; + + +/** + * Trino implementation of the {@link FilterOperator}. + */ +public class TrinoFilterOperator extends JdbcFilterOperator implements TrinoExecutionOperator { + + /** + * Creates a new instance. + */ + public TrinoFilterOperator(PredicateDescriptor predicateDescriptor) { + super(predicateDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public TrinoFilterOperator(FilterOperator that) { + super(that); + } + + @Override + protected TrinoFilterOperator createCopy() { + return new TrinoFilterOperator(this); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoJoinOperator.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoJoinOperator.java new file mode 100644 index 000000000..6937cac79 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoJoinOperator.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.jdbc.operators.JdbcJoinOperator; + + +/** + * Trino implementation of the {@link JoinOperator}. + */ +public class TrinoJoinOperator extends JdbcJoinOperator implements TrinoExecutionOperator { + + /** + * Creates a new instance. + */ + public TrinoJoinOperator( + TransformationDescriptor keyDescriptor0, + TransformationDescriptor keyDescriptor1) { + super(keyDescriptor0,keyDescriptor1); + } + + public TrinoJoinOperator(JoinOperator that) { + super(that); + } + + @Override + protected TrinoJoinOperator createCopy() { + return new TrinoJoinOperator(this); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoProjectionOperator.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoProjectionOperator.java new file mode 100644 index 000000000..f567fe86f --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoProjectionOperator.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.jdbc.operators.JdbcProjectionOperator; + +/** + * Trino implementation of the {@link FilterOperator}. + */ +public class TrinoProjectionOperator extends JdbcProjectionOperator implements TrinoExecutionOperator { + + public TrinoProjectionOperator(String... fieldNames) { + super(fieldNames); + } + + public TrinoProjectionOperator(ProjectionDescriptor functionDescriptor) { + super(functionDescriptor); + } + + public TrinoProjectionOperator(MapOperator that) { + super(that); + } + + @Override + protected TrinoProjectionOperator createCopy() { + return new TrinoProjectionOperator(this); + } + +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoTableSource.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoTableSource.java new file mode 100644 index 000000000..cae67530d --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoTableSource.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.operators; + +import org.apache.wayang.basic.operators.TableSource; +import org.apache.wayang.core.platform.ChannelDescriptor; +import org.apache.wayang.jdbc.operators.JdbcTableSource; + +import java.util.List; + +/** + * Trino implementation for the {@link TableSource}. + * + *

Table names use Trino's three-part naming convention: + * {@code catalog.schema.table} (e.g. {@code iceberg.sales.orders}). + */ +public class TrinoTableSource extends JdbcTableSource implements TrinoExecutionOperator { + + /** + * Creates a new instance. + * + * @see TableSource#TableSource(String, String...) + */ + public TrinoTableSource(String tableName, String... columnNames) { + super(tableName, columnNames); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public TrinoTableSource(JdbcTableSource that) { + super(that); + } + + @Override + public List getSupportedInputChannels(int index) { + throw new UnsupportedOperationException("This operator has no input channels."); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/platform/TrinoPlatform.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/platform/TrinoPlatform.java new file mode 100644 index 000000000..ba6ec6111 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/platform/TrinoPlatform.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.platform; + +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.jdbc.platform.JdbcPlatformTemplate; + +/** + * {@link Platform} implementation for Trino. + */ +public class TrinoPlatform extends JdbcPlatformTemplate { + + private static final String PLATFORM_NAME = "Trino"; + + private static final String CONFIG_NAME = "trino"; + + private static TrinoPlatform instance = null; + + public static TrinoPlatform getInstance() { + if (instance == null) { + instance = new TrinoPlatform(); + } + return instance; + } + + protected TrinoPlatform() { + super(PLATFORM_NAME, CONFIG_NAME); + } + + @Override + public String getJdbcDriverClassName() { + return "io.trino.jdbc.TrinoDriver"; + } + +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/plugin/TrinoConversionsPlugin.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/plugin/TrinoConversionsPlugin.java new file mode 100644 index 000000000..b2dedd1a7 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/plugin/TrinoConversionsPlugin.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.plugin; + +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.plan.wayangplan.Operator; +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.plugin.Plugin; +import org.apache.wayang.java.platform.JavaPlatform; +import org.apache.wayang.trino.channels.ChannelConversions; +import org.apache.wayang.trino.platform.TrinoPlatform; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; + +/** + * This {@link Plugin} enables to use some basic Wayang {@link Operator}s on the {@link TrinoPlatform}. + */ +public class TrinoConversionsPlugin implements Plugin { + + @Override + public Collection getRequiredPlatforms() { + return Arrays.asList(TrinoPlatform.getInstance(), JavaPlatform.getInstance()); + } + + @Override + public Collection getMappings() { + return Collections.emptyList(); + } + + @Override + public Collection getChannelConversions() { + return ChannelConversions.ALL; + } + + @Override + public void setProperties(Configuration configuration) { + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/plugin/TrinoPlugin.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/plugin/TrinoPlugin.java new file mode 100644 index 000000000..a04e07c0d --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/plugin/TrinoPlugin.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.plugin; + +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.plan.wayangplan.Operator; +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.plugin.Plugin; +import org.apache.wayang.java.platform.JavaPlatform; +import org.apache.wayang.trino.channels.ChannelConversions; +import org.apache.wayang.trino.mapping.Mappings; +import org.apache.wayang.trino.platform.TrinoPlatform; + +import java.util.Arrays; +import java.util.Collection; + +/** + * This {@link Plugin} enables to use some basic Wayang {@link Operator}s on the {@link TrinoPlatform}. + */ +public class TrinoPlugin implements Plugin { + + @Override + public Collection getRequiredPlatforms() { + return Arrays.asList(TrinoPlatform.getInstance(), JavaPlatform.getInstance()); + } + + @Override + public Collection getMappings() { + return Mappings.ALL; + } + + @Override + public Collection getChannelConversions() { + return ChannelConversions.ALL; + } + + @Override + public void setProperties(Configuration configuration) { + } +} diff --git a/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties b/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties new file mode 100644 index 000000000..651ed564f --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties @@ -0,0 +1,105 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Connection (override per deployment in wayang.properties) +# wayang.trino.jdbc.url = jdbc:trino://localhost:8080 +# wayang.trino.jdbc.user = admin +# wayang.trino.jdbc.password = +wayang.trino.jdbc.driverName = io.trino.jdbc.TrinoDriver + +# Hardware profile used by LoadProfileToTimeConverter. +# Trino runs as a distributed cluster; model a representative coordinator node. +wayang.trino.cpu.mhz = 2700 +wayang.trino.cores = 4 +wayang.trino.costs.fix = 0.0 +wayang.trino.costs.per-ms = 1.0 + +# ── Cost model ──────────────────────────────────────────────────────────────── +# +# Formula: cpu = α * rows + β +# +# Trino is a distributed MPP engine: very low per-row cost (small α) because +# scans are parallelised across workers, but noticeable fixed overhead (larger β) +# from query planning and cluster coordination. +# +# Compared to a single-node source (Postgres α=55, β=380k): +# α = 10 — parallel scan makes per-row cost ~5× cheaper +# β = 800k — cluster startup + query dispatch overhead +# +# These are initial estimates; tune after real benchmarks by fitting the +# template formula below on measured data and updating the 'load' key. +# ────────────────────────────────────────────────────────────────────────────── + +wayang.trino.tablesource.load.template = {\ + "type":"mathex", "in":0, "out":1,\ + "cpu":"?*out0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.trino.tablesource.load = {\ + "in":0, "out":1,\ + "cpu":"${10*out0 + 800000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.trino.filter.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.trino.filter.load = {\ + "in":1, "out":1,\ + "cpu":"${10*in0 + 800000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.trino.projection.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.trino.projection.load = {\ + "in":1, "out":1,\ + "cpu":"${10*in0 + 800000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.trino.sqltostream.load.query.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*out0 + ?"\ +} +wayang.trino.sqltostream.load.query = {\ + "in":1, "out":1,\ + "cpu":"${10*out0 + 800000}",\ + "ram":"0",\ + "p":0.9\ +} +wayang.trino.sqltostream.load.output.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*out0"\ +} +wayang.trino.sqltostream.load.output = {\ + "in":1, "out":1,\ + "cpu":"${10*out0}",\ + "ram":"0",\ + "p":0.9\ +} From 8efa98f3279b84c93631eaea3894773f9601d060 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Sat, 6 Jun 2026 15:59:08 +0800 Subject: [PATCH 02/18] Document Trino platform tests --- trino-setup/README.md | 117 +++++++++++++++++++++++++++++++++--------- 1 file changed, 92 insertions(+), 25 deletions(-) diff --git a/trino-setup/README.md b/trino-setup/README.md index 960c53fec..4133bd9af 100644 --- a/trino-setup/README.md +++ b/trino-setup/README.md @@ -2,6 +2,20 @@ Local Trino environment backed by an **Iceberg** data lake, completely containerised. +The current validation has two parts: + +1. Build the Wayang Trino platform and run the shared JDBC SQL-generation tests. +2. Run JDBC integration tests against the local Trino, Iceberg, and MinIO stack. + +Run the commands below from the repository root. Java 17 and Docker with +Docker Compose are required; Maven is provided by the repository wrapper. + +The pure Trino platform branch is named `wayang-trino`: + +```bash +git checkout wayang-trino +``` + ## Stack | Component | Image | Port | Role | @@ -17,53 +31,104 @@ HMS is the battle-tested Iceberg catalog for Trino. Parquet data files are writt ``` trino-setup/ -├── docker-compose.yml # Full stack definition -├── trino/ -│ ├── config.properties # Trino node config -│ └── catalog/ -│ ├── iceberg.properties # Iceberg via HMS + MinIO -│ └── tpch.properties # Built-in TPC-H (no storage needed) -├── scripts/ -│ ├── init.sql # Creates iceberg.sales.orders + sample rows -│ └── run-init.sh # Helper: waits for Trino then runs init.sql -├── pom.xml # Standalone Maven project (Java 17) -└── src/test/java/.../ - └── TrinoIntegrationTest.java # JUnit 5 integration tests -``` - -## Quick Start +|-- docker-compose.yml # Full stack definition +|-- trino/ +| |-- config.properties # Trino node config +| `-- catalog/ +| |-- iceberg.properties # Iceberg via HMS + MinIO +| `-- tpch.properties # Built-in TPC-H (no storage needed) +|-- scripts/ +| |-- init.sql # Creates iceberg.sales.orders + sample rows +| `-- run-init.sh # Helper: waits for Trino then runs init.sql +|-- pom.xml # Standalone Maven project (Java 17) +`-- src/test/java/.../ + `-- TrinoIntegrationTest.java # JUnit 5 integration tests +``` + +## 1. Test the Wayang Trino Platform + +Build the Trino platform and its required modules: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-trino -am -DskipTests -Drat.skip=true test +``` + +On PowerShell: + +```powershell +.\mvnw.cmd --% -Pskip-prerequisite-check -pl wayang-platforms/wayang-trino -am -DskipTests -Drat.skip=true test +``` + +Then run the shared JDBC SQL-generation tests: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-jdbc-template -am -Dtest=JdbcExecutorTest -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false -Drat.skip=true test +``` + +On PowerShell: + +```powershell +.\mvnw.cmd --% -Pskip-prerequisite-check -pl wayang-platforms/wayang-jdbc-template -am -Dtest=JdbcExecutorTest -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false -Drat.skip=true test +``` + +Expected result: + +```text +Wayang Platform Trino ... SUCCESS +Tests run: 4, Failures: 0, Errors: 0, Skipped: 0 +``` + +## 2. Test the Local Trino Stack ### 1. Start the stack ```bash -cd trino-setup -docker-compose up -d +docker compose -f trino-setup/docker-compose.yml up -d ``` Wait ~30 seconds for all services to become healthy. Check with: ```bash -docker-compose ps +docker compose -f trino-setup/docker-compose.yml ps # or watch the Trino UI at http://localhost:8080 ``` ### 2. Load sample Iceberg data ```bash -./scripts/run-init.sh +bash trino-setup/scripts/run-init.sh +``` + +On PowerShell: + +```powershell +Get-Content -Raw trino-setup/scripts/init.sql | docker exec -i trino trino --server http://localhost:8080 --user admin ``` -This creates the schema `iceberg.sales` and inserts 10 sample orders into +This creates the schema `iceberg.sales` and inserts 20 sample orders into `iceberg.sales.orders` (Parquet files on MinIO). ### 3. Run the integration tests ```bash -mvn test -Pintegration +./mvnw -f trino-setup/pom.xml -Pintegration -Dtest=TrinoIntegrationTest test +``` + +On PowerShell: + +```powershell +.\mvnw.cmd --% -f trino-setup/pom.xml -Pintegration -Dtest=TrinoIntegrationTest test ``` Tests are skipped by default (no `-Pintegration`) to avoid requiring Docker in CI. +Expected result: + +```text +Tests run: 10, Failures: 0, Errors: 0, Skipped: 0 +BUILD SUCCESS +``` + ### 4. Manual exploration Open the **Trino UI**: http://localhost:8080 @@ -94,18 +159,20 @@ Look for Parquet files under `warehouse/sales/orders/`. ### 5. Tear down ```bash -docker-compose down -v # -v removes volumes (clears MinIO data) +docker compose -f trino-setup/docker-compose.yml down -v ``` +The `-v` option removes volumes and clears the local MinIO and PostgreSQL data. + ## Test Coverage | Test | What it checks | |------|----------------| -| `testConnectivity` | `SELECT 1` — JDBC connection works | +| `testConnectivity` | `SELECT 1`, JDBC connection works | | `testTpchConnector` | TPC-H built-in connector, no storage needed | | `testTpchTopOrders` | ORDER BY + LIMIT on TPC-H | | `testIcebergSchemaVisible` | Schema created by `init.sql` is visible | -| `testIcebergSelectAll` | Full table scan, 10 rows | +| `testIcebergSelectAll` | Full table scan, 20 rows | | `testIcebergFilterByRegion` | WHERE pushdown on string column | | `testIcebergAggregate` | GROUP BY + SUM aggregation | | `testIcebergFilterByAmount` | WHERE pushdown on double column | @@ -117,5 +184,5 @@ docker-compose down -v # -v removes volumes (clears MinIO data) Override defaults if running Trino on a different host/port: ```bash -TRINO_HOST=my-trino-host TRINO_PORT=8080 mvn test -Pintegration +TRINO_HOST=my-trino-host TRINO_PORT=8080 ./mvnw -f trino-setup/pom.xml -Pintegration -Dtest=TrinoIntegrationTest test ``` From ff41c68c6a2ed8ad9a1b217357e47f19959dce42 Mon Sep 17 00:00:00 2001 From: JunWang222 <466529050@qq.com> Date: Wed, 10 Jun 2026 02:26:35 -0400 Subject: [PATCH 03/18] Implement the Trino operators for JDBC template, add integration tests into TrinoOperatorsIT --- wayang-platforms/wayang-trino/pom.xml | 6 + .../trino/mapping/GlobalReduceMapping.java | 62 +++ .../apache/wayang/trino/mapping/Mappings.java | 6 +- .../wayang/trino/mapping/ReduceByMapping.java | 65 +++ .../wayang/trino/mapping/SortMapping.java | 64 +++ .../trino/mapping/TableSinkMapping.java | 60 +++ .../operators/TrinoGlobalReduceOperator.java | 51 +++ .../operators/TrinoReduceByOperator.java | 53 +++ .../trino/operators/TrinoSortOperator.java | 51 +++ .../operators/TrinoTableSinkOperator.java | 51 +++ .../wayang-trino-defaults.properties | 65 +++ .../apache/wayang/trino/TrinoOperatorsIT.java | 427 ++++++++++++++++++ 12 files changed, 960 insertions(+), 1 deletion(-) create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/GlobalReduceMapping.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/ReduceByMapping.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/SortMapping.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/TableSinkMapping.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoGlobalReduceOperator.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoReduceByOperator.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoSortOperator.java create mode 100644 wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoTableSinkOperator.java create mode 100644 wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java diff --git a/wayang-platforms/wayang-trino/pom.xml b/wayang-platforms/wayang-trino/pom.xml index 7f4d44695..9f1a6fa83 100644 --- a/wayang-platforms/wayang-trino/pom.xml +++ b/wayang-platforms/wayang-trino/pom.xml @@ -59,6 +59,12 @@ wayang-spark 1.1.2-SNAPSHOT + + org.junit.jupiter + junit-jupiter + 5.10.2 + test + diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/GlobalReduceMapping.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/GlobalReduceMapping.java new file mode 100644 index 000000000..c395cbfd8 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/GlobalReduceMapping.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.trino.operators.TrinoGlobalReduceOperator; +import org.apache.wayang.trino.platform.TrinoPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link GlobalReduceOperator} to {@link TrinoGlobalReduceOperator}. + */ +@SuppressWarnings("unchecked") +public class GlobalReduceMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + TrinoPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "reduce", new GlobalReduceOperator(null, DataSetType.createDefault(Record.class)), false) + .withAdditionalTest(op -> op.getReduceDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new TrinoGlobalReduceOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java index 1ffd69491..2912ed7ed 100644 --- a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/Mappings.java @@ -30,8 +30,12 @@ public class Mappings { public static final Collection ALL = Arrays.asList( new FilterMapping(), + new GlobalReduceMapping(), new JoinMapping(), - new ProjectionMapping() + new ProjectionMapping(), + new ReduceByMapping(), + new SortMapping(), + new TableSinkMapping() ); } diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/ReduceByMapping.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/ReduceByMapping.java new file mode 100644 index 000000000..8870c9d8c --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/ReduceByMapping.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.trino.operators.TrinoReduceByOperator; +import org.apache.wayang.trino.platform.TrinoPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link ReduceByOperator} to {@link TrinoReduceByOperator}. + */ +@SuppressWarnings("unchecked") +public class ReduceByMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + TrinoPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "reduceBy", + new ReduceByOperator(null, null, DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(op -> op.getKeyDescriptor().getSqlImplementation() != null + && op.getReduceDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new TrinoReduceByOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/SortMapping.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/SortMapping.java new file mode 100644 index 000000000..c952deb0c --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/SortMapping.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.trino.operators.TrinoSortOperator; +import org.apache.wayang.trino.platform.TrinoPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link SortOperator} to {@link TrinoSortOperator}. + */ +@SuppressWarnings("unchecked") +public class SortMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + TrinoPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "sort", + new SortOperator(null, DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(op -> op.getKeyDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new TrinoSortOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/TableSinkMapping.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/TableSinkMapping.java new file mode 100644 index 000000000..9539f50b0 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/mapping/TableSinkMapping.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.mapping; + +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.trino.operators.TrinoTableSinkOperator; +import org.apache.wayang.trino.platform.TrinoPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link TableSink} to {@link TrinoTableSinkOperator}. + */ +@SuppressWarnings("unchecked") +public class TableSinkMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + TrinoPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern operatorPattern = new OperatorPattern<>( + "sink", new TableSink<>(null, null, null), false + ); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new TrinoTableSinkOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoGlobalReduceOperator.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoGlobalReduceOperator.java new file mode 100644 index 000000000..2e608faaa --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoGlobalReduceOperator.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.jdbc.operators.JdbcGlobalReduceOperator; + +/** + * Trino implementation of the {@link GlobalReduceOperator}. + */ +public class TrinoGlobalReduceOperator extends JdbcGlobalReduceOperator implements TrinoExecutionOperator { + + /** + * Creates a new instance. + */ + public TrinoGlobalReduceOperator(ReduceDescriptor reduceDescriptor) { + super(reduceDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public TrinoGlobalReduceOperator(GlobalReduceOperator that) { + super(that); + } + + @Override + protected TrinoGlobalReduceOperator createCopy() { + return new TrinoGlobalReduceOperator(this); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoReduceByOperator.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoReduceByOperator.java new file mode 100644 index 000000000..d825ca82b --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoReduceByOperator.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.jdbc.operators.JdbcReduceByOperator; + +/** + * Trino implementation of the {@link ReduceByOperator}. + */ +public class TrinoReduceByOperator extends JdbcReduceByOperator implements TrinoExecutionOperator { + + /** + * Creates a new instance. + */ + public TrinoReduceByOperator(TransformationDescriptor keyDescriptor, + ReduceDescriptor reduceDescriptor) { + super(keyDescriptor, reduceDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public TrinoReduceByOperator(ReduceByOperator that) { + super(that); + } + + @Override + protected TrinoReduceByOperator createCopy() { + return new TrinoReduceByOperator(this); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoSortOperator.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoSortOperator.java new file mode 100644 index 000000000..5fd4dcb84 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoSortOperator.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.jdbc.operators.JdbcSortOperator; + +/** + * Trino implementation of the {@link SortOperator}. + */ +public class TrinoSortOperator extends JdbcSortOperator implements TrinoExecutionOperator { + + /** + * Creates a new instance. + */ + public TrinoSortOperator(TransformationDescriptor keyDescriptor) { + super(keyDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public TrinoSortOperator(SortOperator that) { + super(that); + } + + @Override + protected TrinoSortOperator createCopy() { + return new TrinoSortOperator(this); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoTableSinkOperator.java b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoTableSinkOperator.java new file mode 100644 index 000000000..cbc276e30 --- /dev/null +++ b/wayang-platforms/wayang-trino/src/main/java/org/apache/wayang/trino/operators/TrinoTableSinkOperator.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.jdbc.operators.JdbcTableSinkOperator; + +/** + * Trino implementation of the {@link JdbcTableSinkOperator}. The sink stays + * entirely within Trino: the composed query is wrapped in a + * {@code CREATE TABLE ... AS} (mode {@code overwrite}) or + * {@code INSERT INTO ... } statement. + * + *

Table names use Trino's three-part naming convention: + * {@code catalog.schema.table} (e.g. {@code iceberg.sales.orders}). + */ +public class TrinoTableSinkOperator extends JdbcTableSinkOperator implements TrinoExecutionOperator { + + /** + * Creates a new instance. + */ + public TrinoTableSinkOperator(String tableName, String[] columnNames) { + super(tableName, columnNames); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public TrinoTableSinkOperator(TableSink that) { + super(that); + } +} diff --git a/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties b/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties index 651ed564f..fe0429b85 100644 --- a/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties +++ b/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties @@ -83,6 +83,71 @@ wayang.trino.projection.load = {\ "p":0.9\ } +wayang.trino.join.load.template = {\ + "type":"mathex", "in":2, "out":1,\ + "cpu":"?*(in0 + in1) + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.trino.join.load = {\ + "in":2, "out":1,\ + "cpu":"${10*(in0 + in1) + 800000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.trino.globalreduce.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.trino.globalreduce.load = {\ + "in":1, "out":1,\ + "cpu":"${10*in0 + 800000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.trino.reduceby.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.trino.reduceby.load = {\ + "in":1, "out":1,\ + "cpu":"${10*in0 + 800000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.trino.sort.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.trino.sort.load = {\ + "in":1, "out":1,\ + "cpu":"${10*in0 + 800000}",\ + "ram":"0",\ + "p":0.9\ +} + +wayang.trino.tablesink.load.template = {\ + "type":"mathex", "in":1, "out":0,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.trino.tablesink.load = {\ + "in":1, "out":0,\ + "cpu":"${10*in0 + 800000}",\ + "ram":"0",\ + "p":0.9\ +} + wayang.trino.sqltostream.load.query.template = {\ "type":"mathex", "in":1, "out":1,\ "cpu":"?*out0 + ?"\ diff --git a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java new file mode 100644 index 000000000..1586ce55b --- /dev/null +++ b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java @@ -0,0 +1,427 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.trino; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.basic.operators.LocalCallbackSink; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.basic.types.RecordType; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.java.Java; +import org.apache.wayang.jdbc.compiler.FunctionCompiler; +import org.apache.wayang.trino.operators.TrinoJoinOperator; +import org.apache.wayang.trino.operators.TrinoTableSource; +import org.apache.wayang.trino.platform.TrinoPlatform; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end integration tests for every operator the Trino platform implements, + * driven through the Wayang API against a live Trino cluster. + * + *

Coverage: {@code TableSource}, {@code Filter}, {@code Projection}, + * {@code Join}, {@code GlobalReduce}, {@code ReduceBy}, {@code Sort}, + * {@code TableSink} — plus the SQL→Stream channel conversion that materialises + * every result. Each test also asserts, via Trino's {@code system.runtime.queries}, + * that the expected SQL actually reached Trino (i.e. the operator was pushed + * down, not silently executed elsewhere). + * + *

Prerequisites: a Trino reachable at {@code TRINO_HOST:TRINO_PORT} + * (defaults {@code localhost:8080}); e.g. {@code cd trino-setup && docker compose up -d}. + * If Trino is not reachable the whole class is skipped (not failed). + * + *

Run: + * 

+ *   JAVA_HOME=<jdk17> mvn -o test -pl wayang-platforms/wayang-trino \
+ *     -Dtest=TrinoOperatorsIT -Dsurefire.failIfNoSpecifiedTests=false \
+ *     -Drat.skip=true -Dlicense.skip=true -Pskip-prerequisite-check
+ *
+ */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class TrinoOperatorsIT { + + private static final String HOST = System.getenv().getOrDefault("TRINO_HOST", "localhost"); + private static final int PORT = Integer.parseInt(System.getenv().getOrDefault("TRINO_PORT", "8080")); + private static final String USER = System.getenv().getOrDefault("TRINO_USER", "admin"); + private static final String JDBC_URL = String.format("jdbc:trino://%s:%d", HOST, PORT); + + // Dedicated schema so the test is self-contained and side-effect free. + private static final String SCHEMA = "iceberg.wayang_it"; + private static final String ORDERS = SCHEMA + ".orders"; + private static final String CUSTOMERS = SCHEMA + ".customers"; + private static final String SINK_TABLE = SCHEMA + ".amer_orders"; + + private static boolean trinoAvailable = false; + + // ── Lifecycle ─────────────────────────────────────────────────────────── + + @BeforeAll + static void setUp() { + try (Connection c = jdbc()) { + Statement st = c.createStatement(); + st.execute("CREATE SCHEMA IF NOT EXISTS " + SCHEMA); + + st.execute("DROP TABLE IF EXISTS " + ORDERS); + st.execute("CREATE TABLE " + ORDERS + " (" + + "order_id BIGINT, customer_id BIGINT, region VARCHAR, amount DOUBLE)" + + " WITH (format = 'PARQUET')"); + st.execute("INSERT INTO " + ORDERS + " VALUES " + + "(1, 100, 'AMER', 2200.0)," + + "(2, 101, 'EMEA', 800.5)," + + "(3, 100, 'AMER', 680.5)," + + "(4, 102, 'APAC', 1500.0)," + + "(5, 101, 'EMEA', 1100.0)," + + "(6, 100, 'AMER', 950.25)"); + // Scale up so Wayang's cost optimizer actually elects SQL pushdown + // (on a tiny table it prefers a full scan + Java-side ops, which + // makes the query-history assertions fail). Trino caps sequence() + // at 10000 entries, so scale in two steps: x10000 then x2 = 120000 + // rows total; ratios preserved (AMER = 60000). + st.execute("INSERT INTO " + ORDERS + + " SELECT order_id + (n * 10), customer_id, region, amount" + + " FROM " + ORDERS + ", UNNEST(sequence(1, 9999)) AS t(n)"); // 6 -> 60000 + st.execute("INSERT INTO " + ORDERS + + " SELECT order_id + 600000, customer_id, region, amount" + + " FROM " + ORDERS); // 60000 -> 120000 + + st.execute("DROP TABLE IF EXISTS " + CUSTOMERS); + st.execute("CREATE TABLE " + CUSTOMERS + " (" + + "customer_id BIGINT, name VARCHAR, tier VARCHAR)" + + " WITH (format = 'PARQUET')"); + st.execute("INSERT INTO " + CUSTOMERS + " VALUES " + + "(100, 'Acme', 'GOLD')," + + "(101, 'Globex', 'SILVER')," + + "(102, 'Initech','BRONZE')"); + + trinoAvailable = true; + System.out.println("[TrinoOperatorsIT] Connected to Trino at " + JDBC_URL + " — fixtures created."); + } catch (Exception e) { + System.err.println("[TrinoOperatorsIT] Trino not available (" + e.getMessage() + ") — skipping."); + } + } + + @AfterAll + static void tearDown() { + if (!trinoAvailable) return; + try (Connection c = jdbc()) { + Statement st = c.createStatement(); + st.execute("DROP TABLE IF EXISTS " + ORDERS); + st.execute("DROP TABLE IF EXISTS " + CUSTOMERS); + st.execute("DROP TABLE IF EXISTS " + SINK_TABLE); + } catch (Exception e) { + System.err.println("[TrinoOperatorsIT] cleanup failed: " + e.getMessage()); + } + } + + // ── Tests (one per operator) ────────────────────────────────────────────── + + /** TableSource: full scan returns every row. */ + @Test + @Order(1) + void tableSource() { + Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); + List rows = execute(results -> { + TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + src.connectTo(0, sink, 0); + return sink; + }); + assertEquals(120000, rows.size(), "TableSource should return all orders"); + assertSqlReachedTrino("SELECT * FROM " + ORDERS); + } + + /** Filter: WHERE region = 'AMER' pushed to Trino. */ + @Test + @Order(2) + void filter() { + Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); + List rows = execute(results -> { + TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record r) -> "AMER".equals(r.getField(2)), Record.class + ).withSqlImplementation("region = 'AMER'")); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + src.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + return sink; + }); + assertEquals(60000, rows.size(), "60000 AMER orders expected"); + assertTrue(rows.stream().allMatch(r -> "AMER".equals(r.getField(2))), "all rows AMER"); + assertSqlReachedTrino("WHERE region = 'AMER'"); + } + + /** Projection (+filter): only the projected columns are fetched. */ + @Test + @Order(3) + void projection() { + Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); + List rows = execute(results -> { + TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record r) -> "AMER".equals(r.getField(2)), Record.class + ).withSqlImplementation("region = 'AMER'")); + MapOperator projection = new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType("order_id", "customer_id", "region", "amount"), + "region", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + src.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + return sink; + }); + assertEquals(60000, rows.size(), "60000 AMER rows expected"); + assertEquals(2, rows.get(0).size(), "projection keeps only 2 columns"); + assertSqlReachedTrino("SELECT region, amount FROM " + ORDERS); + } + + /** + * Join: orders ⋈ customers on customer_id. + * + *

Unlike the other operators, a JDBC join cannot be driven through the + * high-level {@link WayangContext} API here: the logical {@link JoinOperator} + * emits {@code Tuple2}, which has no valid connection to a + * {@code Record} sink (the SQL pushdown that would flatten it happens only + * after optimization). So we verify the operator's real contract directly: + * {@link TrinoJoinOperator#createSqlClause} must produce a Trino-valid JOIN + * clause that, executed against live Trino, returns the correct rows. + */ + @Test + @Order(4) + void join() throws Exception { + Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); + TrinoJoinOperator join = new TrinoJoinOperator<>( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(1)), Record.class, Record.class + ).withSqlImplementation(ORDERS, "customer_id"), + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(0)), Record.class, Record.class + ).withSqlImplementation(CUSTOMERS, "customer_id")); + + // The operator routes to the Trino platform. + assertEquals(TrinoPlatform.getInstance(), join.getPlatform()); + + try (Connection c = jdbc()) { + // Operator-generated JOIN clause, assembled into the full SELECT exactly + // as JdbcExecutor would (FROM + ). + String joinClause = join.createSqlClause(c, new FunctionCompiler()); + assertTrue(joinClause.startsWith("JOIN " + CUSTOMERS + " ON"), + "unexpected join clause: " + joinClause); + + String sql = "SELECT * FROM " + ORDERS + " " + joinClause; + ResultSet rs = c.createStatement().executeQuery(sql); + int n = 0; + while (rs.next()) n++; + // Every order's customer_id exists in customers → one joined row per order. + assertEquals(120000, n, "join should yield one row per order"); + } + } + + /** GlobalReduce: SUM(amount) over the whole table collapses to a single row. */ + @Test + @Order(5) + void globalReduce() { + Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); + List rows = execute(results -> { + TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + GlobalReduceOperator reduce = new GlobalReduceOperator<>( + new ReduceDescriptor<>((a, b) -> a, Record.class) + .withSqlImplementation("SUM(amount)"), + DataSetType.createDefault(Record.class)); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + src.connectTo(0, reduce, 0); + reduce.connectTo(0, sink, 0); + return sink; + }); + assertEquals(1, rows.size(), "global reduce must collapse to a single row"); + // 6 base rows sum to 7231.25; scaled x20000 → 144,625,000 (exact in doubles). + assertEquals(144_625_000.0, ((Number) rows.get(0).getField(0)).doubleValue(), 0.01); + assertSqlReachedTrino("SELECT SUM(amount) FROM " + ORDERS); + } + + /** ReduceBy: SUM(amount) GROUP BY region yields one row per region. */ + @Test + @Order(6) + void reduceBy() { + Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); + List rows = execute(results -> { + TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + ReduceByOperator reduceBy = new ReduceByOperator<>( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(2)), Record.class, Record.class + ).withSqlImplementation("region", "region"), + new ReduceDescriptor<>((a, b) -> a, Record.class) + .withSqlImplementation("SUM(amount)"), + DataSetType.createDefault(Record.class)); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + src.connectTo(0, reduceBy, 0); + reduceBy.connectTo(0, sink, 0); + return sink; + }); + assertEquals(3, rows.size(), "one row per region expected"); + Map sums = new HashMap<>(); + for (Record r : rows) { + sums.put((String) r.getField(0), ((Number) r.getField(1)).doubleValue()); + } + // Base sums (AMER 3830.75, EMEA 1900.5, APAC 1500.0) scaled x20000. + assertEquals(76_615_000.0, sums.get("AMER"), 0.01); + assertEquals(38_010_000.0, sums.get("EMEA"), 0.01); + assertEquals(30_000_000.0, sums.get("APAC"), 0.01); + assertSqlReachedTrino("GROUP BY region"); + } + + /** Sort: ORDER BY amount ASC pushed to Trino, order preserved to the sink. */ + @Test + @Order(7) + void sort() { + Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); + List rows = execute(results -> { + TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + SortOperator sort = new SortOperator<>( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(3)), Record.class, Record.class + ).withSqlImplementation("amount", "ASC"), + DataSetType.createDefault(Record.class)); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + src.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return sink; + }); + assertEquals(120000, rows.size(), "sort must not change the cardinality"); + assertEquals(680.5, ((Number) rows.get(0).getField(3)).doubleValue(), 0.001, "smallest amount first"); + assertEquals(2200.0, ((Number) rows.get(rows.size() - 1).getField(3)).doubleValue(), 0.001, "largest amount last"); + for (int i = 1; i < rows.size(); i++) { + double prev = ((Number) rows.get(i - 1).getField(3)).doubleValue(); + double curr = ((Number) rows.get(i).getField(3)).doubleValue(); + assertTrue(prev <= curr, "rows must be non-decreasing by amount at index " + i); + } + assertSqlReachedTrino("ORDER BY amount ASC"); + } + + /** + * TableSink: filter + sink composed into a single {@code CREATE TABLE ... AS + * SELECT} that runs entirely inside Trino — no data leaves the database. + */ + @Test + @Order(8) + void tableSink() throws Exception { + Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); + + TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record r) -> "AMER".equals(r.getField(2)), Record.class + ).withSqlImplementation("region = 'AMER'")); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", SINK_TABLE, + "order_id", "customer_id", "region", "amount"); + src.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + try (Connection c = jdbc()) { + ResultSet rs = c.createStatement().executeQuery( + "SELECT count(*), count_if(region <> 'AMER') FROM " + SINK_TABLE); + rs.next(); + assertEquals(60000, rs.getLong(1), "sink table must hold all AMER orders"); + assertEquals(0, rs.getLong(2), "sink table must hold only AMER orders"); + } + assertSqlReachedTrino("CREATE TABLE " + SINK_TABLE + " AS"); + } + + // ── Helpers ─────────────────────────────────────────────────────────────── + + /** Functional builder: given the result list, wire a plan and return its sink. */ + private interface PlanBuilder { + LocalCallbackSink build(List resultCollector); + } + + private WayangContext wayangContext() { + Configuration config = new Configuration(); + config.setProperty("wayang.trino.jdbc.url", JDBC_URL); + config.setProperty("wayang.trino.jdbc.user", USER); + config.setProperty("wayang.trino.jdbc.password", ""); + return new WayangContext(config) + .withPlugin(Java.basicPlugin()) + .withPlugin(Trino.plugin()); + } + + private List execute(PlanBuilder builder) { + List results = new ArrayList<>(); + LocalCallbackSink sink = builder.build(results); + wayangContext().execute(new WayangPlan(sink)); + return results; + } + + /** Assert that Trino actually ran a query containing the given fragment. */ + private void assertSqlReachedTrino(String fragment) { + try (Connection c = jdbc()) { + ResultSet rs = c.createStatement().executeQuery( + "SELECT count(*) FROM system.runtime.queries " + + "WHERE query LIKE '%" + fragment.replace("'", "''") + "%' " + + "AND query NOT LIKE '%system.runtime%'"); + rs.next(); + assertTrue(rs.getLong(1) > 0, + "Expected a Trino query containing: " + fragment); + } catch (Exception e) { + throw new RuntimeException("query-history check failed", e); + } + } + + private static Connection jdbc() throws Exception { + Properties p = new Properties(); + p.setProperty("user", USER); + return DriverManager.getConnection(JDBC_URL, p); + } +} From 43da0240d7006b74a73fbd5f9154a1d2b0a315dc Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Thu, 11 Jun 2026 12:51:51 +0800 Subject: [PATCH 04/18] Document Trino operator integration tests --- trino-setup/README.md | 66 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/trino-setup/README.md b/trino-setup/README.md index 4133bd9af..4562625dc 100644 --- a/trino-setup/README.md +++ b/trino-setup/README.md @@ -2,10 +2,11 @@ Local Trino environment backed by an **Iceberg** data lake, completely containerised. -The current validation has two parts: +The current validation has three parts: 1. Build the Wayang Trino platform and run the shared JDBC SQL-generation tests. -2. Run JDBC integration tests against the local Trino, Iceberg, and MinIO stack. +2. Run the Wayang Trino operator tests against the live local stack. +3. Run standalone JDBC integration tests against the local Trino, Iceberg, and MinIO stack. Run the commands below from the repository root. Java 17 and Docker with Docker Compose are required; Maven is provided by the repository wrapper. @@ -78,7 +79,7 @@ Wayang Platform Trino ... SUCCESS Tests run: 4, Failures: 0, Errors: 0, Skipped: 0 ``` -## 2. Test the Local Trino Stack +## 2. Test Against the Local Trino Stack ### 1. Start the stack @@ -93,7 +94,41 @@ docker compose -f trino-setup/docker-compose.yml ps # or watch the Trino UI at http://localhost:8080 ``` -### 2. Load sample Iceberg data +### 2. Run the Wayang Trino operator tests + +`TrinoOperatorsIT` exercises the Wayang Trino implementation against the live +Trino stack. It checks `TableSource`, `Filter`, `Projection`, `Join`, +`GlobalReduce`, `ReduceBy`, `Sort`, and `TableSink`, and confirms that the +expected SQL reached Trino. Most tests run a Wayang plan end to end; the join +test directly validates and executes the SQL clause generated by +`TrinoJoinOperator`. + +The suite is self-contained: it creates `iceberg.wayang_it`, scales its test +data to 120,000 rows so the optimizer selects SQL pushdown, and drops its test +tables afterward. It does not require `scripts/init.sql`. + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-trino -am \ + -Dtest=TrinoOperatorsIT -Dsurefire.failIfNoSpecifiedTests=false \ + -DfailIfNoTests=false -Drat.skip=true -Dlicense.skip=true test +``` + +On PowerShell: + +```powershell +.\mvnw.cmd --% -Pskip-prerequisite-check -pl wayang-platforms/wayang-trino -am -Dtest=TrinoOperatorsIT -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false -Drat.skip=true -Dlicense.skip=true test +``` + +Expected result: + +```text +Tests run: 8, Failures: 0, Errors: 0, Skipped: 0 +``` + +If Trino is unreachable, these tests are skipped instead of failed. A result +with skipped tests does not confirm that the operators work. + +### 3. Load sample Iceberg data ```bash bash trino-setup/scripts/run-init.sh @@ -108,7 +143,7 @@ Get-Content -Raw trino-setup/scripts/init.sql | docker exec -i trino trino --ser This creates the schema `iceberg.sales` and inserts 20 sample orders into `iceberg.sales.orders` (Parquet files on MinIO). -### 3. Run the integration tests +### 4. Run the standalone stack integration tests ```bash ./mvnw -f trino-setup/pom.xml -Pintegration -Dtest=TrinoIntegrationTest test @@ -121,6 +156,8 @@ On PowerShell: ``` Tests are skipped by default (no `-Pintegration`) to avoid requiring Docker in CI. +These tests validate the stack and direct JDBC queries independently of the +Wayang operator implementation. Expected result: @@ -129,7 +166,7 @@ Tests run: 10, Failures: 0, Errors: 0, Skipped: 0 BUILD SUCCESS ``` -### 4. Manual exploration +### 5. Manual exploration Open the **Trino UI**: http://localhost:8080 @@ -156,7 +193,7 @@ SELECT * FROM iceberg.sales."orders$history"; **MinIO console**: http://localhost:9001 (login: `minioadmin` / `minioadmin`) Look for Parquet files under `warehouse/sales/orders/`. -### 5. Tear down +### 6. Tear down ```bash docker compose -f trino-setup/docker-compose.yml down -v @@ -166,6 +203,21 @@ The `-v` option removes volumes and clears the local MinIO and PostgreSQL data. ## Test Coverage +### Wayang operator integration tests + +| Test | What it checks | +|------|----------------| +| `tableSource` | Full table scan through `TrinoTableSource` | +| `filter` | Wayang `FilterOperator` and SQL `WHERE` pushdown | +| `projection` | Column projection pushed into the Trino query | +| `join` | Trino join operator generates and executes a valid join clause | +| `globalReduce` | Global aggregation such as `SUM` | +| `reduceBy` | Grouped aggregation and SQL `GROUP BY` | +| `sort` | Wayang sort and SQL `ORDER BY` | +| `tableSink` | Filtered result written with `CREATE TABLE AS` | + +### Standalone stack integration tests + | Test | What it checks | |------|----------------| | `testConnectivity` | `SELECT 1`, JDBC connection works | From 48e7d4bea23714a9e120423ff2f21e0e9bf6bcbb Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Tue, 16 Jun 2026 02:15:07 +0800 Subject: [PATCH 05/18] Add SQL metadata methods to JavaPlanBuilder operators --- .../apache/wayang/api/DataQuantaBuilder.scala | 114 ++++++++++++++++-- 1 file changed, 107 insertions(+), 7 deletions(-) diff --git a/wayang-api/wayang-api-scala-java/src/main/scala/org/apache/wayang/api/DataQuantaBuilder.scala b/wayang-api/wayang-api-scala-java/src/main/scala/org/apache/wayang/api/DataQuantaBuilder.scala index d18ed3f85..9d37aa930 100644 --- a/wayang-api/wayang-api-scala-java/src/main/scala/org/apache/wayang/api/DataQuantaBuilder.scala +++ b/wayang-api/wayang-api-scala-java/src/main/scala/org/apache/wayang/api/DataQuantaBuilder.scala @@ -28,7 +28,7 @@ import org.apache.wayang.api.graph.{Edge, EdgeDataQuantaBuilder, EdgeDataQuantaB import org.apache.wayang.api.util.{DataQuantaBuilderCache, TypeTrap} import org.apache.wayang.basic.data.{Record, Tuple2 => RT2} import org.apache.wayang.basic.model.{DLModel, Model, LogisticRegressionModel,DecisionTreeRegressionModel} -import org.apache.wayang.basic.operators.{DLTrainingOperator, GlobalReduceOperator, LocalCallbackSink, MapOperator, SampleOperator, LogisticRegressionOperator,DecisionTreeRegressionOperator, LinearSVCOperator} +import org.apache.wayang.basic.operators.{DLTrainingOperator, GlobalReduceOperator, JoinOperator, LocalCallbackSink, MapOperator, ReduceByOperator, SampleOperator, SortOperator, LogisticRegressionOperator,DecisionTreeRegressionOperator, LinearSVCOperator} import org.apache.wayang.commons.util.profiledb.model.Experiment import org.apache.wayang.core.api.spatial.{SpatialGeometry, SpatialPredicate} import org.apache.wayang.core.function.FunctionDescriptor.{SerializableBiFunction, SerializableBinaryOperator, SerializableFunction, SerializableIntUnaryOperator, SerializablePredicate} @@ -1020,6 +1020,10 @@ class SortDataQuantaBuilder[T, Key](inputDataQuanta: DataQuantaBuilder[_, T], /** [[LoadEstimator]] to estimate the RAM load of the [[keyUdf]]. */ private var keyUdfRamEstimator: LoadEstimator = _ + /** SQL column and direction implementing the sort key. */ + private var sqlColumnName: String = _ + private var sqlDirection: String = _ + // Try to infer the type classes from the UDFs. locally { @@ -1060,8 +1064,27 @@ class SortDataQuantaBuilder[T, Key](inputDataQuanta: DataQuantaBuilder[_, T], this } - override protected def build = - applyTargetPlatforms(inputDataQuanta.dataQuanta().sortJava(keyUdf)(this.keyTag), this.getTargetPlatforms()) + /** + * Add a SQL implementation of the sort key. + * + * @param columnName SQL column to sort by + * @param direction SQL sort direction, e.g. `ASC` or `DESC` + * @return this instance + */ + def withSqlUdf(columnName: String, direction: String) = { + this.sqlColumnName = columnName + this.sqlDirection = direction + this + } + + override protected def build = { + val result = inputDataQuanta.dataQuanta().sortJava(keyUdf)(this.keyTag) + if (this.sqlColumnName != null) { + result.operator.asInstanceOf[SortOperator[T, Key]] + .getKeyDescriptor.withSqlImplementation(this.sqlColumnName, this.sqlDirection) + } + applyTargetPlatforms(result, this.getTargetPlatforms()) + } } @@ -1283,6 +1306,10 @@ class ReduceByDataQuantaBuilder[Key, T](inputDataQuanta: DataQuantaBuilder[_, T] /** [[LoadProfileEstimator]] to estimate the [[LoadProfile]] of the [[udf]]. */ private var udfLoadProfileEstimator: LoadProfileEstimator = _ + /** SQL implementations of the grouping key and reduction. */ + private var keySqlUdf: String = _ + private var reduceSqlUdf: String = _ + // TODO: Add these estimators. // /** [[LoadEstimator]] to estimate the CPU load of the [[keyUdf]]. */ // private var keyUdfCpuEstimator: LoadEstimator = _ @@ -1322,7 +1349,29 @@ class ReduceByDataQuantaBuilder[Key, T](inputDataQuanta: DataQuantaBuilder[_, T] this } - override protected def build = applyTargetPlatforms(inputDataQuanta.dataQuanta().reduceByKeyJava(keyUdf, udf, this.udfLoadProfileEstimator), this.getTargetPlatforms()) + /** + * Add SQL implementations of the grouping key and reduction. + * + * @param keySqlUdf SQL grouping column + * @param reduceSqlUdf SQL aggregate expression + * @return this instance + */ + def withSqlUdfs(keySqlUdf: String, reduceSqlUdf: String) = { + this.keySqlUdf = keySqlUdf + this.reduceSqlUdf = reduceSqlUdf + this + } + + override protected def build = { + val result = inputDataQuanta.dataQuanta() + .reduceByKeyJava(keyUdf, udf, this.udfLoadProfileEstimator) + if (this.keySqlUdf != null) { + val operator = result.operator.asInstanceOf[ReduceByOperator[T, Key]] + operator.getKeyDescriptor.withSqlImplementation(this.keySqlUdf, this.keySqlUdf) + operator.getReduceDescriptor.withSqlImplementation(this.reduceSqlUdf) + } + applyTargetPlatforms(result, this.getTargetPlatforms()) + } } /** @@ -1402,6 +1451,9 @@ class GlobalReduceDataQuantaBuilder[T](inputDataQuanta: DataQuantaBuilder[_, T], /** [[LoadProfileEstimator]] to estimate the [[LoadProfile]] of the [[udf]]. */ private var udfLoadProfileEstimator: LoadProfileEstimator = _ + /** SQL implementation of the reduction. */ + private var sqlUdf: String = _ + // Try to infer the type classes from the udf. locally { val parameters = ReflectionUtils.getTypeParameters(udf.getClass, classOf[SerializableBinaryOperator[_]]) @@ -1422,7 +1474,25 @@ class GlobalReduceDataQuantaBuilder[T](inputDataQuanta: DataQuantaBuilder[_, T], this } - override protected def build = applyTargetPlatforms(inputDataQuanta.dataQuanta().reduceJava(udf, this.udfLoadProfileEstimator), this.getTargetPlatforms()) + /** + * Add a SQL implementation of the reduction. + * + * @param sqlUdf SQL aggregate expression + * @return this instance + */ + def withSqlUdf(sqlUdf: String) = { + this.sqlUdf = sqlUdf + this + } + + override protected def build = { + val result = inputDataQuanta.dataQuanta().reduceJava(udf, this.udfLoadProfileEstimator) + if (this.sqlUdf != null) { + result.operator.asInstanceOf[GlobalReduceOperator[T]] + .getReduceDescriptor.withSqlImplementation(this.sqlUdf) + } + applyTargetPlatforms(result, this.getTargetPlatforms()) + } } @@ -1490,6 +1560,12 @@ class JoinDataQuantaBuilder[In0, In1, Key](inputDataQuanta0: DataQuantaBuilder[_ /** [[LoadEstimator]] to estimate the RAM load of the [[keyUdf1]]. */ private var keyUdf1RamEstimator: LoadEstimator = _ + /** SQL implementations of both join keys. */ + private var keyUdf0TableName: String = _ + private var keyUdf0SqlUdf: String = _ + private var keyUdf1TableName: String = _ + private var keyUdf1SqlUdf: String = _ + // Try to infer the type classes from the UDFs. locally { val parameters = ReflectionUtils.getTypeParameters(keyUdf0.getClass, classOf[SerializableFunction[_, _]]) @@ -1568,6 +1644,22 @@ class JoinDataQuantaBuilder[In0, In1, Key](inputDataQuanta0: DataQuantaBuilder[_ this } + /** + * Add SQL implementations of both join keys. + * + * @return this instance + */ + def withSqlUdfs(thisTableName: String, + thisKeySqlUdf: String, + thatTableName: String, + thatKeySqlUdf: String) = { + this.keyUdf0TableName = thisTableName + this.keyUdf0SqlUdf = thisKeySqlUdf + this.keyUdf1TableName = thatTableName + this.keyUdf1SqlUdf = thatKeySqlUdf + this + } + /** * Assemble the joined elements to new elements. * @@ -1579,8 +1671,16 @@ class JoinDataQuantaBuilder[In0, In1, Key](inputDataQuanta0: DataQuantaBuilder[_ override def apply(joinTuple: RT2[In0, In1]): NewOut = udf.apply(joinTuple.field0, joinTuple.field1) }) - override protected def build = - applyTargetPlatforms(inputDataQuanta0.dataQuanta().joinJava(keyUdf0, inputDataQuanta1.dataQuanta(), keyUdf1)(inputDataQuanta1.classTag, this.keyTag), this.getTargetPlatforms()) + override protected def build = { + val result = inputDataQuanta0.dataQuanta() + .joinJava(keyUdf0, inputDataQuanta1.dataQuanta(), keyUdf1)(inputDataQuanta1.classTag, this.keyTag) + if (this.keyUdf0SqlUdf != null) { + val operator = result.operator.asInstanceOf[JoinOperator[In0, In1, Key]] + operator.getKeyDescriptor0.withSqlImplementation(this.keyUdf0TableName, this.keyUdf0SqlUdf) + operator.getKeyDescriptor1.withSqlImplementation(this.keyUdf1TableName, this.keyUdf1SqlUdf) + } + applyTargetPlatforms(result, this.getTargetPlatforms()) + } } From fe71536c33c5d515607b39eb78f24c870e6ead6f Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Tue, 16 Jun 2026 02:15:20 +0800 Subject: [PATCH 06/18] Select the left JDBC source for join stages --- .../wayang/jdbc/execution/JdbcExecutor.java | 29 ++++++++++++++++++- .../jdbc/operators/JdbcJoinOperatorTest.java | 9 +++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java index 0339fb03b..8596c90f1 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java @@ -134,7 +134,7 @@ protected static Tuple2 createSqlQuery(final E final Collection startTasks = stage.getStartTasks(); // Verify that we can handle this instance. - final ExecutionTask startTask = (ExecutionTask) startTasks.toArray()[0]; + final ExecutionTask startTask = JdbcExecutor.selectStartTask(startTasks, stage); assert startTask.getOperator() instanceof TableSource : "Invalid JDBC stage: Start task has to be a TableSource"; @@ -191,6 +191,33 @@ protected static Tuple2 createSqlQuery(final E return new Tuple2<>(query.toString(), tipChannelInstance); } + /** + * Selects the table source that belongs on the left-hand side of a JDBC join. + * Stage start tasks are not ordered, but {@link JdbcJoinOperator#createSqlClause} + * assumes its first key descriptor's table is used in the {@code FROM} clause. + */ + private static ExecutionTask selectStartTask(final Collection startTasks, final ExecutionStage stage) { + if (startTasks.size() == 1) { + return (ExecutionTask) startTasks.iterator().next(); + } + + for (ExecutionTask task : stage.getAllTasks()) { + if (task.getOperator() instanceof JdbcJoinOperator) { + final JdbcJoinOperator joinOperator = (JdbcJoinOperator) task.getOperator(); + final String leftTableName = joinOperator.getKeyDescriptor0().getSqlImplementation().field0; + for (Object startTaskObject : startTasks) { + final ExecutionTask startTask = (ExecutionTask) startTaskObject; + if (startTask.getOperator() instanceof JdbcTableSource + && ((JdbcTableSource) startTask.getOperator()).getTableName().equals(leftTableName)) { + return startTask; + } + } + } + } + + throw new WayangException("Could not determine the left table source for JDBC stage."); + } + /** * Handles execution stages that end with a {@link JdbcTableSinkOperator}. * Composes a SQL query from the stage's operators and executes it directly on diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java index 57d181791..47e920b62 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java @@ -38,7 +38,9 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashSet; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; @@ -100,7 +102,12 @@ void testWithHsqldb() throws SQLException { joinTask.setOutputChannel(0, new SqlQueryChannel(sqlChannelDescriptor, joinOperator.getOutput(0))); joinTask.setStage(sqlStage); - when(sqlStage.getStartTasks()).thenReturn(Collections.singleton(tableSourceATask)); + // Deliberately list the right source first: JdbcExecutor must still choose + // the join's left source for the FROM clause. + when(sqlStage.getStartTasks()).thenReturn(new LinkedHashSet<>(Arrays.asList( + tableSourceBTask, tableSourceATask))); + when(sqlStage.getAllTasks()).thenReturn(new LinkedHashSet<>(Arrays.asList( + tableSourceBTask, tableSourceATask, joinTask))); when(sqlStage.getTerminalTasks()).thenReturn(Collections.singleton(joinTask)); ExecutionStage nextStage = mock(ExecutionStage.class); From 96170578b7489c1cb695755bcd7177dfb8828494 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Tue, 16 Jun 2026 02:15:29 +0800 Subject: [PATCH 07/18] Add JavaPlanBuilder Trino combination tests --- wayang-platforms/wayang-trino/pom.xml | 6 + .../apache/wayang/trino/TrinoOperatorsIT.java | 237 +++++++++++++++--- 2 files changed, 207 insertions(+), 36 deletions(-) diff --git a/wayang-platforms/wayang-trino/pom.xml b/wayang-platforms/wayang-trino/pom.xml index 9f1a6fa83..5e8592352 100644 --- a/wayang-platforms/wayang-trino/pom.xml +++ b/wayang-platforms/wayang-trino/pom.xml @@ -59,6 +59,12 @@ wayang-spark 1.1.2-SNAPSHOT + + org.apache.wayang + wayang-api-scala-java + 1.1.2-SNAPSHOT + test + org.junit.jupiter junit-jupiter diff --git a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java index 1586ce55b..9437a159a 100644 --- a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java +++ b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java @@ -18,10 +18,14 @@ package org.apache.wayang.trino; +import org.apache.wayang.api.DataQuantaBuilder; +import org.apache.wayang.api.JavaPlanBuilder; import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.data.Tuple2; import org.apache.wayang.basic.function.ProjectionDescriptor; import org.apache.wayang.basic.operators.FilterOperator; import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.basic.operators.JoinOperator; import org.apache.wayang.basic.operators.LocalCallbackSink; import org.apache.wayang.basic.operators.MapOperator; import org.apache.wayang.basic.operators.ReduceByOperator; @@ -36,10 +40,7 @@ import org.apache.wayang.core.plan.wayangplan.WayangPlan; import org.apache.wayang.core.types.DataSetType; import org.apache.wayang.java.Java; -import org.apache.wayang.jdbc.compiler.FunctionCompiler; -import org.apache.wayang.trino.operators.TrinoJoinOperator; import org.apache.wayang.trino.operators.TrinoTableSource; -import org.apache.wayang.trino.platform.TrinoPlatform; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; @@ -53,6 +54,7 @@ import java.sql.ResultSet; import java.sql.Statement; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -226,45 +228,44 @@ void projection() { } /** - * Join: orders ⋈ customers on customer_id. + * Join: orders and customers on customer_id. * - *

Unlike the other operators, a JDBC join cannot be driven through the - * high-level {@link WayangContext} API here: the logical {@link JoinOperator} - * emits {@code Tuple2}, which has no valid connection to a - * {@code Record} sink (the SQL pushdown that would flatten it happens only - * after optimization). So we verify the operator's real contract directly: - * {@link TrinoJoinOperator#createSqlClause} must produce a Trino-valid JOIN - * clause that, executed against live Trino, returns the correct rows. + *

The logical {@link JoinOperator} emits {@code Tuple2}, + * while a pushed-down JDBC join already emits a flat {@link Record}. The + * following map normalizes both representations before the result reaches + * the sink. */ @Test @Order(4) - void join() throws Exception { + void join() { Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); - TrinoJoinOperator join = new TrinoJoinOperator<>( - new TransformationDescriptor<>( - (Record r) -> new Record(r.getField(1)), Record.class, Record.class - ).withSqlImplementation(ORDERS, "customer_id"), - new TransformationDescriptor<>( - (Record r) -> new Record(r.getField(0)), Record.class, Record.class - ).withSqlImplementation(CUSTOMERS, "customer_id")); - - // The operator routes to the Trino platform. - assertEquals(TrinoPlatform.getInstance(), join.getPlatform()); + List rows = execute(results -> { + TrinoTableSource orders = new TrinoTableSource( + ORDERS, "order_id", "customer_id", "region", "amount"); + TrinoTableSource customers = new TrinoTableSource( + CUSTOMERS, "customer_id", "name", "tier"); + JoinOperator join = new JoinOperator<>( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(1)), Record.class, Record.class + ).withSqlImplementation(ORDERS, "customer_id"), + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(0)), Record.class, Record.class + ).withSqlImplementation(CUSTOMERS, "customer_id")); + MapOperator flatten = new MapOperator<>( + TrinoOperatorsIT::flattenJoinResult, Object.class, Record.class); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + orders.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sink, 0); + return sink; + }); - try (Connection c = jdbc()) { - // Operator-generated JOIN clause, assembled into the full SELECT exactly - // as JdbcExecutor would (FROM + ). - String joinClause = join.createSqlClause(c, new FunctionCompiler()); - assertTrue(joinClause.startsWith("JOIN " + CUSTOMERS + " ON"), - "unexpected join clause: " + joinClause); - - String sql = "SELECT * FROM " + ORDERS + " " + joinClause; - ResultSet rs = c.createStatement().executeQuery(sql); - int n = 0; - while (rs.next()) n++; - // Every order's customer_id exists in customers → one joined row per order. - assertEquals(120000, n, "join should yield one row per order"); - } + assertEquals(120000, rows.size(), "join should yield one row per order"); + // Every order's customer_id exists in customers, so there is one joined row per order. + assertTrue(rows.stream().allMatch(r -> r.getField(1).equals(r.getField(4))), + "joined customer IDs should match"); + assertSqlReachedTrino("JOIN " + CUSTOMERS); } /** GlobalReduce: SUM(amount) over the whole table collapses to a single row. */ @@ -382,6 +383,153 @@ void tableSink() throws Exception { // ── Helpers ─────────────────────────────────────────────────────────────── + /** + * JavaPlanBuilder API: read a table, filter it, project two columns, and + * collect the result through a complete high-level Wayang plan. + */ + @Test + @Order(9) + void javaPlanBuilderReadTableFilterProjection() { + Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); + + Collection rows = new JavaPlanBuilder( + wayangContext(), "Trino JavaPlanBuilder readTable integration test") + .readTable(new TrinoTableSource( + ORDERS, "order_id", "customer_id", "region", "amount")) + .filter(record -> "AMER".equals(record.getField(2))) + .withSqlUdf("region = 'AMER'") + .asRecords() + .projectRecords(new String[]{"order_id", "amount"}) + .collect(); + + assertEquals(60000, rows.size(), "60000 projected AMER orders expected"); + assertTrue(rows.stream().allMatch(record -> record.size() == 2), + "projection should retain only order_id and amount"); + assertTrue(rows.stream().allMatch(record -> { + double amount = ((Number) record.getField(1)).doubleValue(); + return amount == 2200.0 || amount == 680.5 || amount == 950.25; + }), "only AMER order amounts should remain"); + assertSqlReachedTrino( + "SELECT order_id, amount FROM " + ORDERS + " WHERE region = 'AMER'"); + } + + /** + * JavaPlanBuilder API: combine a filter with a global reduction. + */ + @Test + @Order(10) + void javaPlanBuilderReadTableFilterGlobalReduce() { + Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); + + Collection rows = new JavaPlanBuilder( + wayangContext(), "Trino JavaPlanBuilder global reduce integration test") + .readTable(new TrinoTableSource( + ORDERS, "order_id", "customer_id", "region", "amount")) + .filter(record -> "AMER".equals(record.getField(2))) + .withSqlUdf("region = 'AMER'") + .reduce((left, right) -> left) + .withSqlUdf("SUM(amount)") + .collect(); + + assertEquals(1, rows.size(), "global reduction should return one row"); + Record sum = rows.iterator().next(); + assertEquals(76_615_000.0, ((Number) sum.getField(0)).doubleValue(), 0.01); + assertSqlReachedTrino( + "SELECT SUM(amount) FROM " + ORDERS + " WHERE region = 'AMER'"); + } + + /** + * JavaPlanBuilder API: group by region, aggregate each group, and sort the + * grouped result. + */ + @Test + @Order(11) + void javaPlanBuilderReadTableReduceBySort() { + Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); + + List rows = new ArrayList<>(new JavaPlanBuilder( + wayangContext(), "Trino JavaPlanBuilder reduce-by and sort integration test") + .readTable(new TrinoTableSource( + ORDERS, "order_id", "customer_id", "region", "amount")) + .reduceByKey( + record -> new Record(record.getField(2)), + (left, right) -> left) + .withSqlUdfs("region", "SUM(amount)") + .sort(record -> new Record(record.getField(0))) + .withSqlUdf("region", "ASC") + .collect()); + + assertEquals(3, rows.size(), "one row per region expected"); + assertEquals("AMER", rows.get(0).getField(0)); + assertEquals("APAC", rows.get(1).getField(0)); + assertEquals("EMEA", rows.get(2).getField(0)); + assertSqlReachedTrino( + "SELECT region,SUM(amount) FROM " + ORDERS + " GROUP BY region ORDER BY region ASC"); + } + + /** + * JavaPlanBuilder API: compose a filtered projection directly into a table + * sink so all processing remains in Trino. + */ + @Test + @Order(12) + void javaPlanBuilderReadTableFilterProjectionTableSink() throws Exception { + Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); + + new JavaPlanBuilder(wayangContext(), "Trino JavaPlanBuilder table sink integration test") + .readTable(new TrinoTableSource( + ORDERS, "order_id", "customer_id", "region", "amount")) + .filter(record -> "AMER".equals(record.getField(2))) + .withSqlUdf("region = 'AMER'") + .asRecords() + .projectRecords(new String[]{"order_id", "amount"}) + .writeTable( + SINK_TABLE, + "overwrite", + new String[]{"order_id", "amount"}, + new Properties()); + + try (Connection c = jdbc()) { + ResultSet rs = c.createStatement().executeQuery("SELECT count(*) FROM " + SINK_TABLE); + rs.next(); + assertEquals(60000, rs.getLong(1), "sink table should contain projected AMER orders"); + } + assertSqlReachedTrino( + "CREATE TABLE " + SINK_TABLE + + " AS SELECT order_id, amount FROM " + ORDERS + " WHERE region = 'AMER'"); + } + + /** + * JavaPlanBuilder API: join two tables and normalize the current logical + * and JDBC join output representations before collecting the result. + */ + @Test + @Order(13) + void javaPlanBuilderReadTableJoin() { + Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); + + JavaPlanBuilder plan = new JavaPlanBuilder( + wayangContext(), "Trino JavaPlanBuilder join integration test"); + DataQuantaBuilder orders = plan.readTable(new TrinoTableSource( + ORDERS, "order_id", "customer_id", "region", "amount")); + DataQuantaBuilder customers = plan.readTable(new TrinoTableSource( + CUSTOMERS, "customer_id", "name", "tier")); + + Collection rows = orders + .join( + record -> new Record(record.getField(1)), + customers, + record -> new Record(record.getField(0))) + .withSqlUdfs(ORDERS, "customer_id", CUSTOMERS, "customer_id") + .asRecords() + .collect(); + + assertEquals(120000, rows.size(), "join should yield one row per order"); + assertTrue(rows.stream().allMatch(row -> row.getField(1).equals(row.getField(4))), + "joined customer IDs should match"); + assertSqlReachedTrino("JOIN " + CUSTOMERS); + } + /** Functional builder: given the result list, wire a plan and return its sink. */ private interface PlanBuilder { LocalCallbackSink build(List resultCollector); @@ -404,6 +552,23 @@ private List execute(PlanBuilder builder) { return results; } + private static Record flattenJoinResult(Object joinResult) { + if (joinResult instanceof Record) { + return (Record) joinResult; + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record( + left.getField(0), + left.getField(1), + left.getField(2), + left.getField(3), + right.getField(0), + right.getField(1), + right.getField(2)); + } + /** Assert that Trino actually ran a query containing the given fragment. */ private void assertSqlReachedTrino(String fragment) { try (Connection c = jdbc()) { From 2f9dbfb89d7af3c44979c4e67559ce07ff3b0949 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Tue, 16 Jun 2026 02:37:45 +0800 Subject: [PATCH 08/18] Normalize Trino test comments to ASCII --- .../org/apache/wayang/trino/TrinoOperatorsIT.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java index 9437a159a..6910d61ba 100644 --- a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java +++ b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java @@ -69,7 +69,7 @@ * *

Coverage: {@code TableSource}, {@code Filter}, {@code Projection}, * {@code Join}, {@code GlobalReduce}, {@code ReduceBy}, {@code Sort}, - * {@code TableSink} — plus the SQL→Stream channel conversion that materialises + * {@code TableSink}, plus the SQL-to-Stream channel conversion that materialises * every result. Each test also asserts, via Trino's {@code system.runtime.queries}, * that the expected SQL actually reached Trino (i.e. the operator was pushed * down, not silently executed elsewhere). @@ -101,7 +101,7 @@ class TrinoOperatorsIT { private static boolean trinoAvailable = false; - // ── Lifecycle ─────────────────────────────────────────────────────────── + // Lifecycle @BeforeAll static void setUp() { @@ -161,7 +161,7 @@ static void tearDown() { } } - // ── Tests (one per operator) ────────────────────────────────────────────── + // Tests (one per operator) /** TableSource: full scan returns every row. */ @Test @@ -285,7 +285,7 @@ void globalReduce() { return sink; }); assertEquals(1, rows.size(), "global reduce must collapse to a single row"); - // 6 base rows sum to 7231.25; scaled x20000 → 144,625,000 (exact in doubles). + // 6 base rows sum to 7231.25; scaled x20000 gives 144,625,000 (exact in doubles). assertEquals(144_625_000.0, ((Number) rows.get(0).getField(0)).doubleValue(), 0.01); assertSqlReachedTrino("SELECT SUM(amount) FROM " + ORDERS); } @@ -351,7 +351,7 @@ void sort() { /** * TableSink: filter + sink composed into a single {@code CREATE TABLE ... AS - * SELECT} that runs entirely inside Trino — no data leaves the database. + * SELECT} that runs entirely inside Trino; no data leaves the database. */ @Test @Order(8) @@ -381,7 +381,7 @@ void tableSink() throws Exception { assertSqlReachedTrino("CREATE TABLE " + SINK_TABLE + " AS"); } - // ── Helpers ─────────────────────────────────────────────────────────────── + // JavaPlanBuilder combination tests /** * JavaPlanBuilder API: read a table, filter it, project two columns, and From fa9af3d59ec6c430cef62f553ba35e809d15e62d Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Thu, 18 Jun 2026 00:52:02 +0800 Subject: [PATCH 09/18] Update Trino setup docs for full join suite --- trino-setup/README.md | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/trino-setup/README.md b/trino-setup/README.md index 4562625dc..91b7526b6 100644 --- a/trino-setup/README.md +++ b/trino-setup/README.md @@ -99,13 +99,17 @@ docker compose -f trino-setup/docker-compose.yml ps `TrinoOperatorsIT` exercises the Wayang Trino implementation against the live Trino stack. It checks `TableSource`, `Filter`, `Projection`, `Join`, `GlobalReduce`, `ReduceBy`, `Sort`, and `TableSink`, and confirms that the -expected SQL reached Trino. Most tests run a Wayang plan end to end; the join -test directly validates and executes the SQL clause generated by -`TrinoJoinOperator`. +expected SQL reached Trino. The standalone join test now runs a full Wayang +plan and normalizes both possible join result shapes before collecting records: +logical joins can produce `Tuple2`, while pushed-down JDBC joins +can return a flat `Record`. The suite is self-contained: it creates `iceberg.wayang_it`, scales its test data to 120,000 rows so the optimizer selects SQL pushdown, and drops its test -tables afterward. It does not require `scripts/init.sql`. +tables afterward. It does not require `scripts/init.sql`. The suite also +contains five JavaPlanBuilder `readTable` combination tests that cover filter, +projection, global reduce, reduce-by plus sort, table sink, and join through +the public API. ```bash ./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-trino -am \ @@ -122,9 +126,12 @@ On PowerShell: Expected result: ```text -Tests run: 8, Failures: 0, Errors: 0, Skipped: 0 +Tests run: 13, Failures: 0, Errors: 0, Skipped: 0 ``` +Verified on June 18, 2026 against the local Docker stack with the full-plan +join test and all five JavaPlanBuilder combination tests enabled. + If Trino is unreachable, these tests are skipped instead of failed. A result with skipped tests does not confirm that the operators work. @@ -210,11 +217,16 @@ The `-v` option removes volumes and clears the local MinIO and PostgreSQL data. | `tableSource` | Full table scan through `TrinoTableSource` | | `filter` | Wayang `FilterOperator` and SQL `WHERE` pushdown | | `projection` | Column projection pushed into the Trino query | -| `join` | Trino join operator generates and executes a valid join clause | +| `join` | Full Wayang join plan with normalization before the collecting sink | | `globalReduce` | Global aggregation such as `SUM` | | `reduceBy` | Grouped aggregation and SQL `GROUP BY` | | `sort` | Wayang sort and SQL `ORDER BY` | | `tableSink` | Filtered result written with `CREATE TABLE AS` | +| `javaPlanBuilderReadTableFilterProjection` | `readTable -> filter -> projection -> collect` | +| `javaPlanBuilderReadTableFilterGlobalReduce` | `readTable -> filter -> globalReduce -> collect` | +| `javaPlanBuilderReadTableReduceBySort` | `readTable -> reduceByKey -> sort -> collect` | +| `javaPlanBuilderReadTableFilterProjectionTableSink` | `readTable -> filter -> projection -> writeTable` | +| `javaPlanBuilderReadTableJoin` | `readTable + readTable -> join -> collect` | ### Standalone stack integration tests From bca1e032fed983121d5e5251af2d8c7360b8cd51 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Thu, 18 Jun 2026 00:57:34 +0800 Subject: [PATCH 10/18] Document Trino macOS and Windows commands --- trino-setup/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/trino-setup/README.md b/trino-setup/README.md index 91b7526b6..e92c4014b 100644 --- a/trino-setup/README.md +++ b/trino-setup/README.md @@ -17,6 +17,12 @@ The pure Trino platform branch is named `wayang-trino`: git checkout wayang-trino ``` +## Command Conventions + +Use the `bash` blocks on macOS/Linux terminals. Use the `powershell` blocks on +Windows PowerShell from the repository root. Docker Compose commands are the +same on both platforms. + ## Stack | Component | Image | Port | Role | @@ -250,3 +256,12 @@ Override defaults if running Trino on a different host/port: ```bash TRINO_HOST=my-trino-host TRINO_PORT=8080 ./mvnw -f trino-setup/pom.xml -Pintegration -Dtest=TrinoIntegrationTest test ``` + +On PowerShell: + +```powershell +$env:TRINO_HOST="my-trino-host" +$env:TRINO_PORT="8080" +.\mvnw.cmd --% -f trino-setup/pom.xml -Pintegration -Dtest=TrinoIntegrationTest test +Remove-Item Env:TRINO_HOST, Env:TRINO_PORT +``` From 651fd14f25820d8de0e8dca8fe1f02313a74f52b Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Sat, 20 Jun 2026 23:44:37 +0800 Subject: [PATCH 11/18] Make Trino integration tests engine-only --- .../wayang/jdbc/execution/JdbcExecutor.java | 22 +- .../apache/wayang/trino/TrinoOperatorsIT.java | 465 +++++++++++------- 2 files changed, 303 insertions(+), 184 deletions(-) diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java index 8596c90f1..ae27c1aef 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java @@ -232,8 +232,7 @@ private static void executeSinkStage(final ExecutionStage stage, final Optimizat final Collection startTasks = stage.getStartTasks(); final Collection termTasks = stage.getTerminalTasks(); - assert startTasks.size() == 1 : "Invalid JDBC stage: multiple sources are not currently supported"; - final ExecutionTask startTask = (ExecutionTask) startTasks.toArray()[0]; + final ExecutionTask startTask = JdbcExecutor.selectStartTask(startTasks, stage); assert termTasks.size() == 1 : "Invalid JDBC stage: multiple terminal tasks are not currently supported."; final ExecutionTask termTask = (ExecutionTask) termTasks.toArray()[0]; assert startTask.getOperator() instanceof TableSource @@ -246,6 +245,9 @@ private static void executeSinkStage(final ExecutionStage stage, final Optimizat final JdbcTableSinkOperator sinkOp = (JdbcTableSinkOperator) termTask.getOperator(); final Collection filterTasks = new ArrayList<>(4); JdbcProjectionOperator projectionTask = null; + JdbcGlobalReduceOperator globalReduceTask = null; + JdbcReduceByOperator reduceByTask = null; + JdbcSortOperator sortTask = null; final Collection joinTasks = new ArrayList<>(); // Walk through intermediate operators, stopping at the sink @@ -258,6 +260,18 @@ private static void executeSinkStage(final ExecutionStage stage, final Optimizat final JdbcProjectionOperator projectionOperator = (JdbcProjectionOperator) nextTask.getOperator(); assert projectionTask == null; projectionTask = projectionOperator; + } else if (nextTask.getOperator() instanceof JdbcGlobalReduceOperator) { + final JdbcGlobalReduceOperator globalReduceOperator = (JdbcGlobalReduceOperator) nextTask.getOperator(); + assert globalReduceTask == null; + globalReduceTask = globalReduceOperator; + } else if (nextTask.getOperator() instanceof JdbcReduceByOperator) { + final JdbcReduceByOperator reduceByOperator = (JdbcReduceByOperator) nextTask.getOperator(); + assert reduceByTask == null; + reduceByTask = reduceByOperator; + } else if (nextTask.getOperator() instanceof JdbcSortOperator) { + final JdbcSortOperator sortOperator = (JdbcSortOperator) nextTask.getOperator(); + assert sortTask == null; + sortTask = sortOperator; } else if (nextTask.getOperator() instanceof JdbcJoinOperator) { final JdbcJoinOperator joinOperator = (JdbcJoinOperator) nextTask.getOperator(); joinTasks.add(joinOperator); @@ -268,8 +282,8 @@ private static void executeSinkStage(final ExecutionStage stage, final Optimizat } // Compose the SELECT query - final StringBuilder selectQuery = createSqlString(jdbcExecutor, tableOp, filterTasks, projectionTask, null, null, null, - joinTasks); + final StringBuilder selectQuery = createSqlString(jdbcExecutor, tableOp, filterTasks, projectionTask, + globalReduceTask, reduceByTask, sortTask, joinTasks); // Remove trailing semicolon from SELECT String selectSql = selectQuery.toString(); diff --git a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java index 6910d61ba..44b027b49 100644 --- a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java +++ b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoOperatorsIT.java @@ -26,7 +26,6 @@ import org.apache.wayang.basic.operators.FilterOperator; import org.apache.wayang.basic.operators.GlobalReduceOperator; import org.apache.wayang.basic.operators.JoinOperator; -import org.apache.wayang.basic.operators.LocalCallbackSink; import org.apache.wayang.basic.operators.MapOperator; import org.apache.wayang.basic.operators.ReduceByOperator; import org.apache.wayang.basic.operators.SortOperator; @@ -34,13 +33,21 @@ import org.apache.wayang.basic.types.RecordType; import org.apache.wayang.core.api.Configuration; import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.FunctionDescriptor; import org.apache.wayang.core.function.PredicateDescriptor; import org.apache.wayang.core.function.ReduceDescriptor; import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.types.DataUnitType; import org.apache.wayang.core.types.DataSetType; -import org.apache.wayang.java.Java; +import org.apache.wayang.trino.operators.TrinoProjectionOperator; import org.apache.wayang.trino.operators.TrinoTableSource; +import org.apache.wayang.trino.platform.TrinoPlatform; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; @@ -53,14 +60,14 @@ import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.Statement; -import java.util.ArrayList; -import java.util.Collection; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Properties; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -69,10 +76,9 @@ * *

Coverage: {@code TableSource}, {@code Filter}, {@code Projection}, * {@code Join}, {@code GlobalReduce}, {@code ReduceBy}, {@code Sort}, - * {@code TableSink}, plus the SQL-to-Stream channel conversion that materialises - * every result. Each test also asserts, via Trino's {@code system.runtime.queries}, - * that the expected SQL actually reached Trino (i.e. the operator was pushed - * down, not silently executed elsewhere). + * and {@code TableSink}. Every Wayang plan ends in a Trino table sink so the + * execution itself does not require the Java plugin. Result assertions use + * plain JDBC only after the Wayang execution has completed. * *

Prerequisites: a Trino reachable at {@code TRINO_HOST:TRINO_PORT} * (defaults {@code localhost:8080}); e.g. {@code cd trino-setup && docker compose up -d}. @@ -97,7 +103,12 @@ class TrinoOperatorsIT { private static final String SCHEMA = "iceberg.wayang_it"; private static final String ORDERS = SCHEMA + ".orders"; private static final String CUSTOMERS = SCHEMA + ".customers"; - private static final String SINK_TABLE = SCHEMA + ".amer_orders"; + private static final String SINK_TABLE_NAME = "operator_result"; + private static final String SINK_TABLE = SCHEMA + "." + SINK_TABLE_NAME; + private static final String[] JOIN_COLUMNS = { + "order_id", "customer_id", "region", "amount", "cust_id", "name", "tier" + }; + private static final String JOIN_FLATTEN_NAME = "Trino test-only join flatten"; private static boolean trinoAvailable = false; @@ -134,7 +145,7 @@ static void setUp() { st.execute("DROP TABLE IF EXISTS " + CUSTOMERS); st.execute("CREATE TABLE " + CUSTOMERS + " (" - + "customer_id BIGINT, name VARCHAR, tier VARCHAR)" + + "cust_id BIGINT, name VARCHAR, tier VARCHAR)" + " WITH (format = 'PARQUET')"); st.execute("INSERT INTO " + CUSTOMERS + " VALUES " + "(100, 'Acme', 'GOLD')," @@ -168,13 +179,14 @@ static void tearDown() { @Order(1) void tableSource() { Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); - List rows = execute(results -> { - TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); - LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); - src.connectTo(0, sink, 0); - return sink; - }); - assertEquals(120000, rows.size(), "TableSource should return all orders"); + TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + TableSink sink = tableSink("order_id", "customer_id", "region", "amount"); + src.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(120000, queryLong("SELECT count(*) FROM " + SINK_TABLE), + "TableSource should return all orders"); assertSqlReachedTrino("SELECT * FROM " + ORDERS); } @@ -183,19 +195,20 @@ void tableSource() { @Order(2) void filter() { Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); - List rows = execute(results -> { - TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); - FilterOperator filter = new FilterOperator<>( - new PredicateDescriptor<>( - (Record r) -> "AMER".equals(r.getField(2)), Record.class - ).withSqlImplementation("region = 'AMER'")); - LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); - src.connectTo(0, filter, 0); - filter.connectTo(0, sink, 0); - return sink; - }); - assertEquals(60000, rows.size(), "60000 AMER orders expected"); - assertTrue(rows.stream().allMatch(r -> "AMER".equals(r.getField(2))), "all rows AMER"); + TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record r) -> "AMER".equals(r.getField(2)), Record.class + ).withSqlImplementation("region = 'AMER'")); + TableSink sink = tableSink("order_id", "customer_id", "region", "amount"); + src.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(60000, queryLong("SELECT count(*) FROM " + SINK_TABLE), + "60000 AMER orders expected"); + assertEquals(0, queryLong("SELECT count_if(region <> 'AMER') FROM " + SINK_TABLE), "all rows AMER"); assertSqlReachedTrino("WHERE region = 'AMER'"); } @@ -204,66 +217,67 @@ void filter() { @Order(3) void projection() { Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); - List rows = execute(results -> { - TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); - FilterOperator filter = new FilterOperator<>( - new PredicateDescriptor<>( - (Record r) -> "AMER".equals(r.getField(2)), Record.class - ).withSqlImplementation("region = 'AMER'")); - MapOperator projection = new MapOperator<>( - ProjectionDescriptor.createForRecords( - new RecordType("order_id", "customer_id", "region", "amount"), - "region", "amount"), - DataSetType.createDefault(Record.class), - DataSetType.createDefault(Record.class)); - LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); - src.connectTo(0, filter, 0); - filter.connectTo(0, projection, 0); - projection.connectTo(0, sink, 0); - return sink; - }); - assertEquals(60000, rows.size(), "60000 AMER rows expected"); - assertEquals(2, rows.get(0).size(), "projection keeps only 2 columns"); + TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record r) -> "AMER".equals(r.getField(2)), Record.class + ).withSqlImplementation("region = 'AMER'")); + MapOperator projection = new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType("order_id", "customer_id", "region", "amount"), + "region", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("region", "amount"); + src.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(60000, queryLong("SELECT count(*) FROM " + SINK_TABLE), "60000 AMER rows expected"); + assertEquals(2, queryLong( + "SELECT count(*) FROM iceberg.information_schema.columns " + + "WHERE table_schema = 'wayang_it' AND table_name = '" + SINK_TABLE_NAME + "'"), + "projection keeps only 2 columns"); assertSqlReachedTrino("SELECT region, amount FROM " + ORDERS); } /** * Join: orders and customers on customer_id. * - *

The logical {@link JoinOperator} emits {@code Tuple2}, - * while a pushed-down JDBC join already emits a flat {@link Record}. The - * following map normalizes both representations before the result reaches - * the sink. + *

The logical join emits {@code Tuple2}. A test-only + * mapping turns the following flatten map into a Trino SQL projection, so + * this plan still executes entirely in Trino without deciding the general + * Tuple-to-Record semantics for JDBC platforms. */ @Test @Order(4) void join() { Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); - List rows = execute(results -> { - TrinoTableSource orders = new TrinoTableSource( - ORDERS, "order_id", "customer_id", "region", "amount"); - TrinoTableSource customers = new TrinoTableSource( - CUSTOMERS, "customer_id", "name", "tier"); - JoinOperator join = new JoinOperator<>( - new TransformationDescriptor<>( - (Record r) -> new Record(r.getField(1)), Record.class, Record.class - ).withSqlImplementation(ORDERS, "customer_id"), - new TransformationDescriptor<>( - (Record r) -> new Record(r.getField(0)), Record.class, Record.class - ).withSqlImplementation(CUSTOMERS, "customer_id")); - MapOperator flatten = new MapOperator<>( - TrinoOperatorsIT::flattenJoinResult, Object.class, Record.class); - LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); - orders.connectTo(0, join, 0); - customers.connectTo(0, join, 1); - join.connectTo(0, flatten, 0); - flatten.connectTo(0, sink, 0); - return sink; - }); - - assertEquals(120000, rows.size(), "join should yield one row per order"); - // Every order's customer_id exists in customers, so there is one joined row per order. - assertTrue(rows.stream().allMatch(r -> r.getField(1).equals(r.getField(4))), + TrinoTableSource orders = new TrinoTableSource( + ORDERS, "order_id", "customer_id", "region", "amount"); + TrinoTableSource customers = new TrinoTableSource( + CUSTOMERS, "cust_id", "name", "tier"); + JoinOperator join = new JoinOperator<>( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(1)), Record.class, Record.class + ).withSqlImplementation(ORDERS, "customer_id"), + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(0)), Record.class, Record.class + ).withSqlImplementation(CUSTOMERS, "cust_id")); + MapOperator, Record> flatten = joinFlattenOperator(); + TableSink sink = tableSink(JOIN_COLUMNS); + orders.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(120000, queryLong("SELECT count(*) FROM " + SINK_TABLE), + "join should yield one row per order"); + assertEquals(0, queryLong("SELECT count_if(customer_id <> cust_id) FROM " + SINK_TABLE), "joined customer IDs should match"); assertSqlReachedTrino("JOIN " + CUSTOMERS); } @@ -273,21 +287,20 @@ void join() { @Order(5) void globalReduce() { Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); - List rows = execute(results -> { - TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); - GlobalReduceOperator reduce = new GlobalReduceOperator<>( - new ReduceDescriptor<>((a, b) -> a, Record.class) - .withSqlImplementation("SUM(amount)"), - DataSetType.createDefault(Record.class)); - LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); - src.connectTo(0, reduce, 0); - reduce.connectTo(0, sink, 0); - return sink; - }); - assertEquals(1, rows.size(), "global reduce must collapse to a single row"); + TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + GlobalReduceOperator reduce = new GlobalReduceOperator<>( + new ReduceDescriptor<>((a, b) -> a, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("total_amount"); + src.connectTo(0, reduce, 0); + reduce.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + // 6 base rows sum to 7231.25; scaled x20000 gives 144,625,000 (exact in doubles). - assertEquals(144_625_000.0, ((Number) rows.get(0).getField(0)).doubleValue(), 0.01); - assertSqlReachedTrino("SELECT SUM(amount) FROM " + ORDERS); + assertSingleDoubleResult(144_625_000.0, "global reduce must collapse to a single row"); + assertSqlReachedTrino("SELECT SUM(amount) AS total_amount FROM " + ORDERS); } /** ReduceBy: SUM(amount) GROUP BY region yields one row per region. */ @@ -295,25 +308,22 @@ void globalReduce() { @Order(6) void reduceBy() { Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); - List rows = execute(results -> { - TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); - ReduceByOperator reduceBy = new ReduceByOperator<>( - new TransformationDescriptor<>( - (Record r) -> new Record(r.getField(2)), Record.class, Record.class - ).withSqlImplementation("region", "region"), - new ReduceDescriptor<>((a, b) -> a, Record.class) - .withSqlImplementation("SUM(amount)"), - DataSetType.createDefault(Record.class)); - LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); - src.connectTo(0, reduceBy, 0); - reduceBy.connectTo(0, sink, 0); - return sink; - }); - assertEquals(3, rows.size(), "one row per region expected"); - Map sums = new HashMap<>(); - for (Record r : rows) { - sums.put((String) r.getField(0), ((Number) r.getField(1)).doubleValue()); - } + TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + ReduceByOperator reduceBy = new ReduceByOperator<>( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(2)), Record.class, Record.class + ).withSqlImplementation("region", "region"), + new ReduceDescriptor<>((a, b) -> a, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("region", "total_amount"); + src.connectTo(0, reduceBy, 0); + reduceBy.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + Map sums = readRegionSums(); + assertEquals(3, sums.size(), "one row per region expected"); // Base sums (AMER 3830.75, EMEA 1900.5, APAC 1500.0) scaled x20000. assertEquals(76_615_000.0, sums.get("AMER"), 0.01); assertEquals(38_010_000.0, sums.get("EMEA"), 0.01); @@ -326,26 +336,22 @@ void reduceBy() { @Order(7) void sort() { Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); - List rows = execute(results -> { - TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); - SortOperator sort = new SortOperator<>( - new TransformationDescriptor<>( - (Record r) -> new Record(r.getField(3)), Record.class, Record.class - ).withSqlImplementation("amount", "ASC"), - DataSetType.createDefault(Record.class)); - LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); - src.connectTo(0, sort, 0); - sort.connectTo(0, sink, 0); - return sink; - }); - assertEquals(120000, rows.size(), "sort must not change the cardinality"); - assertEquals(680.5, ((Number) rows.get(0).getField(3)).doubleValue(), 0.001, "smallest amount first"); - assertEquals(2200.0, ((Number) rows.get(rows.size() - 1).getField(3)).doubleValue(), 0.001, "largest amount last"); - for (int i = 1; i < rows.size(); i++) { - double prev = ((Number) rows.get(i - 1).getField(3)).doubleValue(); - double curr = ((Number) rows.get(i).getField(3)).doubleValue(); - assertTrue(prev <= curr, "rows must be non-decreasing by amount at index " + i); - } + TrinoTableSource src = new TrinoTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + SortOperator sort = new SortOperator<>( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(3)), Record.class, Record.class + ).withSqlImplementation("amount", "ASC"), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("order_id", "customer_id", "region", "amount"); + src.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(120000, queryLong("SELECT count(*) FROM " + SINK_TABLE), + "sort must not change the cardinality"); + assertEquals(680.5, queryDouble("SELECT min(amount) FROM " + SINK_TABLE), 0.001); + assertEquals(2200.0, queryDouble("SELECT max(amount) FROM " + SINK_TABLE), 0.001); assertSqlReachedTrino("ORDER BY amount ASC"); } @@ -385,14 +391,14 @@ void tableSink() throws Exception { /** * JavaPlanBuilder API: read a table, filter it, project two columns, and - * collect the result through a complete high-level Wayang plan. + * write the result through a complete high-level Wayang plan. */ @Test @Order(9) void javaPlanBuilderReadTableFilterProjection() { Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); - Collection rows = new JavaPlanBuilder( + new JavaPlanBuilder( wayangContext(), "Trino JavaPlanBuilder readTable integration test") .readTable(new TrinoTableSource( ORDERS, "order_id", "customer_id", "region", "amount")) @@ -400,15 +406,13 @@ void javaPlanBuilderReadTableFilterProjection() { .withSqlUdf("region = 'AMER'") .asRecords() .projectRecords(new String[]{"order_id", "amount"}) - .collect(); - - assertEquals(60000, rows.size(), "60000 projected AMER orders expected"); - assertTrue(rows.stream().allMatch(record -> record.size() == 2), - "projection should retain only order_id and amount"); - assertTrue(rows.stream().allMatch(record -> { - double amount = ((Number) record.getField(1)).doubleValue(); - return amount == 2200.0 || amount == 680.5 || amount == 950.25; - }), "only AMER order amounts should remain"); + .writeTable(SINK_TABLE, "overwrite", new String[]{"order_id", "amount"}, new Properties()); + + assertEquals(60000, queryLong("SELECT count(*) FROM " + SINK_TABLE), + "60000 projected AMER orders expected"); + assertEquals(0, queryLong( + "SELECT count_if(amount NOT IN (2200.0, 680.5, 950.25)) FROM " + SINK_TABLE), + "only AMER order amounts should remain"); assertSqlReachedTrino( "SELECT order_id, amount FROM " + ORDERS + " WHERE region = 'AMER'"); } @@ -421,21 +425,19 @@ void javaPlanBuilderReadTableFilterProjection() { void javaPlanBuilderReadTableFilterGlobalReduce() { Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); - Collection rows = new JavaPlanBuilder( + new JavaPlanBuilder( wayangContext(), "Trino JavaPlanBuilder global reduce integration test") .readTable(new TrinoTableSource( ORDERS, "order_id", "customer_id", "region", "amount")) .filter(record -> "AMER".equals(record.getField(2))) .withSqlUdf("region = 'AMER'") .reduce((left, right) -> left) - .withSqlUdf("SUM(amount)") - .collect(); + .withSqlUdf("SUM(amount) AS total_amount") + .writeTable(SINK_TABLE, "overwrite", new String[]{"total_amount"}, new Properties()); - assertEquals(1, rows.size(), "global reduction should return one row"); - Record sum = rows.iterator().next(); - assertEquals(76_615_000.0, ((Number) sum.getField(0)).doubleValue(), 0.01); + assertSingleDoubleResult(76_615_000.0, "global reduction should return one row"); assertSqlReachedTrino( - "SELECT SUM(amount) FROM " + ORDERS + " WHERE region = 'AMER'"); + "SELECT SUM(amount) AS total_amount FROM " + ORDERS + " WHERE region = 'AMER'"); } /** @@ -447,24 +449,24 @@ void javaPlanBuilderReadTableFilterGlobalReduce() { void javaPlanBuilderReadTableReduceBySort() { Assumptions.assumeTrue(trinoAvailable, "Trino not reachable"); - List rows = new ArrayList<>(new JavaPlanBuilder( + new JavaPlanBuilder( wayangContext(), "Trino JavaPlanBuilder reduce-by and sort integration test") .readTable(new TrinoTableSource( ORDERS, "order_id", "customer_id", "region", "amount")) .reduceByKey( record -> new Record(record.getField(2)), (left, right) -> left) - .withSqlUdfs("region", "SUM(amount)") + .withSqlUdfs("region", "SUM(amount) AS total_amount") .sort(record -> new Record(record.getField(0))) .withSqlUdf("region", "ASC") - .collect()); + .writeTable(SINK_TABLE, "overwrite", new String[]{"region", "total_amount"}, new Properties()); - assertEquals(3, rows.size(), "one row per region expected"); - assertEquals("AMER", rows.get(0).getField(0)); - assertEquals("APAC", rows.get(1).getField(0)); - assertEquals("EMEA", rows.get(2).getField(0)); + assertEquals("AMER,APAC,EMEA", queryString( + "SELECT array_join(array_agg(region ORDER BY region), ',') FROM " + SINK_TABLE), + "one row per region expected"); assertSqlReachedTrino( - "SELECT region,SUM(amount) FROM " + ORDERS + " GROUP BY region ORDER BY region ASC"); + "SELECT region,SUM(amount) AS total_amount FROM " + ORDERS + + " GROUP BY region ORDER BY region ASC"); } /** @@ -500,8 +502,8 @@ void javaPlanBuilderReadTableFilterProjectionTableSink() throws Exception { } /** - * JavaPlanBuilder API: join two tables and normalize the current logical - * and JDBC join output representations before collecting the result. + * JavaPlanBuilder API: join two tables, flatten through the test-only Trino + * mapping, and write the result in Trino. */ @Test @Order(13) @@ -513,43 +515,50 @@ void javaPlanBuilderReadTableJoin() { DataQuantaBuilder orders = plan.readTable(new TrinoTableSource( ORDERS, "order_id", "customer_id", "region", "amount")); DataQuantaBuilder customers = plan.readTable(new TrinoTableSource( - CUSTOMERS, "customer_id", "name", "tier")); + CUSTOMERS, "cust_id", "name", "tier")); - Collection rows = orders + orders .join( record -> new Record(record.getField(1)), customers, record -> new Record(record.getField(0))) - .withSqlUdfs(ORDERS, "customer_id", CUSTOMERS, "customer_id") - .asRecords() - .collect(); - - assertEquals(120000, rows.size(), "join should yield one row per order"); - assertTrue(rows.stream().allMatch(row -> row.getField(1).equals(row.getField(4))), + .withSqlUdfs(ORDERS, "customer_id", CUSTOMERS, "cust_id") + .map(new JoinFlattenFunction()) + .withName(JOIN_FLATTEN_NAME) + .writeTable(SINK_TABLE, "overwrite", JOIN_COLUMNS, new Properties()); + + assertEquals(120000, queryLong("SELECT count(*) FROM " + SINK_TABLE), + "join should yield one row per order"); + assertEquals(0, queryLong("SELECT count_if(customer_id <> cust_id) FROM " + SINK_TABLE), "joined customer IDs should match"); assertSqlReachedTrino("JOIN " + CUSTOMERS); } - /** Functional builder: given the result list, wire a plan and return its sink. */ - private interface PlanBuilder { - LocalCallbackSink build(List resultCollector); - } - private WayangContext wayangContext() { Configuration config = new Configuration(); config.setProperty("wayang.trino.jdbc.url", JDBC_URL); config.setProperty("wayang.trino.jdbc.user", USER); config.setProperty("wayang.trino.jdbc.password", ""); + config.getMappingProvider().addAllToWhitelist( + Collections.singleton(new JoinFlattenMapping())); return new WayangContext(config) - .withPlugin(Java.basicPlugin()) .withPlugin(Trino.plugin()); } - private List execute(PlanBuilder builder) { - List results = new ArrayList<>(); - LocalCallbackSink sink = builder.build(results); - wayangContext().execute(new WayangPlan(sink)); - return results; + private TableSink tableSink(String... columnNames) { + return new TableSink<>(new Properties(), "overwrite", SINK_TABLE, columnNames); + } + + private static MapOperator, Record> joinFlattenOperator() { + MapOperator, Record> operator = new MapOperator<>( + new TransformationDescriptor<>( + new JoinFlattenFunction(), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class)), + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class)); + operator.setName(JOIN_FLATTEN_NAME); + return operator; } private static Record flattenJoinResult(Object joinResult) { @@ -569,6 +578,102 @@ private static Record flattenJoinResult(Object joinResult) { right.getField(2)); } + private long queryLong(String sql) { + try (Connection c = jdbc(); Statement statement = c.createStatement(); ResultSet rs = statement.executeQuery(sql)) { + rs.next(); + return rs.getLong(1); + } catch (Exception e) { + throw new RuntimeException("query failed: " + sql, e); + } + } + + private double queryDouble(String sql) { + try (Connection c = jdbc(); Statement statement = c.createStatement(); ResultSet rs = statement.executeQuery(sql)) { + rs.next(); + return rs.getDouble(1); + } catch (Exception e) { + throw new RuntimeException("query failed: " + sql, e); + } + } + + private String queryString(String sql) { + try (Connection c = jdbc(); Statement statement = c.createStatement(); ResultSet rs = statement.executeQuery(sql)) { + rs.next(); + return rs.getString(1); + } catch (Exception e) { + throw new RuntimeException("query failed: " + sql, e); + } + } + + private void assertSingleDoubleResult(double expected, String message) { + try (Connection c = jdbc(); Statement statement = c.createStatement(); + ResultSet rs = statement.executeQuery("SELECT * FROM " + SINK_TABLE)) { + assertTrue(rs.next(), message); + assertEquals(expected, rs.getDouble(1), 0.01, message); + assertFalse(rs.next(), message); + } catch (Exception e) { + throw new RuntimeException("query failed: SELECT * FROM " + SINK_TABLE, e); + } + } + + private Map readRegionSums() { + Map sums = new HashMap<>(); + try (Connection c = jdbc(); Statement statement = c.createStatement(); + ResultSet rs = statement.executeQuery("SELECT * FROM " + SINK_TABLE)) { + while (rs.next()) { + sums.put(rs.getString(1), rs.getDouble(2)); + } + return sums; + } catch (Exception e) { + throw new RuntimeException("query failed: SELECT * FROM " + SINK_TABLE, e); + } + } + + private static final class JoinFlattenFunction implements + FunctionDescriptor.SerializableFunction, Record> { + + @Override + public Record apply(Tuple2 tuple) { + return flattenJoinResult(tuple); + } + } + + /** Test-only mapping for the unresolved logical join Tuple-to-Record mismatch. */ + @SuppressWarnings({"rawtypes", "unchecked"}) + private static final class JoinFlattenMapping implements Mapping { + + @Override + public java.util.Collection getTransformations() { + OperatorPattern pattern = new OperatorPattern( + "joinFlatten", + new MapOperator(null, DataSetType.none(), DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(operator -> JOIN_FLATTEN_NAME.equals(((MapOperator) operator).getName())); + + ReplacementSubplanFactory factory = new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> createTrinoProjection().at(epoch)); + + return Collections.singleton(new PlanTransformation( + SubplanPattern.createSingleton(pattern), + factory, + TrinoPlatform.getInstance())); + } + + private static TrinoProjectionOperator createTrinoProjection() { + ProjectionDescriptor, Record> descriptor = new ProjectionDescriptor<>( + new JoinFlattenFunction(), + Arrays.asList(JOIN_COLUMNS), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class)); + MapOperator, Record> projection = new MapOperator<>( + descriptor, + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class)); + projection.setName(JOIN_FLATTEN_NAME); + return new TrinoProjectionOperator((MapOperator) (MapOperator) projection); + } + } + /** Assert that Trino actually ran a query containing the given fragment. */ private void assertSqlReachedTrino(String fragment) { try (Connection c = jdbc()) { From 73f1ecb309f8403ef5faec11df95152f5eeb28c2 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Sun, 28 Jun 2026 21:04:30 +0800 Subject: [PATCH 12/18] Add JDBC sink profiling support --- .../wayang/jdbc/execution/JdbcExecutor.java | 42 ++++++++++++++++++- .../execution/JdbcTableSinkExecutorTest.java | 40 +++++++++++++++++- .../profiler/log/GeneticOptimizerApp.java | 15 +++++++ 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java index ae27c1aef..e6b00fe69 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java @@ -44,7 +44,9 @@ import org.apache.wayang.core.platform.ExecutionState; import org.apache.wayang.core.platform.Executor; import org.apache.wayang.core.platform.ExecutorTemplate; +import org.apache.wayang.core.platform.PartialExecution; import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.platform.lineage.ExecutionLineageNode; import org.apache.wayang.core.util.WayangCollections; import org.apache.wayang.jdbc.channels.SqlQueryChannel; import org.apache.wayang.jdbc.compiler.FunctionCompiler; @@ -227,7 +229,7 @@ private static ExecutionTask selectStartTask(final Collection startTasks, fin * @param optimizationContext provides optimization information * @param jdbcExecutor the executor with the database connection */ - private static void executeSinkStage(final ExecutionStage stage, final OptimizationContext optimizationContext, + private static long executeSinkStage(final ExecutionStage stage, final OptimizationContext optimizationContext, final JdbcExecutor jdbcExecutor) { final Collection startTasks = stage.getStartTasks(); final Collection termTasks = stage.getTerminalTasks(); @@ -303,14 +305,43 @@ private static void executeSinkStage(final ExecutionStage stage, final Optimizat // Execute the composed query: CREATE TABLE x AS SELECT ... or INSERT INTO x // SELECT ... final String fullSql = sinkClause + " " + selectSql + sinkOp.createSqlSuffix(); + final long startTime = System.currentTimeMillis(); stmt.execute(fullSql); + final long executionDuration = System.currentTimeMillis() - startTime; jdbcExecutor.logger.info("Executed SQL sink: {}", fullSql); System.out.println("Executed sql sink: " + fullSql); + return executionDuration; } catch (final SQLException e) { throw new WayangException("Failed to execute SQL sink on table: " + sinkOp.getTableName(), e); } } + /** + * Creates lineage nodes for the JDBC operators that were executed as one SQL + * statement. Operators without an optimization context or load estimator are + * skipped, so JDBC platforms without cost specifications can still execute. + */ + private Collection createExecutionLineageNodes( + final ExecutionStage stage, + final OptimizationContext optimizationContext) { + final Collection executionLineageNodes = new ArrayList<>(); + for (ExecutionTask task : stage.getAllTasks()) { + final OptimizationContext.OperatorContext operatorContext = + optimizationContext.getOperatorContext(task.getOperator()); + if (operatorContext == null) { + this.logger.warn("Cannot profile {} because its optimization context is missing.", task); + continue; + } + if (operatorContext.getLoadProfileEstimator() == null) { + this.logger.warn("Cannot profile {} because its load profile estimator is missing.", task); + continue; + } + executionLineageNodes.add( + new ExecutionLineageNode(operatorContext).addAtomicExecutionFromOperatorContext()); + } + return executionLineageNodes; + } + /** * Retrieves the follow-up {@link ExecutionTask} of the given {@code task} * unless it is not comprising a {@link JdbcExecutionOperator} and/or not in the @@ -396,7 +427,14 @@ public void execute(final ExecutionStage stage, final OptimizationContext optimi final ExecutionTask termTask = (ExecutionTask) termTasks.toArray()[0]; if (termTask.getOperator() instanceof JdbcTableSinkOperator) { - JdbcExecutor.executeSinkStage(stage, optimizationContext, this); + final long executionDuration = JdbcExecutor.executeSinkStage(stage, optimizationContext, this); + final PartialExecution partialExecution = this.createPartialExecution( + this.createExecutionLineageNodes(stage, optimizationContext), + executionDuration + ); + if (partialExecution != null) { + executionState.add(partialExecution); + } } else { // If it is normal stage: compose SQL and store in channel for downstream // consumption diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java index 263730629..0b11fab1b 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java @@ -23,7 +23,9 @@ import org.apache.wayang.core.optimizer.DefaultOptimizationContext; import org.apache.wayang.core.plan.executionplan.ExecutionStage; import org.apache.wayang.core.plan.executionplan.ExecutionTask; +import org.apache.wayang.core.platform.AtomicExecution; import org.apache.wayang.core.platform.CrossPlatformExecutor; +import org.apache.wayang.core.platform.PartialExecution; import org.apache.wayang.core.profiling.NoInstrumentationStrategy; import org.apache.wayang.jdbc.channels.SqlQueryChannel; import org.apache.wayang.jdbc.operators.JdbcTableSinkOperator; @@ -37,7 +39,11 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; @@ -51,6 +57,16 @@ class JdbcTableSinkExecutorTest { @Test void testOverwriteModeCreatesNewTable() throws SQLException { Configuration configuration = new Configuration(); + configuration.setProperty("wayang.hsqldb.cpu.mhz", "2700"); + configuration.setProperty("wayang.hsqldb.cores", "1"); + configuration.setProperty( + "wayang.hsqldb.tablesource.load", + "{\"in\":0,\"out\":1,\"cpu\":\"${1}\",\"ram\":\"0\",\"p\":1.0}" + ); + configuration.setProperty( + "wayang.hsqldb.tablesink.load", + "{\"in\":1,\"out\":0,\"cpu\":\"${1}\",\"ram\":\"0\",\"p\":1.0}" + ); HsqldbPlatform hsqldbPlatform = new HsqldbPlatform(); // Create source table with data @@ -89,10 +105,30 @@ void testOverwriteModeCreatesNewTable() throws SQLException { when(sqlStage.getStartTasks()).thenReturn(Collections.singleton(tableSourceTask)); when(sqlStage.getTerminalTasks()).thenReturn(Collections.singleton(sinkTask)); + when(sqlStage.getAllTasks()).thenReturn(new HashSet<>(Arrays.asList(tableSourceTask, sinkTask))); // Execute JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); - executor.execute(sqlStage, new DefaultOptimizationContext(job), job.getCrossPlatformExecutor()); + DefaultOptimizationContext optimizationContext = new DefaultOptimizationContext(job); + optimizationContext.addOneTimeOperator(tableSource); + optimizationContext.addOneTimeOperator(sinkOp); + executor.execute(sqlStage, optimizationContext, job.getCrossPlatformExecutor()); + + assertEquals(1, job.getCrossPlatformExecutor().getPartialExecutions().size()); + PartialExecution partialExecution = + job.getCrossPlatformExecutor().getPartialExecutions().iterator().next(); + Set estimatorKeys = partialExecution.getAtomicExecutionGroups().stream() + .flatMap(group -> group.getAtomicExecutions().stream()) + .map(AtomicExecution::getLoadProfileEstimator) + .map(estimator -> estimator.getConfigurationKey()) + .collect(Collectors.toSet()); + assertEquals( + new HashSet<>(Arrays.asList( + "wayang.hsqldb.tablesource.load", + "wayang.hsqldb.tablesink.load" + )), + estimatorKeys + ); // Verify table was created and contains all 3 rows try (Connection conn = hsqldbPlatform.createDatabaseDescriptor(configuration).createJdbcConnection()) { @@ -252,4 +288,4 @@ void testAppendClauseGeneration() { sinkOp.setMode("append"); assertEquals("INSERT INTO my_table", sinkOp.createSqlClause(null, null)); } -} \ No newline at end of file +} diff --git a/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizerApp.java b/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizerApp.java index 154819b5f..22c016b03 100644 --- a/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizerApp.java +++ b/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizerApp.java @@ -109,6 +109,7 @@ public GeneticOptimizerApp(Configuration configuration) { Spark.platform(); Sqlite3.platform(); Postgres.platform(); + initializeOptionalPlatform("org.apache.wayang.trino.Trino"); // Load the ExecutionLog. double samplingFactor = this.configuration.getDoubleProperty("wayang.profiler.ga.sampling", 1d); @@ -201,6 +202,20 @@ public GeneticOptimizerApp(Configuration configuration) { ); } + /** + * Initializes a platform integration when it is available on the runtime + * classpath without making it a mandatory profiler dependency. + */ + private static void initializeOptionalPlatform(String platformFacadeClassName) { + try { + Class.forName(platformFacadeClassName).getMethod("platform").invoke(null); + } catch (ClassNotFoundException e) { + logger.debug("Optional platform {} is not on the classpath.", platformFacadeClassName); + } catch (ReflectiveOperationException e) { + throw new WayangException("Could not initialize optional platform " + platformFacadeClassName, e); + } + } + /** * Check if all {@link CardinalityEstimate}s for the {@link PartialExecution} are sufficiently confident. * From 668a7ffb924fbdb4481233fc07dd6f170d9036b8 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Sun, 5 Jul 2026 23:40:29 +0800 Subject: [PATCH 13/18] Disable JDBC profiling unless logging is enabled --- conf/wayang-defaults.properties | 2 +- .../resources/wayang-core-defaults.properties | 2 +- wayang-docs/src/main/resources/index.md | 2 +- .../wayang/jdbc/execution/JdbcExecutor.java | 18 ++++++++++++------ .../execution/JdbcTableSinkExecutorTest.java | 4 ++++ 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/conf/wayang-defaults.properties b/conf/wayang-defaults.properties index 736b73ebe..ff2dcf33a 100644 --- a/conf/wayang-defaults.properties +++ b/conf/wayang-defaults.properties @@ -16,7 +16,7 @@ # # Configure statistics collection. -wayang.core.log.enabled = true +wayang.core.log.enabled = false wayang.core.explain.enabled = false wayang.core.explain.directrory = ~/.wayang/ diff --git a/wayang-commons/wayang-core/src/main/resources/wayang-core-defaults.properties b/wayang-commons/wayang-core/src/main/resources/wayang-core-defaults.properties index 6dd9d1d7a..d0372ed67 100644 --- a/wayang-commons/wayang-core/src/main/resources/wayang-core-defaults.properties +++ b/wayang-commons/wayang-core/src/main/resources/wayang-core-defaults.properties @@ -26,7 +26,7 @@ wayang.core.optimizer.enumeration.invertconcatenations = false wayang.core.optimizer.enumeration.branchesfirst = false # Configure statistics collection. -wayang.core.log.enabled = true +wayang.core.log.enabled = false # wayang.core.log.cardinalities = ~/.wayang/cardinalities.json # wayang.core.log.executions = ~/.wayang/executions.json wayang.core.explain.enabled = false diff --git a/wayang-docs/src/main/resources/index.md b/wayang-docs/src/main/resources/index.md index ab0217dbe..2d16eb1ba 100644 --- a/wayang-docs/src/main/resources/index.md +++ b/wayang-docs/src/main/resources/index.md @@ -106,7 +106,7 @@ $ java -Dwayang.configuration=url://to/my/wayang.properties ... Essential configuration settings: * General settings - * `wayang.core.log.enabled (= true)`: whether to log execution statistics to allow learning better cardinality and cost estimators for the optimizer + * `wayang.core.log.enabled (= false)`: whether to log execution statistics to allow learning better cardinality and cost estimators for the optimizer * `wayang.core.log.executions (= ~/.wayang/executions.json)` where to log execution times of operator groups * `wayang.core.log.cardinalities (= ~/.wayang/cardinalities.json)` where to log cardinality measurements * `wayang.core.optimizer.instrumentation (= org.apache.wayang.core.profiling.OutboundInstrumentationStrategy)`: where to measure cardinalities in Wayang plans; other options are `org.apache.wayang.core.profiling.NoInstrumentationStrategy` and `org.apache.wayang.core.profiling.FullInstrumentationStrategy` diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java index e6b00fe69..a2f1d123a 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java @@ -428,12 +428,14 @@ public void execute(final ExecutionStage stage, final OptimizationContext optimi if (termTask.getOperator() instanceof JdbcTableSinkOperator) { final long executionDuration = JdbcExecutor.executeSinkStage(stage, optimizationContext, this); - final PartialExecution partialExecution = this.createPartialExecution( - this.createExecutionLineageNodes(stage, optimizationContext), - executionDuration - ); - if (partialExecution != null) { - executionState.add(partialExecution); + if (this.isProfilingEnabled()) { + final PartialExecution partialExecution = this.createPartialExecution( + this.createExecutionLineageNodes(stage, optimizationContext), + executionDuration + ); + if (partialExecution != null) { + executionState.add(partialExecution); + } } } else { // If it is normal stage: compose SQL and store in channel for downstream @@ -447,6 +449,10 @@ public void execute(final ExecutionStage stage, final OptimizationContext optimi } } + private boolean isProfilingEnabled() { + return this.getConfiguration().getBooleanProperty("wayang.core.log.enabled", false); + } + @Override public void dispose() { try { diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java index 0b11fab1b..483bd4f81 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java @@ -57,6 +57,7 @@ class JdbcTableSinkExecutorTest { @Test void testOverwriteModeCreatesNewTable() throws SQLException { Configuration configuration = new Configuration(); + configuration.setProperty("wayang.core.log.enabled", "true"); configuration.setProperty("wayang.hsqldb.cpu.mhz", "2700"); configuration.setProperty("wayang.hsqldb.cores", "1"); configuration.setProperty( @@ -194,6 +195,7 @@ void testOverwriteModeReplacesExistingTable() throws SQLException { JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); executor.execute(sqlStage, new DefaultOptimizationContext(job), job.getCrossPlatformExecutor()); + assertEquals(0, job.getCrossPlatformExecutor().getPartialExecutions().size()); // Verify target was replaced. Old data should be gone, new schema and data present try (Connection conn = hsqldbPlatform.createDatabaseDescriptor(configuration).createJdbcConnection()) { @@ -212,6 +214,7 @@ void testOverwriteModeReplacesExistingTable() throws SQLException { @Test void testAppendModeInsertsIntoExistingTable() throws SQLException { Configuration configuration = new Configuration(); + configuration.setProperty("wayang.core.log.enabled", "false"); HsqldbPlatform hsqldbPlatform = new HsqldbPlatform(); //Create source and target table. Target has existing data. @@ -253,6 +256,7 @@ void testAppendModeInsertsIntoExistingTable() throws SQLException { JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); executor.execute(sqlStage, new DefaultOptimizationContext(job), job.getCrossPlatformExecutor()); + assertEquals(0, job.getCrossPlatformExecutor().getPartialExecutions().size()); // Verify existing data remains and new data is appended try (Connection conn = hsqldbPlatform.createDatabaseDescriptor(configuration).createJdbcConnection()) { From 677dd150e69a1185772d945ac03a93c95fb4c0be Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Sun, 5 Jul 2026 23:40:43 +0800 Subject: [PATCH 14/18] Add Trino cost profiling pilot --- .../apache/wayang/trino/TrinoCostPilotIT.java | 515 ++++++++++++++++++ 1 file changed, 515 insertions(+) create mode 100644 wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoCostPilotIT.java diff --git a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoCostPilotIT.java b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoCostPilotIT.java new file mode 100644 index 000000000..bb7fec18a --- /dev/null +++ b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoCostPilotIT.java @@ -0,0 +1,515 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + */ + +package org.apache.wayang.trino; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.data.Tuple2; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.basic.types.RecordType; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.FunctionDescriptor; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.core.types.DataUnitType; +import org.apache.wayang.trino.operators.TrinoProjectionOperator; +import org.apache.wayang.trino.operators.TrinoTableSource; +import org.apache.wayang.trino.platform.TrinoPlatform; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; + +import java.io.BufferedWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Small Trino cost-profiling pilot. + */ +class TrinoCostPilotIT { + + private static final String HOST = System.getenv().getOrDefault("TRINO_HOST", "localhost"); + private static final int PORT = Integer.parseInt(System.getenv().getOrDefault("TRINO_PORT", "8080")); + private static final String USER = System.getenv().getOrDefault("TRINO_USER", "admin"); + private static final String JDBC_URL = String.format("jdbc:trino://%s:%d", HOST, PORT); + + private static final String SCHEMA = "iceberg.wayang_profile"; + private static final String CUSTOMERS_1K = SCHEMA + ".customers_1k"; + private static final int[] ROW_COUNTS = {1000, 10000, 100000}; + private static final String[] COLUMNS = {"order_id", "customer_id", "region", "amount", "bucket"}; + private static final String[] JOIN_COLUMNS = { + "order_id", "customer_id", "region", "amount", "bucket", "cust_id", "tier" + }; + private static final String JOIN_FLATTEN_NAME = "Trino profile join flatten"; + private static final Path OUTPUT_DIR = Paths.get(System.getProperty( + "trino.profile.outputDir", + "C:/Users/jizhi/Desktop/2026GSoC/Trino Profiling/week6" + )); + private static final Path EXECUTIONS_PATH = OUTPUT_DIR.resolve("executions.json"); + private static final Path CARDINALITIES_PATH = OUTPUT_DIR.resolve("cardinalities.json"); + private static final Path MANIFEST_PATH = OUTPUT_DIR.resolve("manifest.csv"); + private static final List PLAN_IDS = Arrays.asList( + System.getProperty("trino.profile.plans", "S01,S02").split(",") + ); + private static final boolean RESET_OUTPUT = Boolean.parseBoolean( + System.getProperty("trino.profile.reset", "true") + ); + + @Test + void runPilot() throws Exception { + Assumptions.assumeTrue(isTrinoAvailable(), "Trino not reachable"); + Files.createDirectories(OUTPUT_DIR); + initializeOutputFiles(); + + prepareTables(); + + for (int rowCount : ROW_COUNTS) { + for (String planId : PLAN_IDS) { + String normalizedPlanId = planId.trim(); + runPlan( + normalizedPlanId, + getOperatorChain(normalizedPlanId), + rowCount, + getExpectedRows(normalizedPlanId, rowCount) + ); + } + } + } + + private void runPlan(String planId, String operatorChain, int rowCount, long expectedRows) throws Exception { + for (int repetition = 0; repetition < 4; repetition++) { + boolean isWarmup = repetition == 0; + String runId = String.format("%s_%s_r%02d", planId, formatRows(rowCount), repetition); + String sourceTable = SCHEMA + ".orders_" + formatRows(rowCount); + String sinkTable = SCHEMA + ".sink_" + runId.toLowerCase(); + + dropTable(sinkTable); + WayangPlan plan = createPlan(planId, sourceTable, sinkTable); + wayangContext().execute(runId, plan); + + long actualRows = queryLong("SELECT count(*) FROM " + sinkTable); + assertEquals(expectedRows, actualRows, runId + " row count"); + appendManifest(runId, planId, operatorChain, rowCount, expectedRows, repetition, isWarmup, sinkTable, "ok", ""); + dropTable(sinkTable); + } + } + + private WayangPlan createPlan(String planId, String sourceTable, String sinkTable) { + TrinoTableSource source = new TrinoTableSource(sourceTable, COLUMNS); + + if ("S01".equals(planId)) { + TableSink sink = new TableSink<>(new Properties(), "overwrite", sinkTable, COLUMNS); + source.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S02".equals(planId)) { + TableSink sink = new TableSink<>(new Properties(), "overwrite", sinkTable, COLUMNS); + FilterOperator filter = createAmerFilter(); + source.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S03".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "order_id", "amount"); + MapOperator projection = createOrderAmountProjection(); + source.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S04".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "order_id", "amount"); + FilterOperator filter = createAmerFilter(); + MapOperator projection = createOrderAmountProjection(); + source.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S05".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "total_amount"); + GlobalReduceOperator reduce = new GlobalReduceOperator<>( + new ReduceDescriptor<>((left, right) -> left, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + source.connectTo(0, reduce, 0); + reduce.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S06".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "bucket", "total_amount"); + ReduceByOperator reduceBy = new ReduceByOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(4)), + Record.class, + Record.class + ).withSqlImplementation("bucket", "bucket"), + new ReduceDescriptor<>((left, right) -> left, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + source.connectTo(0, reduceBy, 0); + reduceBy.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S07".equals(planId)) { + TableSink sink = new TableSink<>(new Properties(), "overwrite", sinkTable, COLUMNS); + SortOperator sort = new SortOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(3)), + Record.class, + Record.class + ).withSqlImplementation("amount", "ASC"), + DataSetType.createDefault(Record.class)); + source.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S08".equals(planId)) { + TrinoTableSource customers = new TrinoTableSource(CUSTOMERS_1K, "cust_id", "tier"); + JoinOperator join = new JoinOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(1)), + Record.class, + Record.class + ).withSqlImplementation(sourceTable, "customer_id"), + new TransformationDescriptor<>( + record -> new Record(record.getField(0)), + Record.class, + Record.class + ).withSqlImplementation(CUSTOMERS_1K, "cust_id")); + MapOperator, Record> flatten = createJoinFlattenOperator(); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, JOIN_COLUMNS); + source.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + throw new IllegalArgumentException("Unsupported pilot plan: " + planId); + } + + private static FilterOperator createAmerFilter() { + return new FilterOperator<>( + new PredicateDescriptor<>( + (Record record) -> "AMER".equals(record.getField(2)), + Record.class + ).withSqlImplementation("region = 'AMER'") + ); + } + + private static MapOperator createOrderAmountProjection() { + return new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType(COLUMNS), + "order_id", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + } + + private static MapOperator, Record> createJoinFlattenOperator() { + MapOperator, Record> operator = new MapOperator<>( + new TransformationDescriptor<>( + new JoinFlattenFunction(), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class)), + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class)); + operator.setName(JOIN_FLATTEN_NAME); + return operator; + } + + private WayangContext wayangContext() { + Configuration configuration = new Configuration(); + configuration.setProperty("wayang.trino.jdbc.url", JDBC_URL); + configuration.setProperty("wayang.trino.jdbc.user", USER); + configuration.setProperty("wayang.trino.jdbc.password", ""); + configuration.setProperty("wayang.core.log.enabled", "true"); + configuration.setProperty("wayang.core.explain.enabled", "false"); + configuration.setProperty("wayang.core.log.executions", EXECUTIONS_PATH.toString().replace('\\', '/')); + configuration.setProperty("wayang.core.log.cardinalities", CARDINALITIES_PATH.toString().replace('\\', '/')); + configuration.getMappingProvider().addAllToWhitelist( + Collections.singleton(new JoinFlattenMapping())); + return new WayangContext(configuration).withPlugin(Trino.plugin()); + } + + private void prepareTables() throws Exception { + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + statement.execute("CREATE SCHEMA IF NOT EXISTS " + SCHEMA); + for (int rowCount : ROW_COUNTS) { + String table = SCHEMA + ".orders_" + formatRows(rowCount); + statement.execute("DROP TABLE IF EXISTS " + table); + statement.execute("CREATE TABLE " + table + " WITH (format = 'PARQUET') AS " + + "SELECT " + + "CAST(n AS BIGINT) AS order_id, " + + "CAST(n % 1000 AS BIGINT) AS customer_id, " + + "CASE WHEN n % 2 = 0 THEN 'AMER' ELSE 'EMEA' END AS region, " + + "CAST(n % 10000 AS DOUBLE) AS amount, " + + "CAST(n % 100 AS BIGINT) AS bucket " + + "FROM " + createRowsSql(rowCount)); + assertEquals(rowCount, queryLong("SELECT count(*) FROM " + table), table + " row count"); + assertEquals(rowCount / 2, queryLong("SELECT count(*) FROM " + table + " WHERE region = 'AMER'"), + table + " AMER row count"); + } + statement.execute("DROP TABLE IF EXISTS " + CUSTOMERS_1K); + statement.execute("CREATE TABLE " + CUSTOMERS_1K + " WITH (format = 'PARQUET') AS " + + "SELECT " + + "CAST(n - 1 AS BIGINT) AS cust_id, " + + "CASE WHEN n % 2 = 0 THEN 'GOLD' ELSE 'SILVER' END AS tier " + + "FROM UNNEST(sequence(1, 1000)) AS t(n)"); + assertEquals(1000, queryLong("SELECT count(*) FROM " + CUSTOMERS_1K), CUSTOMERS_1K + " row count"); + } + } + + private static String createRowsSql(int rowCount) { + if (rowCount <= 10000) { + return "UNNEST(sequence(1, " + rowCount + ")) AS t(n)"; + } + + int chunks = (rowCount + 9999) / 10000; + return "(" + + "SELECT chunk * 10000 + offset AS n " + + "FROM UNNEST(sequence(0, " + (chunks - 1) + ")) AS c(chunk) " + + "CROSS JOIN UNNEST(sequence(1, 10000)) AS o(offset) " + + "WHERE chunk * 10000 + offset <= " + rowCount + + ") AS t"; + } + + private static String formatRows(int rowCount) { + if (rowCount % 1000 == 0) { + return (rowCount / 1000) + "k"; + } + return String.valueOf(rowCount); + } + + private void initializeOutputFiles() throws Exception { + if (RESET_OUTPUT) { + Files.deleteIfExists(EXECUTIONS_PATH); + Files.deleteIfExists(CARDINALITIES_PATH); + writeManifestHeader(); + } else if (!Files.exists(MANIFEST_PATH)) { + writeManifestHeader(); + } + } + + private void writeManifestHeader() throws Exception { + try (BufferedWriter writer = Files.newBufferedWriter(MANIFEST_PATH, StandardCharsets.UTF_8)) { + writer.write("run_id,plan_id,operator_chain,input_rows_left,input_rows_right,expected_output_rows," + + "selectivity,repetition,is_warmup,sink_table,status,notes"); + writer.newLine(); + } + } + + private void appendManifest( + String runId, + String planId, + String operatorChain, + int inputRows, + long expectedOutputRows, + int repetition, + boolean isWarmup, + String sinkTable, + String status, + String notes) throws Exception { + try (BufferedWriter writer = Files.newBufferedWriter( + MANIFEST_PATH, + StandardCharsets.UTF_8, + java.nio.file.StandardOpenOption.APPEND)) { + writer.write(String.join(",", + runId, + planId, + operatorChain, + String.valueOf(inputRows), + "", + String.valueOf(expectedOutputRows), + hasFilter(planId) ? "0.5" : "1.0", + String.valueOf(repetition), + String.valueOf(isWarmup), + sinkTable, + status, + notes)); + writer.newLine(); + } + } + + private static String getOperatorChain(String planId) { + switch (planId) { + case "S01": + return "TableSource->TableSink"; + case "S02": + return "TableSource->Filter(50%)->TableSink"; + case "S03": + return "TableSource->Projection->TableSink"; + case "S04": + return "TableSource->Filter(50%)->Projection->TableSink"; + case "S05": + return "TableSource->GlobalReduce->TableSink"; + case "S06": + return "TableSource->ReduceBy(bucket)->TableSink"; + case "S07": + return "TableSource->Sort(amount)->TableSink"; + case "S08": + return "Orders->Join(Customers 1k)->Projection->TableSink"; + default: + throw new IllegalArgumentException("Unsupported pilot plan: " + planId); + } + } + + private static long getExpectedRows(String planId, int rowCount) { + if ("S05".equals(planId)) { + return 1; + } + if ("S06".equals(planId)) { + return 100; + } + return hasFilter(planId) ? rowCount / 2 : rowCount; + } + + private static boolean hasFilter(String planId) { + return "S02".equals(planId) || "S04".equals(planId); + } + + private long queryLong(String sql) throws Exception { + try (Connection connection = jdbc(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getLong(1); + } + } + + private void dropTable(String table) throws Exception { + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS " + table); + } + } + + private static boolean isTrinoAvailable() { + try (Connection connection = jdbc(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT 1")) { + return resultSet.next(); + } catch (Exception e) { + return false; + } + } + + private static Connection jdbc() throws Exception { + return DriverManager.getConnection(JDBC_URL, USER, ""); + } + + private static Record flattenJoinResult(Object joinResult) { + if (joinResult instanceof Record) { + return (Record) joinResult; + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record( + left.getField(0), + left.getField(1), + left.getField(2), + left.getField(3), + left.getField(4), + right.getField(0), + right.getField(1)); + } + + private static final class JoinFlattenFunction implements + FunctionDescriptor.SerializableFunction, Record> { + + @Override + public Record apply(Tuple2 tuple) { + return flattenJoinResult(tuple); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static final class JoinFlattenMapping implements Mapping { + + @Override + public java.util.Collection getTransformations() { + OperatorPattern pattern = new OperatorPattern( + "joinFlatten", + new MapOperator(null, DataSetType.none(), DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(operator -> JOIN_FLATTEN_NAME.equals(((MapOperator) operator).getName())); + + ReplacementSubplanFactory factory = new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> createTrinoProjection().at(epoch)); + + return Collections.singleton(new PlanTransformation( + SubplanPattern.createSingleton(pattern), + factory, + TrinoPlatform.getInstance())); + } + + private static TrinoProjectionOperator createTrinoProjection() { + ProjectionDescriptor, Record> descriptor = new ProjectionDescriptor<>( + new JoinFlattenFunction(), + Arrays.asList(JOIN_COLUMNS), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class)); + MapOperator, Record> projection = new MapOperator<>( + descriptor, + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class)); + projection.setName(JOIN_FLATTEN_NAME); + return new TrinoProjectionOperator((MapOperator) (MapOperator) projection); + } + } +} From 78badeea1989adb8b4bf9d330017dc9dc723653e Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Sun, 19 Jul 2026 02:00:27 +0800 Subject: [PATCH 15/18] Add join-heavy cost profiling cases --- .../apache/wayang/trino/TrinoCostPilotIT.java | 347 ++++++++++++++++-- 1 file changed, 326 insertions(+), 21 deletions(-) diff --git a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoCostPilotIT.java b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoCostPilotIT.java index bb7fec18a..8ac6be631 100644 --- a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoCostPilotIT.java +++ b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoCostPilotIT.java @@ -78,12 +78,19 @@ class TrinoCostPilotIT { private static final String SCHEMA = "iceberg.wayang_profile"; private static final String CUSTOMERS_1K = SCHEMA + ".customers_1k"; - private static final int[] ROW_COUNTS = {1000, 10000, 100000}; + private static final int[] ROW_COUNTS = parseIntList(System.getProperty( + "trino.profile.rowCounts", + "10000,50000,100000,250000" + )); private static final String[] COLUMNS = {"order_id", "customer_id", "region", "amount", "bucket"}; private static final String[] JOIN_COLUMNS = { "order_id", "customer_id", "region", "amount", "bucket", "cust_id", "tier" }; + private static final String[] JOIN_ORDER_TIER_AMOUNT_COLUMNS = {"order_id", "tier", "amount"}; + private static final String[] JOIN_TIER_AMOUNT_COLUMNS = {"tier", "amount"}; private static final String JOIN_FLATTEN_NAME = "Trino profile join flatten"; + private static final String JOIN_ORDER_TIER_AMOUNT_FLATTEN_NAME = "Trino profile join order tier amount flatten"; + private static final String JOIN_TIER_AMOUNT_FLATTEN_NAME = "Trino profile join tier amount flatten"; private static final Path OUTPUT_DIR = Paths.get(System.getProperty( "trino.profile.outputDir", "C:/Users/jizhi/Desktop/2026GSoC/Trino Profiling/week6" @@ -92,7 +99,13 @@ class TrinoCostPilotIT { private static final Path CARDINALITIES_PATH = OUTPUT_DIR.resolve("cardinalities.json"); private static final Path MANIFEST_PATH = OUTPUT_DIR.resolve("manifest.csv"); private static final List PLAN_IDS = Arrays.asList( - System.getProperty("trino.profile.plans", "S01,S02").split(",") + System.getProperty( + "trino.profile.plans", + "S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15,S16" + ).split(",") + ); + private static final int REPETITIONS = Integer.parseInt( + System.getProperty("trino.profile.repetitions", "6") ); private static final boolean RESET_OUTPUT = Boolean.parseBoolean( System.getProperty("trino.profile.reset", "true") @@ -120,7 +133,7 @@ void runPilot() throws Exception { } private void runPlan(String planId, String operatorChain, int rowCount, long expectedRows) throws Exception { - for (int repetition = 0; repetition < 4; repetition++) { + for (int repetition = 0; repetition < REPETITIONS; repetition++) { boolean isWarmup = repetition == 0; String runId = String.format("%s_%s_r%02d", planId, formatRows(rowCount), repetition); String sourceTable = SCHEMA + ".orders_" + formatRows(rowCount); @@ -205,13 +218,7 @@ record -> new Record(record.getField(4)), if ("S07".equals(planId)) { TableSink sink = new TableSink<>(new Properties(), "overwrite", sinkTable, COLUMNS); - SortOperator sort = new SortOperator<>( - new TransformationDescriptor<>( - record -> new Record(record.getField(3)), - Record.class, - Record.class - ).withSqlImplementation("amount", "ASC"), - DataSetType.createDefault(Record.class)); + SortOperator sort = createAmountSortOperator(3); source.connectTo(0, sort, 0); sort.connectTo(0, sink, 0); return new WayangPlan(sink); @@ -240,9 +247,165 @@ record -> new Record(record.getField(0)), return new WayangPlan(sink); } + if ("S09".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "total_amount"); + FilterOperator filter = createAmerFilter(); + GlobalReduceOperator reduce = createGlobalAmountReduceOperator(); + source.connectTo(0, filter, 0); + filter.connectTo(0, reduce, 0); + reduce.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S10".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "bucket", "total_amount"); + FilterOperator filter = createAmerFilter(); + ReduceByOperator reduceBy = createBucketReduceByOperator(); + source.connectTo(0, filter, 0); + filter.connectTo(0, reduceBy, 0); + reduceBy.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S11".equals(planId)) { + TableSink sink = new TableSink<>(new Properties(), "overwrite", sinkTable, COLUMNS); + FilterOperator filter = createAmerFilter(); + SortOperator sort = createAmountSortOperator(3); + source.connectTo(0, filter, 0); + filter.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S12".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "order_id", "amount"); + MapOperator projection = createOrderAmountProjection(); + SortOperator sort = createAmountSortOperator(1); + source.connectTo(0, projection, 0); + projection.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S13".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "order_id", "amount"); + FilterOperator filter = createAmerFilter(); + MapOperator projection = createOrderAmountProjection(); + SortOperator sort = createAmountSortOperator(1); + source.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S14".equals(planId)) { + TrinoTableSource customers = new TrinoTableSource(CUSTOMERS_1K, "cust_id", "tier"); + FilterOperator filter = createAmerFilter(); + JoinOperator join = createCustomerJoinOperator(sourceTable); + MapOperator, Record> flatten = createJoinFlattenOperator(); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, JOIN_COLUMNS); + source.connectTo(0, filter, 0); + filter.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S15".equals(planId)) { + TrinoTableSource customers = new TrinoTableSource(CUSTOMERS_1K, "cust_id", "tier"); + JoinOperator join = createCustomerJoinOperator(sourceTable); + MapOperator, Record> flatten = createJoinOrderTierAmountFlattenOperator(); + SortOperator sort = createAmountSortOperator(2); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, JOIN_ORDER_TIER_AMOUNT_COLUMNS); + source.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S16".equals(planId)) { + TrinoTableSource customers = new TrinoTableSource(CUSTOMERS_1K, "cust_id", "tier"); + JoinOperator join = createCustomerJoinOperator(sourceTable); + MapOperator, Record> flatten = createJoinTierAmountFlattenOperator(); + ReduceByOperator reduceBy = createTierReduceByOperator(); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "tier", "total_amount"); + source.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, reduceBy, 0); + reduceBy.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + throw new IllegalArgumentException("Unsupported pilot plan: " + planId); } + private static GlobalReduceOperator createGlobalAmountReduceOperator() { + return new GlobalReduceOperator<>( + new ReduceDescriptor<>((left, right) -> left, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + } + + private static ReduceByOperator createBucketReduceByOperator() { + return new ReduceByOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(4)), + Record.class, + Record.class + ).withSqlImplementation("bucket", "bucket"), + new ReduceDescriptor<>((left, right) -> left, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + } + + private static ReduceByOperator createTierReduceByOperator() { + return new ReduceByOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(0)), + Record.class, + Record.class + ).withSqlImplementation("tier", "tier"), + new ReduceDescriptor<>((left, right) -> left, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + } + + private static SortOperator createAmountSortOperator(int amountFieldIndex) { + return new SortOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(amountFieldIndex)), + Record.class, + Record.class + ).withSqlImplementation("amount", "ASC"), + DataSetType.createDefault(Record.class)); + } + + private static JoinOperator createCustomerJoinOperator(String sourceTable) { + return new JoinOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(1)), + Record.class, + Record.class + ).withSqlImplementation(sourceTable, "customer_id"), + new TransformationDescriptor<>( + record -> new Record(record.getField(0)), + Record.class, + Record.class + ).withSqlImplementation(CUSTOMERS_1K, "cust_id")); + } + private static FilterOperator createAmerFilter() { return new FilterOperator<>( new PredicateDescriptor<>( @@ -261,15 +424,47 @@ private static MapOperator createOrderAmountProjection() { DataSetType.createDefault(Record.class)); } + private static MapOperator createOrderTierAmountProjection() { + return new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType(JOIN_COLUMNS), + "order_id", "tier", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + } + + private static MapOperator createTierAmountProjection() { + return new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType(JOIN_COLUMNS), + "tier", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + } + private static MapOperator, Record> createJoinFlattenOperator() { + return createJoinFlattenOperator(new JoinFlattenFunction(), JOIN_FLATTEN_NAME); + } + + private static MapOperator, Record> createJoinOrderTierAmountFlattenOperator() { + return createJoinFlattenOperator(new JoinOrderTierAmountFlattenFunction(), JOIN_ORDER_TIER_AMOUNT_FLATTEN_NAME); + } + + private static MapOperator, Record> createJoinTierAmountFlattenOperator() { + return createJoinFlattenOperator(new JoinTierAmountFlattenFunction(), JOIN_TIER_AMOUNT_FLATTEN_NAME); + } + + private static MapOperator, Record> createJoinFlattenOperator( + FunctionDescriptor.SerializableFunction, Record> function, + String name) { MapOperator, Record> operator = new MapOperator<>( new TransformationDescriptor<>( - new JoinFlattenFunction(), + function, DataUnitType.createBasicUnchecked(Tuple2.class), DataUnitType.createBasic(Record.class)), DataSetType.createDefaultUnchecked(Tuple2.class), DataSetType.createDefault(Record.class)); - operator.setName(JOIN_FLATTEN_NAME); + operator.setName(name); return operator; } @@ -374,7 +569,7 @@ private void appendManifest( planId, operatorChain, String.valueOf(inputRows), - "", + hasJoin(planId) ? "1000" : "", String.valueOf(expectedOutputRows), hasFilter(planId) ? "0.5" : "1.0", String.valueOf(repetition), @@ -404,23 +599,66 @@ private static String getOperatorChain(String planId) { return "TableSource->Sort(amount)->TableSink"; case "S08": return "Orders->Join(Customers 1k)->Projection->TableSink"; + case "S09": + return "TableSource->Filter(50%)->GlobalReduce->TableSink"; + case "S10": + return "TableSource->Filter(50%)->ReduceBy(bucket)->TableSink"; + case "S11": + return "TableSource->Filter(50%)->Sort(amount)->TableSink"; + case "S12": + return "TableSource->Projection(order_id,amount)->Sort(amount)->TableSink"; + case "S13": + return "TableSource->Filter(50%)->Projection(order_id,amount)->Sort(amount)->TableSink"; + case "S14": + return "Orders->Filter(50%)->Join(Customers 1k)->Projection->TableSink"; + case "S15": + return "Orders->Join(Customers 1k)->Projection(order_id,tier,amount)->Sort(amount)->TableSink"; + case "S16": + return "Orders->Join(Customers 1k)->Projection(tier,amount)->ReduceBy(tier)->TableSink"; default: throw new IllegalArgumentException("Unsupported pilot plan: " + planId); } } private static long getExpectedRows(String planId, int rowCount) { - if ("S05".equals(planId)) { + if ("S05".equals(planId) || "S09".equals(planId)) { return 1; } if ("S06".equals(planId)) { return 100; } + if ("S10".equals(planId)) { + return 50; + } + if ("S16".equals(planId)) { + return 2; + } return hasFilter(planId) ? rowCount / 2 : rowCount; } private static boolean hasFilter(String planId) { - return "S02".equals(planId) || "S04".equals(planId); + return "S02".equals(planId) + || "S04".equals(planId) + || "S09".equals(planId) + || "S10".equals(planId) + || "S11".equals(planId) + || "S13".equals(planId) + || "S14".equals(planId); + } + + private static boolean hasJoin(String planId) { + return "S08".equals(planId) + || "S14".equals(planId) + || "S15".equals(planId) + || "S16".equals(planId); + } + + private static int[] parseIntList(String value) { + return Arrays.stream(value.split(",")) + .map(String::trim) + .filter(token -> !token.isEmpty()) + .mapToInt(Integer::parseInt) + .toArray(); } private long queryLong(String sql) throws Exception { @@ -469,6 +707,28 @@ private static Record flattenJoinResult(Object joinResult) { right.getField(1)); } + private static Record flattenJoinOrderTierAmountResult(Object joinResult) { + if (joinResult instanceof Record) { + Record record = (Record) joinResult; + return new Record(record.getField(0), record.getField(6), record.getField(3)); + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record(left.getField(0), right.getField(1), left.getField(3)); + } + + private static Record flattenJoinTierAmountResult(Object joinResult) { + if (joinResult instanceof Record) { + Record record = (Record) joinResult; + return new Record(record.getField(6), record.getField(3)); + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record(right.getField(1), left.getField(3)); + } + private static final class JoinFlattenFunction implements FunctionDescriptor.SerializableFunction, Record> { @@ -478,6 +738,24 @@ public Record apply(Tuple2 tuple) { } } + private static final class JoinOrderTierAmountFlattenFunction implements + FunctionDescriptor.SerializableFunction, Record> { + + @Override + public Record apply(Tuple2 tuple) { + return flattenJoinOrderTierAmountResult(tuple); + } + } + + private static final class JoinTierAmountFlattenFunction implements + FunctionDescriptor.SerializableFunction, Record> { + + @Override + public Record apply(Tuple2 tuple) { + return flattenJoinTierAmountResult(tuple); + } + } + @SuppressWarnings({"rawtypes", "unchecked"}) private static final class JoinFlattenMapping implements Mapping { @@ -487,10 +765,10 @@ public java.util.Collection getTransformations() { "joinFlatten", new MapOperator(null, DataSetType.none(), DataSetType.createDefault(Record.class)), false) - .withAdditionalTest(operator -> JOIN_FLATTEN_NAME.equals(((MapOperator) operator).getName())); + .withAdditionalTest(operator -> isJoinFlattenName(((MapOperator) operator).getName())); ReplacementSubplanFactory factory = new ReplacementSubplanFactory.OfSingleOperators( - (matchedOperator, epoch) -> createTrinoProjection().at(epoch)); + (matchedOperator, epoch) -> createTrinoProjection(matchedOperator.getName()).at(epoch)); return Collections.singleton(new PlanTransformation( SubplanPattern.createSingleton(pattern), @@ -498,18 +776,45 @@ public java.util.Collection getTransformations() { TrinoPlatform.getInstance())); } - private static TrinoProjectionOperator createTrinoProjection() { + private static TrinoProjectionOperator createTrinoProjection(String operatorName) { ProjectionDescriptor, Record> descriptor = new ProjectionDescriptor<>( - new JoinFlattenFunction(), - Arrays.asList(JOIN_COLUMNS), + getJoinFlattenFunction(operatorName), + Arrays.asList(getJoinFlattenColumns(operatorName)), DataUnitType.createBasicUnchecked(Tuple2.class), DataUnitType.createBasic(Record.class)); MapOperator, Record> projection = new MapOperator<>( descriptor, DataSetType.createDefaultUnchecked(Tuple2.class), DataSetType.createDefault(Record.class)); - projection.setName(JOIN_FLATTEN_NAME); + projection.setName(operatorName); return new TrinoProjectionOperator((MapOperator) (MapOperator) projection); } + + private static boolean isJoinFlattenName(String operatorName) { + return JOIN_FLATTEN_NAME.equals(operatorName) + || JOIN_ORDER_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName) + || JOIN_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName); + } + + private static String[] getJoinFlattenColumns(String operatorName) { + if (JOIN_ORDER_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName)) { + return JOIN_ORDER_TIER_AMOUNT_COLUMNS; + } + if (JOIN_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName)) { + return JOIN_TIER_AMOUNT_COLUMNS; + } + return JOIN_COLUMNS; + } + + private static FunctionDescriptor.SerializableFunction, Record> getJoinFlattenFunction( + String operatorName) { + if (JOIN_ORDER_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName)) { + return new JoinOrderTierAmountFlattenFunction(); + } + if (JOIN_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName)) { + return new JoinTierAmountFlattenFunction(); + } + return new JoinFlattenFunction(); + } } } From 7932aa276d754ad2eab2444f79334e99ee402039 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Thu, 23 Jul 2026 01:38:39 +0800 Subject: [PATCH 16/18] Prepare Trino cost profiling docs --- guides/cost-profiling.md | 430 ++++++++++++++++++ trino-setup/README.md | 20 +- .../wayang-trino-defaults.properties | 118 +++-- .../apache/wayang/trino/TrinoCostPilotIT.java | 2 +- 4 files changed, 521 insertions(+), 49 deletions(-) create mode 100644 guides/cost-profiling.md diff --git a/guides/cost-profiling.md b/guides/cost-profiling.md new file mode 100644 index 000000000..25f8f9bd3 --- /dev/null +++ b/guides/cost-profiling.md @@ -0,0 +1,430 @@ + + +# Cost Profiling Guide + +This document explains why Apache Wayang needs platform-specific cost +profiling, how profiling data is collected, how the genetic optimizer learns +cost parameters, and how users can repeat the profiling workflow on their own +hardware. + +The examples below use Trino, but the same workflow also applies to other +JDBC-based platforms such as Presto and BigQuery. + +Version 2.0 uses S01 through S16 as the profiling workload, including the +join-heavy pipelines S14 through S16. It keeps the guide focused on data +collection and parameter learning, leaving follow-up quality checks out of +scope for now. + +## 1. Why Profiling Is Needed + +Wayang can map the same logical plan to different execution platforms, such as +Java, Spark, Trino, Presto, or BigQuery. For example, a user query may contain: + +```text +TableSource -> Filter -> Projection -> TableSink +``` + +The optimizer needs a cost model to decide whether these operators should stay +on a SQL platform or be moved to another platform. In this context, "cost" does +not mean cloud billing cost. It is the numerical value that Wayang uses to +compare alternative execution plans. + +With the default Trino configuration: + +```properties +wayang.trino.costs.fix = 0.0 +wayang.trino.costs.per-ms = 1.0 +``` + +the optimizer cost can be interpreted approximately as: + +```text +cost = estimated execution time in milliseconds +``` + +However, the real execution time depends on the user's machine, cluster size, +network, database configuration, and workload. Therefore, users should profile +their own environment when they need accurate cost parameters. + +## 2. Load Profile Formulas + +Each execution operator has a load profile. For example, a table source may use +a formula like: + +```properties +wayang.trino.tablesource.load = { + "type":"mathex", + "in":0, + "out":1, + "cpu":"((10)*(out0))+(800000)", + "ram":"0", + "disk":"0", + "net":"0", + "p":0.9 +} +``` + +This can be read as: + +```text +CPU load = alpha * number_of_rows + beta +``` + +where: + +- `out0` is the output cardinality. +- `alpha` is the per-row cost. +- `beta` is the fixed overhead, such as query planning, scheduling, and remote + execution startup. +- `p` is the confidence of the estimate. + +The profiling goal is to learn reasonable values for `alpha` and `beta` from +real execution records. + +Wayang can also define templates with unknown parameters: + +```properties +wayang.trino.tablesource.load.template = { + "type":"mathex", + "in":0, + "out":1, + "cpu":"?*out0 + ?", + "ram":"0", + "disk":"0", + "net":"0", + "p":0.9 +} +``` + +The genetic optimizer reads the templates and replaces the `?` placeholders with +learned values. + +## 3. Profiling Workflow + +The expected profiling workflow is: + +```text +Run Wayang jobs with different operators and input cardinalities + | + v +Record platform, operator lineage, cardinalities, and runtime + | + v + executions.json + | + v + GeneticOptimizerApp reads the execution records + | + v +Learn the unknown parameters in *.load.template + | + v +Write learned *.load formulas +``` + +Wayang stores measured executions as `PartialExecution` records. Each record +contains: + +- the measured execution time; +- the platform that executed the stage; +- one or more `ExecutionLineageNode` objects; +- the load profile estimator for each profiled operator; +- input and output cardinalities. + +The default execution log location is usually: + +```text +~/.wayang/executions.json +``` + +For controlled profiling experiments, it is better to write the log to a +dedicated experiment folder, for example: + +```text +C:\Users\\Desktop\Wayang Profiling\trino\week8\executions.json +``` + +## 4. Experiment Design + +Profiling should include both single-operator pipelines and combined pipelines. +Single-operator pipelines help isolate each operator. Combined pipelines help +the optimizer learn parameters from realistic SQL stages, where multiple +operators are executed together. + +Choose input cardinalities according to the machine or cluster being profiled. +The values below are only an example that can run on a laptop-sized local +setup: + +```text +10k, 50k, 100k, 250k +``` + +For a smaller machine, use fewer or smaller cardinalities. For a larger local +or remote platform, add larger cardinalities so the learned model reflects the +scale that users expect to run. + +Recommended repetitions: + +```text +1 warm-up run + 5 measured runs +``` + +Profiling pipelines: + +| Plan | Pipeline | +|------|----------| +| S01 | TableSource -> TableSink | +| S02 | TableSource -> Filter(50%) -> TableSink | +| S03 | TableSource -> Projection(order_id, amount) -> TableSink | +| S04 | TableSource -> Filter(50%) -> Projection(order_id, amount) -> TableSink | +| S05 | TableSource -> GlobalReduce(sum amount) -> TableSink | +| S06 | TableSource -> ReduceBy(bucket) -> TableSink | +| S07 | TableSource -> Sort(amount) -> TableSink | +| S08 | Orders -> Join(Customers 1k) -> Projection -> TableSink | +| S09 | TableSource -> Filter(50%) -> GlobalReduce -> TableSink | +| S10 | TableSource -> Filter(50%) -> ReduceBy -> TableSink | +| S11 | TableSource -> Filter(50%) -> Sort(amount) -> TableSink | +| S12 | TableSource -> Projection(order_id, amount) -> Sort(amount) -> TableSink | +| S13 | TableSource -> Filter(50%) -> Projection(order_id, amount) -> Sort(amount) -> TableSink | +| S14 | Orders -> Filter(50%) -> Join(Customers 1k) -> Projection -> TableSink | +| S15 | Orders -> Join(Customers 1k) -> Projection(order_id, tier, amount) -> Sort(amount) -> TableSink | +| S16 | Orders -> Join(Customers 1k) -> Projection(tier, amount) -> ReduceBy(tier) -> TableSink | + +S01 through S16 should be treated as one profiling workload, including the +join-heavy plans S14 through S16. For example, using 16 plans, 4 cardinalities, +and 6 repetitions produces: + +```text +16 * 4 * 6 = 384 Wayang executions +``` + +If users choose a different number of cardinalities or repetitions, the total +number of executions changes accordingly: + +```text +number_of_plans * number_of_cardinalities * repetitions +``` + +The reference parameters shipped in the platform defaults were learned from our +local Week 8 profiling runs over S01 through S13, with row counts +10k/50k/100k/250k and 1 warm-up plus 5 measured repetitions. S14 through S16 +were added to this guide to document the join-heavy pipelines that users should +include when they rerun profiling in their own environment. The shipped +parameters are intended as reasonable starting values for users who just want +to try Wayang; they are not universal parameters for every deployment. + +## 5. Benchmarking Rules + +To reduce measurement noise: + +1. Create test data before the measured run. Do not include fixture setup time + in operator duration. +2. Run at least one warm-up execution for each plan/cardinality pair. +3. Repeat each measured scenario multiple times. +4. Store every individual measurement instead of storing only averages. +5. Record exact input and output cardinalities. +6. Keep platform settings stable, including worker count, JVM settings, memory + limits, and connector configuration. +7. Record abnormal runs, such as failures caused by GC, cold cache, network + issues, or competing workloads. + +For distributed systems such as Trino, it is also important to define what the +model should predict: + +- If Wayang should predict user-visible runtime, fit wall-clock elapsed time. +- If the platform reports CPU time and the model uses CPU load, make sure the + conversion to Wayang cost is consistent with the resource model. +- Parameters learned on a local Docker setup should be treated as local + reference values, not universal defaults for every deployment. + +## 6. Running a Profiling Experiment + +The exact command depends on the platform module, test class, and property +prefix. + +| Platform | Setup guide | Maven module | Test class | Property prefix | Default output directory | +|----------|-------------|--------------|------------|-----------------|--------------------------| +| Trino | `trino-setup/README.md` | `wayang-platforms/wayang-trino` | `TrinoCostPilotIT` | `trino.profile.*` | `target/cost-profiling/trino` | +| Presto | `presto-setup/README.md` | `wayang-platforms/wayang-presto` | `PrestoCostPilotIT` | `presto.profile.*` | `target/cost-profiling/presto` | +| BigQuery | `bigquery-setup/README.md` | `wayang-platforms/wayang-bigquery` | `BigQueryCostPilotIT` | `bigquery.profile.*` | `target/cost-profiling/bigquery` | + +The commands below use PowerShell. On macOS/Linux, use `./mvnw` instead of +`.\mvnw.cmd` and replace PowerShell backticks with Bash line-continuation +backslashes. + +Trino: + +```powershell +.\mvnw.cmd -Pskip-prerequisite-check -pl wayang-platforms/wayang-trino -am ` + "-Dtest=TrinoCostPilotIT" ` + "-Dsurefire.failIfNoSpecifiedTests=false" ` + "-DfailIfNoTests=false" ` + "-Dtrino.profile.outputDir=target/cost-profiling/trino" ` + "-Dtrino.profile.rowCounts=10000,50000,100000,250000" ` + "-Dtrino.profile.plans=S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15,S16" ` + "-Dtrino.profile.repetitions=6" ` + "-Dtrino.profile.reset=true" ` + "-Drat.skip=true" ` + "-Dlicense.skip=true" ` + "-Dmaven.javadoc.skip=true" ` + test +``` + +Presto: + +```powershell +.\mvnw.cmd -Pskip-prerequisite-check -pl wayang-platforms/wayang-presto -am ` + "-Dtest=PrestoCostPilotIT" ` + "-Dsurefire.failIfNoSpecifiedTests=false" ` + "-DfailIfNoTests=false" ` + "-Dpresto.profile.outputDir=target/cost-profiling/presto" ` + "-Dpresto.profile.rowCounts=10000,50000,100000,250000" ` + "-Dpresto.profile.plans=S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15,S16" ` + "-Dpresto.profile.repetitions=6" ` + "-Dpresto.profile.reset=true" ` + "-Drat.skip=true" ` + "-Dlicense.skip=true" ` + "-Dmaven.javadoc.skip=true" ` + test +``` + +BigQuery: + +```powershell +.\mvnw.cmd -Pskip-prerequisite-check -pl wayang-platforms/wayang-bigquery -am ` + "-Dtest=BigQueryCostPilotIT" ` + "-Dsurefire.failIfNoSpecifiedTests=false" ` + "-DfailIfNoTests=false" ` + "-Dbigquery.project=YOUR_PROJECT_ID" ` + "-Dbigquery.saEmail=wayang-bq@YOUR_PROJECT_ID.iam.gserviceaccount.com" ` + "-Dbigquery.keyPath=C:\path\to\wayang-bq-key.json" ` + "-Dbigquery.location=US" ` + "-Dbigquery.profile.outputDir=target/cost-profiling/bigquery" ` + "-Dbigquery.profile.rowCounts=10000,50000,100000,250000" ` + "-Dbigquery.profile.plans=S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15,S16" ` + "-Dbigquery.profile.repetitions=6" ` + "-Dbigquery.profile.reset=true" ` + "-Drat.skip=true" ` + "-Dlicense.skip=true" ` + "-Dmaven.javadoc.skip=true" ` + test +``` + +Expected output files: + +| File | Purpose | +|------|---------| +| `executions.json` | Wayang execution records consumed by the GA profiler | +| `manifest.csv` | Human-readable mapping from run ID to plan, cardinality, repetition, and status | + +## 7. Running the Genetic Optimizer + +The entry point for learning cost parameters is: + +```text +org.apache.wayang.profiler.log.GeneticOptimizerApp +``` + +A typical profiling configuration contains: + +- platform default properties; +- `wayang..*.load.template` formulas; +- GA settings; +- the path to `executions.json`; +- the output path for learned parameters. + +Example GA settings: + +```properties +wayang.profiler.ga.timelimit.ms = 120000 +wayang.profiler.ga.maxgenerations = 800 +wayang.profiler.ga.maxstablegenerations = 150 +wayang.profiler.ga.superoptimizations = 1 +wayang.profiler.ga.intermediateupdate = 200 +wayang.profiler.ga.min-exec-time = 1 +wayang.profiler.ga.max-cardinality-spread = 100 +wayang.profiler.ga.min-cardinality-confidence = 0 +wayang.profiler.ga.binning = 1.0 +wayang.profiler.ga.output-file = +``` + +The profiler writes learned formulas such as: + +```properties +wayang.trino.tablesource.load = ... +wayang.trino.filter.load = ... +wayang.trino.join.load = ... +``` + +## 8. Cardinality Estimation Note + +For JDBC-based table sources, `JdbcTableSource#getCardinalityEstimator` may open +a JDBC connection and run: + +```sql +SELECT count(*) FROM +``` + +This is used during Wayang's optimization phase to estimate source +cardinalities. + +Important details: + +- The estimator is not called for every registered platform. +- It is called only for operators that appear in the current Wayang plan or plan + implementation being estimated. +- If the current plan contains a Trino, Presto, or BigQuery table source, the + corresponding JDBC cardinality estimator may run. +- If the count query fails, the current implementation falls back to a + conservative estimate. + +For cloud platforms, this extra count query can add overhead or fail because of +network or authentication issues. For profiling, it can be useful to support +cached or user-provided source cardinalities in the future. + +## 9. Completion Criteria + +A profiling run is complete when: + +- the platform execution stage records a `PartialExecution`; +- `executions.json` contains the expected platform; +- execution records contain estimator keys for relevant operators such as + `tablesource`, `filter`, `projection`, `join`, `reduceby`, `sort`, and + `tablesink`; +- input and output cardinalities are available; +- `GeneticOptimizerApp` can read the execution log; +- the profiler outputs learned platform load formulas; +- the learned formulas and experiment settings are documented together so they + can be interpreted as environment-specific profiling results. + +## 10. Recommended Implementation Order + +When adding profiling support for a new platform, a conservative order is: + +1. Create a minimal proof of concept for one stage, for example + `TableSource -> Filter -> TableSink`. +2. Confirm that `executions.json` contains the correct platform, estimator keys, + cardinalities, and measured duration. +3. Make sure the profiler can initialize the platform and deserialize its + execution records. +4. Run a small benchmark and generate candidate parameters. +5. Extend the workload to all important operators and combined pipelines. +6. Decide whether the learned parameters should become reference defaults or + remain documented as environment-specific profiling results. diff --git a/trino-setup/README.md b/trino-setup/README.md index e92c4014b..56dcca59b 100644 --- a/trino-setup/README.md +++ b/trino-setup/README.md @@ -11,10 +11,10 @@ The current validation has three parts: Run the commands below from the repository root. Java 17 and Docker with Docker Compose are required; Maven is provided by the repository wrapper. -The pure Trino platform branch is named `wayang-trino`: +The Trino cost-profiling branch is named `feature/trino-cost-profiling`: ```bash -git checkout wayang-trino +git checkout feature/trino-cost-profiling ``` ## Command Conventions @@ -249,6 +249,22 @@ The `-v` option removes volumes and clears the local MinIO and PostgreSQL data. | `testIcebergProjection` | SELECT subset of columns | | `testIcebergFilesMetadata` | `$files` system table, confirms Parquet on MinIO | +## Cost Profiling + +Follow the shared cost-profiling guide in +[`guides/cost-profiling.md`](../guides/cost-profiling.md). This setup guide +only covers the Trino stack itself. + +Trino-specific profiling values: + +| Item | Value | +|------|-------| +| Maven module | `wayang-platforms/wayang-trino` | +| Profiling test | `TrinoCostPilotIT` | +| Property prefix | `trino.profile.*` | +| Default output directory | `target/cost-profiling/trino` | +| Learned parameters file | `wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties` | + ## Environment Variables Override defaults if running Trino on a different host/port: diff --git a/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties b/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties index fe0429b85..4c1785e7f 100644 --- a/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties +++ b/wayang-platforms/wayang-trino/src/main/resources/wayang-trino-defaults.properties @@ -28,22 +28,16 @@ wayang.trino.cores = 4 wayang.trino.costs.fix = 0.0 wayang.trino.costs.per-ms = 1.0 -# ── Cost model ──────────────────────────────────────────────────────────────── +# Cost model # -# Formula: cpu = α * rows + β +# Formula: cpu = alpha * rows + beta # -# Trino is a distributed MPP engine: very low per-row cost (small α) because -# scans are parallelised across workers, but noticeable fixed overhead (larger β) -# from query planning and cluster coordination. -# -# Compared to a single-node source (Postgres α=55, β=380k): -# α = 10 — parallel scan makes per-row cost ~5× cheaper -# β = 800k — cluster startup + query dispatch overhead -# -# These are initial estimates; tune after real benchmarks by fitting the -# template formula below on measured data and updating the 'load' key. -# ────────────────────────────────────────────────────────────────────────────── - +# The concrete .load entries below are reference parameters learned from the +# local Week 8-2 Trino profiling run. Scope: S01-S13, row counts +# 10k/50k/100k/250k, 1 warm-up plus 5 measured repetitions. They are useful +# starting values for trying Wayang, but users should rerun profiling on their +# own Trino deployment and machine for accurate optimization. +# Keep the matching .load.template entries so the parameters can be relearned. wayang.trino.tablesource.load.template = {\ "type":"mathex", "in":0, "out":1,\ "cpu":"?*out0 + ?",\ @@ -51,10 +45,14 @@ wayang.trino.tablesource.load.template = {\ "p":0.9\ } wayang.trino.tablesource.load = {\ - "in":0, "out":1,\ - "cpu":"${10*out0 + 800000}",\ - "ram":"0",\ - "p":0.9\ + "type":"mathex",\ + "in":0,\ + "out":1,\ + "cpu":"((0.04991194623872751)*(out0))+(5.650937751899713E7)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ } wayang.trino.filter.load.template = {\ @@ -64,10 +62,14 @@ wayang.trino.filter.load.template = {\ "p":0.9\ } wayang.trino.filter.load = {\ - "in":1, "out":1,\ - "cpu":"${10*in0 + 800000}",\ - "ram":"0",\ - "p":0.9\ + "type":"mathex",\ + "in":1,\ + "out":1,\ + "cpu":"((0.1984385279077654)*(in0))+(1.930055572992389E8)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ } wayang.trino.projection.load.template = {\ @@ -77,10 +79,14 @@ wayang.trino.projection.load.template = {\ "p":0.9\ } wayang.trino.projection.load = {\ - "in":1, "out":1,\ - "cpu":"${10*in0 + 800000}",\ - "ram":"0",\ - "p":0.9\ + "type":"mathex",\ + "in":1,\ + "out":1,\ + "cpu":"((0.0033828158004091907)*(in0))+(9667.384755471103)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ } wayang.trino.join.load.template = {\ @@ -90,10 +96,14 @@ wayang.trino.join.load.template = {\ "p":0.9\ } wayang.trino.join.load = {\ - "in":2, "out":1,\ - "cpu":"${10*(in0 + in1) + 800000}",\ - "ram":"0",\ - "p":0.9\ + "type":"mathex",\ + "in":2,\ + "out":1,\ + "cpu":"((0.08209057842372317)*((in0)+(in1)))+(7.914300131274152E8)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ } wayang.trino.globalreduce.load.template = {\ @@ -103,10 +113,14 @@ wayang.trino.globalreduce.load.template = {\ "p":0.9\ } wayang.trino.globalreduce.load = {\ - "in":1, "out":1,\ - "cpu":"${10*in0 + 800000}",\ - "ram":"0",\ - "p":0.9\ + "type":"mathex",\ + "in":1,\ + "out":1,\ + "cpu":"((3.2826093296659775)*(in0))+(418.86246686849927)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ } wayang.trino.reduceby.load.template = {\ @@ -116,10 +130,14 @@ wayang.trino.reduceby.load.template = {\ "p":0.9\ } wayang.trino.reduceby.load = {\ - "in":1, "out":1,\ - "cpu":"${10*in0 + 800000}",\ - "ram":"0",\ - "p":0.9\ + "type":"mathex",\ + "in":1,\ + "out":1,\ + "cpu":"((0.06770126935475743)*(in0))+(815723.9457107394)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ } wayang.trino.sort.load.template = {\ @@ -129,10 +147,14 @@ wayang.trino.sort.load.template = {\ "p":0.9\ } wayang.trino.sort.load = {\ - "in":1, "out":1,\ - "cpu":"${10*in0 + 800000}",\ - "ram":"0",\ - "p":0.9\ + "type":"mathex",\ + "in":1,\ + "out":1,\ + "cpu":"((6345.214432640646)*(in0))+(1.5730398856855012E7)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ } wayang.trino.tablesink.load.template = {\ @@ -142,10 +164,14 @@ wayang.trino.tablesink.load.template = {\ "p":0.9\ } wayang.trino.tablesink.load = {\ - "in":1, "out":0,\ - "cpu":"${10*in0 + 800000}",\ - "ram":"0",\ - "p":0.9\ + "type":"mathex",\ + "in":1,\ + "out":0,\ + "cpu":"((5853.43034225027)*(in0))+(3.5512637525734577E9)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ } wayang.trino.sqltostream.load.query.template = {\ diff --git a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoCostPilotIT.java b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoCostPilotIT.java index 8ac6be631..0a12b3b63 100644 --- a/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoCostPilotIT.java +++ b/wayang-platforms/wayang-trino/src/test/java/org/apache/wayang/trino/TrinoCostPilotIT.java @@ -93,7 +93,7 @@ class TrinoCostPilotIT { private static final String JOIN_TIER_AMOUNT_FLATTEN_NAME = "Trino profile join tier amount flatten"; private static final Path OUTPUT_DIR = Paths.get(System.getProperty( "trino.profile.outputDir", - "C:/Users/jizhi/Desktop/2026GSoC/Trino Profiling/week6" + "target/cost-profiling/trino" )); private static final Path EXECUTIONS_PATH = OUTPUT_DIR.resolve("executions.json"); private static final Path CARDINALITIES_PATH = OUTPUT_DIR.resolve("cardinalities.json"); From 8f643b2a7ebda6407a50a0193c710485a53deced Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Tue, 28 Jul 2026 23:52:05 +0800 Subject: [PATCH 17/18] Add Trino setup license headers --- demo-trino.sh | 15 +++++++++++++++ trino-setup/demo.sh | 15 +++++++++++++++ trino-setup/docker-compose.yml | 15 +++++++++++++++ trino-setup/pom.xml | 16 ++++++++++++++++ trino-setup/scripts/run-init.sh | 16 ++++++++++++++++ .../wayang/trino/TrinoIntegrationTest.java | 17 +++++++++++++++++ 6 files changed, 94 insertions(+) diff --git a/demo-trino.sh b/demo-trino.sh index d41783b30..45b1fad7a 100644 --- a/demo-trino.sh +++ b/demo-trino.sh @@ -1,4 +1,19 @@ #!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. set -euo pipefail diff --git a/trino-setup/demo.sh b/trino-setup/demo.sh index 3ae3e42e1..ea3df7892 100644 --- a/trino-setup/demo.sh +++ b/trino-setup/demo.sh @@ -1,4 +1,19 @@ #!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. set -euo pipefail diff --git a/trino-setup/docker-compose.yml b/trino-setup/docker-compose.yml index d05a39d15..0c057540e 100644 --- a/trino-setup/docker-compose.yml +++ b/trino-setup/docker-compose.yml @@ -1,4 +1,19 @@ --- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Stack: Trino + Hive Metastore + MinIO (S3 storage) # # This is the battle-tested Trino + Iceberg local setup. diff --git a/trino-setup/pom.xml b/trino-setup/pom.xml index 645bb3241..f04955203 100644 --- a/trino-setup/pom.xml +++ b/trino-setup/pom.xml @@ -1,4 +1,20 @@ + Date: Tue, 28 Jul 2026 23:59:17 +0800 Subject: [PATCH 18/18] Align SQL API source test with unterminated JDBC queries --- .../test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java b/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java index 77537826d..91c1d7aef 100755 --- a/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java +++ b/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java @@ -311,7 +311,7 @@ public RelDataType getRowType(final RelDataTypeFactory typeFactory) { final StringBuilder query = JdbcExecutor.createSqlString(jdbcExecutor, table, Arrays.asList(), projection, null, null, null, Arrays.asList()); - assertEquals("SELECT ID, NAME FROM T1;", query.toString()); + assertEquals("SELECT ID, NAME FROM T1", query.toString()); } @Test