diff --git a/bigquery-setup/README.md b/bigquery-setup/README.md new file mode 100644 index 000000000..ad29ac3ee --- /dev/null +++ b/bigquery-setup/README.md @@ -0,0 +1,385 @@ +# BigQuery Local Setup + +Local BigQuery emulator and validation instructions for the Wayang BigQuery +platform. + +The current validation has three parts: + +1. Build the Wayang BigQuery platform and run the shared JDBC SQL-generation tests. +2. Run BigQuery-compatible SQL tests against the local emulator. +3. Run the Wayang BigQuery operator tests through JDBC against real BigQuery. + +Run the commands below from the repository root. Java 17 and Docker with Docker +Compose are required for the emulator tests. A GCP project and service-account +key, plus the `gcloud` SDK, are required only for the real BigQuery operator +tests. Maven is provided by the repository wrapper. + +```bash +git checkout feature/bigquery-cost-profiling +``` + +## Command Conventions + +Use the `bash` blocks on macOS/Linux terminals. Use the `powershell` blocks on +Windows PowerShell from the repository root. Docker Compose commands are the +same on both platforms. The `gcloud` commands also work on Windows; either run +each command on one line or replace Bash line-continuation backslashes with +PowerShell backticks. + +## Stack + +| Component | Image | Port | Role | +|-----------|-------|------|------| +| **BigQuery Emulator** | `ghcr.io/goccy/bigquery-emulator:0.6.6` | 9050 (HTTP) / 9060 (gRPC) | BigQuery-compatible SQL engine | + +Single container. Data is seeded from `data.yaml` on startup and lives in memory. + +## Directory Layout + +``` +bigquery-setup/ +|-- docker-compose.yml # Emulator container +|-- data.yaml # Seed data (test-project.sales.orders) +|-- pom.xml # Standalone Maven project +`-- src/test/java/.../ + `-- BigQueryEmulatorIT.java # JUnit 5 integration tests + +wayang-platforms/wayang-bigquery/src/test/java/.../ +`-- BigQueryOperatorsIT.java # Wayang operator tests against real BigQuery +``` + +## 1. Test the Wayang BigQuery Platform + +Build the BigQuery platform and its required modules: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-bigquery -am -DskipTests -Drat.skip=true test +``` + +On PowerShell: + +```powershell +.\mvnw.cmd --% -Pskip-prerequisite-check -pl wayang-platforms/wayang-bigquery -am -DskipTests -Drat.skip=true test +``` + +Then run the shared JDBC SQL-generation tests: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-jdbc-template -am -Dtest=JdbcExecutorTest -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false -Drat.skip=true test +``` + +On PowerShell: + +```powershell +.\mvnw.cmd --% -Pskip-prerequisite-check -pl wayang-platforms/wayang-jdbc-template -am -Dtest=JdbcExecutorTest -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false -Drat.skip=true test +``` + +Expected result: + +```text +Wayang Platform BigQuery ... SUCCESS +Tests run: 4, Failures: 0, Errors: 0, Skipped: 0 +``` + +## 2. Test the Local BigQuery Emulator + +### 1. Start the emulator + +```bash +docker compose -f bigquery-setup/docker-compose.yml up -d +``` + +The emulator starts in ~2 seconds. Data from `data.yaml` is loaded automatically. + +### 2. Run integration tests + +```bash +./mvnw -f bigquery-setup/pom.xml -Dtest=BigQueryEmulatorIT test +``` + +On PowerShell: + +```powershell +.\mvnw.cmd --% -f bigquery-setup/pom.xml -Dtest=BigQueryEmulatorIT test +``` + +The successful result must show that no tests were skipped: + +```text +Tests run: 7, Failures: 0, Errors: 0, Skipped: 0 +``` + +If the emulator is unavailable, Maven can still print `BUILD SUCCESS` while +showing `Skipped: 7`. That does not count as a successful emulator test. + +### 3. Manual exploration + +Query via curl: + +```bash +curl -s -X POST \ + "http://localhost:9050/bigquery/v2/projects/test-project/queries" \ + -H "Content-Type: application/json" \ + -d '{"query": "SELECT * FROM sales.orders LIMIT 5", "useLegacySql": false}' \ + | python3 -m json.tool +``` + +### 4. Tear down + +```bash +docker compose -f bigquery-setup/docker-compose.yml down +``` + +## 3. Test the Wayang Operators Against Real BigQuery + +`BigQueryOperatorsIT` uses the BigQuery JDBC driver and cannot run against the +local emulator. It requires a real GCP project, a service-account JSON key, and +a reference table containing the same 10 rows as `bigquery-setup/data.yaml`. + +The tests issue `SELECT`, `CREATE TABLE AS`, and `DROP` statements. The +`TableSink` test creates and then drops `sales.wayang_emea_orders`; the +reference `sales.orders` table remains in place. + +### 1. Enable BigQuery and create a service account + +Replace `YOUR_PROJECT_ID` in the following commands: + +```bash +gcloud auth login +gcloud config set project YOUR_PROJECT_ID +gcloud services enable bigquery.googleapis.com + +gcloud iam service-accounts create wayang-bq \ + --display-name="Wayang BigQuery IT" + +gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \ + --member="serviceAccount:wayang-bq@YOUR_PROJECT_ID.iam.gserviceaccount.com" \ + --role="roles/bigquery.jobUser" + +gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \ + --member="serviceAccount:wayang-bq@YOUR_PROJECT_ID.iam.gserviceaccount.com" \ + --role="roles/bigquery.dataEditor" + +gcloud iam service-accounts keys create "$HOME/wayang-bq-key.json" \ + --iam-account="wayang-bq@YOUR_PROJECT_ID.iam.gserviceaccount.com" +``` + +On Windows PowerShell, the same setup can be run as: + +```powershell +gcloud auth login +gcloud config set project YOUR_PROJECT_ID +gcloud services enable bigquery.googleapis.com +gcloud iam service-accounts create wayang-bq --display-name="Wayang BigQuery IT" +gcloud projects add-iam-policy-binding YOUR_PROJECT_ID --member="serviceAccount:wayang-bq@YOUR_PROJECT_ID.iam.gserviceaccount.com" --role="roles/bigquery.jobUser" +gcloud projects add-iam-policy-binding YOUR_PROJECT_ID --member="serviceAccount:wayang-bq@YOUR_PROJECT_ID.iam.gserviceaccount.com" --role="roles/bigquery.dataEditor" +gcloud iam service-accounts keys create "$HOME\wayang-bq-key.json" --iam-account="wayang-bq@YOUR_PROJECT_ID.iam.gserviceaccount.com" +``` + +The service account needs `jobUser` to run queries and `dataEditor` to read the +reference table and create/drop the sink table. + +### 2. Load the reference table + +Create a US dataset, then load the exact rows from `data.yaml` with a load job: + +```bash +bq --location=US mk --dataset YOUR_PROJECT_ID:sales + +cat > /tmp/orders.csv <<'CSV' +1,APAC,Widget A,1500.0 +2,EMEA,Widget B,800.5 +3,AMER,Widget A,2200.0 +4,APAC,Widget C,350.75 +5,EMEA,Widget A,1100.0 +6,AMER,Widget B,950.25 +7,APAC,Widget B,1750.0 +8,EMEA,Widget C,420.0 +9,AMER,Widget C,680.5 +10,APAC,Widget A,3000.0 +CSV + +bq --project_id=YOUR_PROJECT_ID --location=US load --replace \ + --source_format=CSV sales.orders /tmp/orders.csv \ + order_id:INTEGER,region:STRING,product:STRING,amount:FLOAT +``` + +Confirm that the table matches the assertions: + +```bash +bq --project_id=YOUR_PROJECT_ID --location=US query --use_legacy_sql=false \ + 'SELECT count(*) n, round(sum(amount), 2) total FROM `YOUR_PROJECT_ID.sales.orders`' +``` + +Expected values are `n = 10` and `total = 12752.0`. + +### 3. Run the operator tests + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-bigquery -am \ + -Dtest=BigQueryOperatorsIT -Dsurefire.failIfNoSpecifiedTests=false \ + -DfailIfNoTests=false \ + -Dbigquery.project=YOUR_PROJECT_ID \ + -Dbigquery.saEmail=wayang-bq@YOUR_PROJECT_ID.iam.gserviceaccount.com \ + -Dbigquery.keyPath="$HOME/wayang-bq-key.json" \ + -Dbigquery.location=US \ + -Drat.skip=true -Dlicense.skip=true test +``` + +On PowerShell: + +```powershell +.\mvnw.cmd --% -Pskip-prerequisite-check -pl wayang-platforms/wayang-bigquery -am -Dtest=BigQueryOperatorsIT -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false -Dbigquery.project=YOUR_PROJECT_ID -Dbigquery.saEmail=wayang-bq@YOUR_PROJECT_ID.iam.gserviceaccount.com -Dbigquery.keyPath=C:\path\to\wayang-bq-key.json -Dbigquery.location=US -Drat.skip=true -Dlicense.skip=true test +``` + +System properties take precedence over the equivalent environment variables: + +| System property | Environment variable | Default | +|-----------------|----------------------|---------| +| `bigquery.project` | `BIGQUERY_PROJECT` | `your-project` | +| `bigquery.saEmail` | `BIGQUERY_SA_EMAIL` | `wayang-bq@.iam.gserviceaccount.com` | +| `bigquery.keyPath` | `BIGQUERY_KEY_PATH` | `$HOME/wayang-bq-key.json` | +| `bigquery.table` | `BIGQUERY_TABLE` | `` `.sales.orders` `` | +| `bigquery.location` | `BIGQUERY_LOCATION` | `US` | + +Successful real-BigQuery validation must show: + +```text +Tests run: 18, Failures: 0, Errors: 0, Skipped: 0 +``` + +### Previously verified result + +On June 11, 2026, the original 12-test real-BigQuery suite was run successfully +against a non-billing GCP project using the service-account flow documented +above: + +```text +[SETUP] Connected to BigQuery project +[PASS] TableScan: 10 rows +[PASS] Filter(region='APAC'): 4 rows +[PASS] GlobalReduce SUM(amount) = 12752.0 +[PASS] TableSink wrote 3 EMEA rows +Tests run: 12, Failures: 0, Errors: 0, Skipped: 0 +BUILD SUCCESS +``` + +This verified the complete `Wayang -> BigQuery JDBC -> service-account OAuth -> +real BigQuery` path, including reads, SQL pushdown, aggregation, sorting, and +`CREATE TABLE AS SELECT`. The sink table was removed automatically after the +test, while the reference `sales.orders` table was retained for reruns. No +service-account key or credential file is stored in this repository. + +On June 18, 2026, the expanded 18-test suite was also verified successfully +against real BigQuery, using `Location=US` and the local proxy settings when +needed: + +```text +Tests run: 18, Failures: 0, Errors: 0, Skipped: 0 +BUILD SUCCESS +``` + +This includes the full Wayang join plan with join-result normalization and all +five `JavaPlanBuilder` combination tests. On the same date, the local BigQuery +emulator suite was re-run with Docker and passed 7/7 with zero skipped tests. + +If the browser uses a local proxy, pass the same proxy to both CLI tools and +the Maven test JVM. For example, with a proxy at `127.0.0.1:7890`, set +`HTTP_PROXY`/`HTTPS_PROXY` and use `JAVA_TOOL_OPTIONS` with +`-Dhttp.proxyHost`, `-Dhttp.proxyPort`, `-Dhttps.proxyHost`, and +`-Dhttps.proxyPort`. + +On PowerShell: + +```powershell +$env:HTTP_PROXY="http://127.0.0.1:7890" +$env:HTTPS_PROXY="http://127.0.0.1:7890" +$env:JAVA_TOOL_OPTIONS="-Dhttp.proxyHost=127.0.0.1 -Dhttp.proxyPort=7890 -Dhttps.proxyHost=127.0.0.1 -Dhttps.proxyPort=7890" +.\mvnw.cmd --% -Pskip-prerequisite-check -pl wayang-platforms/wayang-bigquery -am -Dtest=BigQueryOperatorsIT -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false -Dbigquery.project=YOUR_PROJECT_ID -Dbigquery.saEmail=wayang-bq@YOUR_PROJECT_ID.iam.gserviceaccount.com -Dbigquery.keyPath=C:\path\to\wayang-bq-key.json -Dbigquery.location=US -Drat.skip=true -Dlicense.skip=true test +Remove-Item Env:HTTP_PROXY, Env:HTTPS_PROXY, Env:JAVA_TOOL_OPTIONS +``` + +If credentials or the project configuration are missing, Maven can still print +`BUILD SUCCESS` with `Skipped: 17`. Only the platform-binding test ran in that +case, so the BigQuery operators were not validated. + +## 4. Re-run Cost Profiling + +Follow the shared cost-profiling guide in +[`guides/cost-profiling.md`](../guides/cost-profiling.md). This setup guide +only covers the BigQuery emulator and real BigQuery validation setup. + +BigQuery-specific profiling values: + +| Item | Value | +|------|-------| +| Maven module | `wayang-platforms/wayang-bigquery` | +| Profiling test | `BigQueryCostPilotIT` | +| Property prefix | `bigquery.profile.*` | +| Default output directory | `target/cost-profiling/bigquery` | +| Learned parameters file | `wayang-platforms/wayang-bigquery/src/main/resources/wayang-bigquery-defaults.properties` | + +## Test Coverage + +### Local emulator tests + +| Test | What it checks | +|------|----------------| +| `testDatasetVisible` | `sales` dataset exists | +| `testFullScan` | Full table scan, 10 rows | +| `testFilterByRegion` | `WHERE region = 'APAC'` | +| `testFilterByAmount` | `WHERE amount > 1000` | +| `testAggregation` | `GROUP BY region` + `SUM(amount)` | +| `testProjection` | `SELECT region, product LIMIT 5` | +| `testCount` | `SELECT count(*)`, used by Wayang for cardinality estimation | + +### Real BigQuery operator tests + +| Test | What it checks | +|------|----------------| +| `testPlatformBinding` | `BigQueryTableSource` is bound to `BigQueryPlatform` | +| `testFailsWithoutJdbcConfig` | Execution fails clearly without the JDBC URL | +| `testTableScan` | Full table scan through Wayang | +| `testFilterString` | String filter pushdown | +| `testFilterNumeric` | Numeric filter pushdown | +| `testProjection` | Multi-column projection pushdown | +| `testFilterAndProjection` | Combined filter and projection pipeline | +| `testCardinalityMatches` | BigQuery `COUNT(*)` cardinality estimate | +| `testGlobalReduce` | Global `SUM(amount)` | +| `testReduceBy` | `SUM(amount) GROUP BY region` | +| `testSort` | BigQuery sort operator SQL-clause contract | +| `testTableSink` | `CREATE TABLE AS SELECT` and cleanup | +| `testJoin` | Full Wayang join plan with normalization before the collecting sink | +| `javaPlanBuilderReadTableFilterProjection` | `readTable -> filter -> projection -> collect` | +| `javaPlanBuilderReadTableFilterGlobalReduce` | `readTable -> filter -> globalReduce -> collect` | +| `javaPlanBuilderReadTableReduceBySort` | `readTable -> reduceByKey -> sort -> collect` | +| `javaPlanBuilderReadTableFilterProjectionTableSink` | `readTable -> filter -> projection -> writeTable` | +| `javaPlanBuilderReadTableJoin` | `readTable + readTable -> join -> collect` | + +The combination tests use `.withTargetPlatform(BigQuery.platform())` so the +small 10-row fixture still exercises BigQuery SQL pushdown. The join test creates +and cleans up a temporary distinct-region lookup table. + +## Emulator Environment Variable + +```bash +BIGQUERY_HOST=http://localhost:9050 ./mvnw -f bigquery-setup/pom.xml -Dtest=BigQueryEmulatorIT test +``` + +On PowerShell: + +```powershell +$env:BIGQUERY_HOST="http://localhost:9050" +.\mvnw.cmd --% -f bigquery-setup/pom.xml -Dtest=BigQueryEmulatorIT test +Remove-Item Env:BIGQUERY_HOST +``` + +## Notes + +- Tests use `google-cloud-bigquery` client library (REST-based, no JDBC). +- The client connects with `NoCredentials`; no GCP account is needed. +- The BigQuery JDBC driver (`google-cloud-bigquery-jdbc`) requires OAuth even + against the emulator, so `BigQueryOperatorsIT` runs only against real + BigQuery. +- Emulator tests validate SQL compatibility, but only `BigQueryOperatorsIT` + validates end-to-end Wayang-to-BigQuery JDBC execution. diff --git a/bigquery-setup/data.yaml b/bigquery-setup/data.yaml new file mode 100644 index 000000000..762d66b6d --- /dev/null +++ b/bigquery-setup/data.yaml @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +projects: +- id: test-project + datasets: + - id: sales + tables: + - id: orders + columns: + - name: order_id + type: INTEGER + - name: region + type: STRING + - name: product + type: STRING + - name: amount + type: FLOAT + data: + - order_id: 1 + region: APAC + product: Widget A + amount: 1500.0 + - order_id: 2 + region: EMEA + product: Widget B + amount: 800.5 + - order_id: 3 + region: AMER + product: Widget A + amount: 2200.0 + - order_id: 4 + region: APAC + product: Widget C + amount: 350.75 + - order_id: 5 + region: EMEA + product: Widget A + amount: 1100.0 + - order_id: 6 + region: AMER + product: Widget B + amount: 950.25 + - order_id: 7 + region: APAC + product: Widget B + amount: 1750.0 + - order_id: 8 + region: EMEA + product: Widget C + amount: 420.0 + - order_id: 9 + region: AMER + product: Widget C + amount: 680.5 + - order_id: 10 + region: APAC + product: Widget A + amount: 3000.0 diff --git a/bigquery-setup/demo.sh b/bigquery-setup/demo.sh new file mode 100644 index 000000000..49fe0521a --- /dev/null +++ b/bigquery-setup/demo.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WAYANG_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +exec "$WAYANG_ROOT/demo-bigquery.sh" "$@" diff --git a/bigquery-setup/docker-compose.yml b/bigquery-setup/docker-compose.yml new file mode 100644 index 000000000..fb6cea8f5 --- /dev/null +++ b/bigquery-setup/docker-compose.yml @@ -0,0 +1,42 @@ +--- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Stack: BigQuery Emulator (goccy/bigquery-emulator) +# +# Single container — no metastore, no object storage needed. +# Data is seeded from data.yaml on startup and lives in memory. +# +# Ports: +# HTTP (REST API): http://localhost:9050 +# gRPC (Storage API): localhost:9060 + +services: + + bigquery: + image: ghcr.io/goccy/bigquery-emulator:0.6.6 + platform: linux/amd64 + container_name: bigquery-emulator + ports: + - "9050:9050" + - "9060:9060" + volumes: + - ./data.yaml:/data.yaml + command: --project=test-project --data-from-yaml=/data.yaml + healthcheck: + test: ["CMD-SHELL", "bash -c ': >/dev/tcp/localhost/9050'"] + interval: 10s + timeout: 5s + retries: 5 diff --git a/bigquery-setup/pom.xml b/bigquery-setup/pom.xml new file mode 100644 index 000000000..5279bb71a --- /dev/null +++ b/bigquery-setup/pom.xml @@ -0,0 +1,78 @@ + + + + 4.0.0 + + org.apache.wayang + bigquery-setup + 1.0-SNAPSHOT + jar + + BigQuery Local Setup — Integration Tests + + Standalone integration tests for a local BigQuery emulator. + Independent of the Wayang codebase. + + + + 11 + 11 + UTF-8 + 5.10.2 + 2.49.0 + + + + + + com.google.cloud + google-cloud-bigquery + ${bigquery.version} + test + + + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + + org.slf4j + slf4j-simple + 2.0.12 + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + + + diff --git a/bigquery-setup/src/test/java/org/apache/wayang/bigquery/BigQueryEmulatorIT.java b/bigquery-setup/src/test/java/org/apache/wayang/bigquery/BigQueryEmulatorIT.java new file mode 100644 index 000000000..07e7d4431 --- /dev/null +++ b/bigquery-setup/src/test/java/org/apache/wayang/bigquery/BigQueryEmulatorIT.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery; + +import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.NoCredentials; +import com.google.cloud.bigquery.*; +import org.junit.jupiter.api.*; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Integration tests for the local BigQuery emulator. + * + * Prerequisites: run `docker-compose up -d` first. + * + * Run tests: + * mvn test -Pintegration + */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class BigQueryEmulatorIT { + + private static final String EMULATOR_HOST = System.getenv().getOrDefault("BIGQUERY_HOST", "http://localhost:9050"); + private static final String PROJECT_ID = "test-project"; + private static final String DATASET = "sales"; + + private static BigQuery bigquery; + private static boolean emulatorAvailable = false; + + @BeforeAll + static void setupClient() { + try { + bigquery = BigQueryOptions.newBuilder() + .setHost(EMULATOR_HOST) + .setLocation("US") + .setProjectId(PROJECT_ID) + .setCredentials(NoCredentials.getInstance()) + .build() + .getService(); + + // Quick connectivity check + bigquery.getDataset(DatasetId.of(PROJECT_ID, DATASET)); + emulatorAvailable = true; + System.out.printf("Connected to BigQuery emulator at %s%n", EMULATOR_HOST); + } catch (Exception e) { + System.err.println("BigQuery emulator not available: " + e.getMessage()); + } + } + + private List> runQuery(String sql) throws InterruptedException { + QueryJobConfiguration config = QueryJobConfiguration.newBuilder(sql) + .setUseLegacySql(false) + .build(); + TableResult result = bigquery.query(config); + List> rows = new ArrayList<>(); + for (FieldValueList row : result.iterateAll()) { + List r = new ArrayList<>(); + for (FieldValue val : row) { + r.add(val.isNull() ? null : val.getValue()); + } + rows.add(r); + } + return rows; + } + + // ── Test 1: Dataset visible ────────────────────────────────────────── + + @Test + @Order(1) + @DisplayName("BigQuery emulator: dataset 'sales' is visible") + void testDatasetVisible() { + Assumptions.assumeTrue(emulatorAvailable, "Emulator not available"); + + Dataset ds = bigquery.getDataset(DatasetId.of(PROJECT_ID, DATASET)); + assertNotNull(ds, "Dataset 'sales' should exist"); + System.out.println("[PASS] Dataset 'sales' is visible"); + } + + // ── Test 2: Full table scan ────────────────────────────────────────── + + @Test + @Order(2) + @DisplayName("BigQuery emulator: full table scan on orders") + void testFullScan() throws Exception { + Assumptions.assumeTrue(emulatorAvailable, "Emulator not available"); + + List> rows = runQuery( + "SELECT * FROM `test-project.sales.orders` ORDER BY order_id" + ); + assertEquals(10, rows.size(), "Expected 10 rows"); + System.out.println("[PASS] Full scan: " + rows.size() + " rows"); + rows.forEach(r -> System.out.println(" " + r)); + } + + // ── Test 3: Filter by region ───────────────────────────────────────── + + @Test + @Order(3) + @DisplayName("BigQuery emulator: filter by region = APAC") + void testFilterByRegion() throws Exception { + Assumptions.assumeTrue(emulatorAvailable, "Emulator not available"); + + List> rows = runQuery( + "SELECT order_id, region, amount FROM `test-project.sales.orders` WHERE region = 'APAC' ORDER BY order_id" + ); + assertFalse(rows.isEmpty(), "Should have APAC rows"); + rows.forEach(r -> assertEquals("APAC", r.get(1), "All rows should be APAC")); + System.out.printf("[PASS] Filter: %d APAC rows%n", rows.size()); + } + + // ── Test 4: Filter by amount ───────────────────────────────────────── + + @Test + @Order(4) + @DisplayName("BigQuery emulator: filter by amount > 1000") + void testFilterByAmount() throws Exception { + Assumptions.assumeTrue(emulatorAvailable, "Emulator not available"); + + List> rows = runQuery( + "SELECT order_id, amount FROM `test-project.sales.orders` WHERE amount > 1000 ORDER BY amount DESC" + ); + assertFalse(rows.isEmpty()); + rows.forEach(r -> assertTrue( + Double.parseDouble(r.get(1).toString()) > 1000.0, + "All amounts should be > 1000" + )); + System.out.printf("[PASS] Amount filter: %d rows with amount > 1000%n", rows.size()); + } + + // ── Test 5: Aggregation ────────────────────────────────────────────── + + @Test + @Order(5) + @DisplayName("BigQuery emulator: aggregate by region") + void testAggregation() throws Exception { + Assumptions.assumeTrue(emulatorAvailable, "Emulator not available"); + + List> rows = runQuery( + "SELECT region, COUNT(*) AS cnt, SUM(amount) AS total " + + "FROM `test-project.sales.orders` GROUP BY region ORDER BY total DESC" + ); + assertFalse(rows.isEmpty()); + System.out.println("[PASS] Aggregation by region:"); + rows.forEach(r -> System.out.printf(" region=%-5s count=%s total=%s%n", + r.get(0), r.get(1), r.get(2))); + } + + // ── Test 6: Projection ─────────────────────────────────────────────── + + @Test + @Order(6) + @DisplayName("BigQuery emulator: projection (region, product)") + void testProjection() throws Exception { + Assumptions.assumeTrue(emulatorAvailable, "Emulator not available"); + + List> rows = runQuery( + "SELECT region, product FROM `test-project.sales.orders` LIMIT 5" + ); + assertEquals(5, rows.size()); + rows.forEach(r -> { + assertNotNull(r.get(0), "region should not be null"); + assertNotNull(r.get(1), "product should not be null"); + }); + System.out.println("[PASS] Projection (region, product): 5 rows"); + } + + // ── Test 7: COUNT(*) ───────────────────────────────────────────────── + + @Test + @Order(7) + @DisplayName("BigQuery emulator: SELECT count(*)") + void testCount() throws Exception { + Assumptions.assumeTrue(emulatorAvailable, "Emulator not available"); + + List> rows = runQuery( + "SELECT count(*) FROM `test-project.sales.orders`" + ); + assertEquals(1, rows.size()); + long count = Long.parseLong(rows.get(0).get(0).toString()); + assertEquals(10, count, "Should have 10 rows"); + System.out.println("[PASS] COUNT(*) = " + count); + } +} diff --git a/conf/wayang-defaults.properties b/conf/wayang-defaults.properties index 736b73ebe..ff2dcf33a 100644 --- a/conf/wayang-defaults.properties +++ b/conf/wayang-defaults.properties @@ -16,7 +16,7 @@ # # Configure statistics collection. -wayang.core.log.enabled = true +wayang.core.log.enabled = false wayang.core.explain.enabled = false wayang.core.explain.directrory = ~/.wayang/ diff --git a/demo-bigquery.sh b/demo-bigquery.sh new file mode 100644 index 000000000..69fe5a188 --- /dev/null +++ b/demo-bigquery.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +WAYANG_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LIVE_MODE=false +[[ "${1:-}" == "--live" ]] && LIVE_MODE=true + +BQ_PROJECT="${BQ_PROJECT:-my-project}" +BQ_URL="${BQ_URL:-}" +MAVEN_FLAGS="-Pskip-prerequisite-check -Drat.skip=true -Dmaven.javadoc.skip=true" + +banner() { + echo + echo "============================================================" + printf " %s\n" "$*" + echo "============================================================" + echo +} + +step() { + echo + echo "-- $*" + echo +} + +pause() { + if [[ "${WAYANG_DEMO_AUTO:-false}" != "true" ]]; then + echo + read -rp "Press ENTER to continue..." _ || true + echo + fi +} + +run_demo_class() { + local main_class="$1" + shift + cd "$WAYANG_ROOT" + "$WAYANG_ROOT/mvnw" exec:java -pl wayang-platforms/wayang-bigquery \ + -Dexec.mainClass="$main_class" \ + "$@" \ + ${MAVEN_FLAGS} -q 2>/dev/null || true +} + +banner "ACT 1: BigQuery cost model" +step "Read cost settings from wayang-bigquery-defaults.properties" +run_demo_class "org.apache.wayang.bigquery.BigQueryDemo" \ + "-Dbigquery.mode=cost" \ + "-Dbigquery.project=${BQ_PROJECT}" + +pause + +banner "ACT 2: BigQuery filter operator" +if [[ "$LIVE_MODE" == true && -n "$BQ_URL" ]]; then + run_demo_class "org.apache.wayang.bigquery.BigQueryDemo" \ + "-Dbigquery.mode=filter" \ + "-Dbigquery.url=${BQ_URL}" \ + "-Dbigquery.project=${BQ_PROJECT}" +else + run_demo_class "org.apache.wayang.bigquery.BigQueryDemo" \ + "-Dbigquery.mode=filter" \ + "-Dbigquery.project=${BQ_PROJECT}" +fi + +pause + +banner "ACT 3: BigQuery projection operator" +if [[ "$LIVE_MODE" == true && -n "$BQ_URL" ]]; then + run_demo_class "org.apache.wayang.bigquery.BigQueryDemo" \ + "-Dbigquery.mode=projection" \ + "-Dbigquery.url=${BQ_URL}" \ + "-Dbigquery.project=${BQ_PROJECT}" +else + run_demo_class "org.apache.wayang.bigquery.BigQueryDemo" \ + "-Dbigquery.mode=projection" \ + "-Dbigquery.project=${BQ_PROJECT}" +fi + +banner "Demo complete" diff --git a/guides/cost-profiling.md b/guides/cost-profiling.md new file mode 100644 index 000000000..25f8f9bd3 --- /dev/null +++ b/guides/cost-profiling.md @@ -0,0 +1,430 @@ + + +# Cost Profiling Guide + +This document explains why Apache Wayang needs platform-specific cost +profiling, how profiling data is collected, how the genetic optimizer learns +cost parameters, and how users can repeat the profiling workflow on their own +hardware. + +The examples below use Trino, but the same workflow also applies to other +JDBC-based platforms such as Presto and BigQuery. + +Version 2.0 uses S01 through S16 as the profiling workload, including the +join-heavy pipelines S14 through S16. It keeps the guide focused on data +collection and parameter learning, leaving follow-up quality checks out of +scope for now. + +## 1. Why Profiling Is Needed + +Wayang can map the same logical plan to different execution platforms, such as +Java, Spark, Trino, Presto, or BigQuery. For example, a user query may contain: + +```text +TableSource -> Filter -> Projection -> TableSink +``` + +The optimizer needs a cost model to decide whether these operators should stay +on a SQL platform or be moved to another platform. In this context, "cost" does +not mean cloud billing cost. It is the numerical value that Wayang uses to +compare alternative execution plans. + +With the default Trino configuration: + +```properties +wayang.trino.costs.fix = 0.0 +wayang.trino.costs.per-ms = 1.0 +``` + +the optimizer cost can be interpreted approximately as: + +```text +cost = estimated execution time in milliseconds +``` + +However, the real execution time depends on the user's machine, cluster size, +network, database configuration, and workload. Therefore, users should profile +their own environment when they need accurate cost parameters. + +## 2. Load Profile Formulas + +Each execution operator has a load profile. For example, a table source may use +a formula like: + +```properties +wayang.trino.tablesource.load = { + "type":"mathex", + "in":0, + "out":1, + "cpu":"((10)*(out0))+(800000)", + "ram":"0", + "disk":"0", + "net":"0", + "p":0.9 +} +``` + +This can be read as: + +```text +CPU load = alpha * number_of_rows + beta +``` + +where: + +- `out0` is the output cardinality. +- `alpha` is the per-row cost. +- `beta` is the fixed overhead, such as query planning, scheduling, and remote + execution startup. +- `p` is the confidence of the estimate. + +The profiling goal is to learn reasonable values for `alpha` and `beta` from +real execution records. + +Wayang can also define templates with unknown parameters: + +```properties +wayang.trino.tablesource.load.template = { + "type":"mathex", + "in":0, + "out":1, + "cpu":"?*out0 + ?", + "ram":"0", + "disk":"0", + "net":"0", + "p":0.9 +} +``` + +The genetic optimizer reads the templates and replaces the `?` placeholders with +learned values. + +## 3. Profiling Workflow + +The expected profiling workflow is: + +```text +Run Wayang jobs with different operators and input cardinalities + | + v +Record platform, operator lineage, cardinalities, and runtime + | + v + executions.json + | + v + GeneticOptimizerApp reads the execution records + | + v +Learn the unknown parameters in *.load.template + | + v +Write learned *.load formulas +``` + +Wayang stores measured executions as `PartialExecution` records. Each record +contains: + +- the measured execution time; +- the platform that executed the stage; +- one or more `ExecutionLineageNode` objects; +- the load profile estimator for each profiled operator; +- input and output cardinalities. + +The default execution log location is usually: + +```text +~/.wayang/executions.json +``` + +For controlled profiling experiments, it is better to write the log to a +dedicated experiment folder, for example: + +```text +C:\Users\\Desktop\Wayang Profiling\trino\week8\executions.json +``` + +## 4. Experiment Design + +Profiling should include both single-operator pipelines and combined pipelines. +Single-operator pipelines help isolate each operator. Combined pipelines help +the optimizer learn parameters from realistic SQL stages, where multiple +operators are executed together. + +Choose input cardinalities according to the machine or cluster being profiled. +The values below are only an example that can run on a laptop-sized local +setup: + +```text +10k, 50k, 100k, 250k +``` + +For a smaller machine, use fewer or smaller cardinalities. For a larger local +or remote platform, add larger cardinalities so the learned model reflects the +scale that users expect to run. + +Recommended repetitions: + +```text +1 warm-up run + 5 measured runs +``` + +Profiling pipelines: + +| Plan | Pipeline | +|------|----------| +| S01 | TableSource -> TableSink | +| S02 | TableSource -> Filter(50%) -> TableSink | +| S03 | TableSource -> Projection(order_id, amount) -> TableSink | +| S04 | TableSource -> Filter(50%) -> Projection(order_id, amount) -> TableSink | +| S05 | TableSource -> GlobalReduce(sum amount) -> TableSink | +| S06 | TableSource -> ReduceBy(bucket) -> TableSink | +| S07 | TableSource -> Sort(amount) -> TableSink | +| S08 | Orders -> Join(Customers 1k) -> Projection -> TableSink | +| S09 | TableSource -> Filter(50%) -> GlobalReduce -> TableSink | +| S10 | TableSource -> Filter(50%) -> ReduceBy -> TableSink | +| S11 | TableSource -> Filter(50%) -> Sort(amount) -> TableSink | +| S12 | TableSource -> Projection(order_id, amount) -> Sort(amount) -> TableSink | +| S13 | TableSource -> Filter(50%) -> Projection(order_id, amount) -> Sort(amount) -> TableSink | +| S14 | Orders -> Filter(50%) -> Join(Customers 1k) -> Projection -> TableSink | +| S15 | Orders -> Join(Customers 1k) -> Projection(order_id, tier, amount) -> Sort(amount) -> TableSink | +| S16 | Orders -> Join(Customers 1k) -> Projection(tier, amount) -> ReduceBy(tier) -> TableSink | + +S01 through S16 should be treated as one profiling workload, including the +join-heavy plans S14 through S16. For example, using 16 plans, 4 cardinalities, +and 6 repetitions produces: + +```text +16 * 4 * 6 = 384 Wayang executions +``` + +If users choose a different number of cardinalities or repetitions, the total +number of executions changes accordingly: + +```text +number_of_plans * number_of_cardinalities * repetitions +``` + +The reference parameters shipped in the platform defaults were learned from our +local Week 8 profiling runs over S01 through S13, with row counts +10k/50k/100k/250k and 1 warm-up plus 5 measured repetitions. S14 through S16 +were added to this guide to document the join-heavy pipelines that users should +include when they rerun profiling in their own environment. The shipped +parameters are intended as reasonable starting values for users who just want +to try Wayang; they are not universal parameters for every deployment. + +## 5. Benchmarking Rules + +To reduce measurement noise: + +1. Create test data before the measured run. Do not include fixture setup time + in operator duration. +2. Run at least one warm-up execution for each plan/cardinality pair. +3. Repeat each measured scenario multiple times. +4. Store every individual measurement instead of storing only averages. +5. Record exact input and output cardinalities. +6. Keep platform settings stable, including worker count, JVM settings, memory + limits, and connector configuration. +7. Record abnormal runs, such as failures caused by GC, cold cache, network + issues, or competing workloads. + +For distributed systems such as Trino, it is also important to define what the +model should predict: + +- If Wayang should predict user-visible runtime, fit wall-clock elapsed time. +- If the platform reports CPU time and the model uses CPU load, make sure the + conversion to Wayang cost is consistent with the resource model. +- Parameters learned on a local Docker setup should be treated as local + reference values, not universal defaults for every deployment. + +## 6. Running a Profiling Experiment + +The exact command depends on the platform module, test class, and property +prefix. + +| Platform | Setup guide | Maven module | Test class | Property prefix | Default output directory | +|----------|-------------|--------------|------------|-----------------|--------------------------| +| Trino | `trino-setup/README.md` | `wayang-platforms/wayang-trino` | `TrinoCostPilotIT` | `trino.profile.*` | `target/cost-profiling/trino` | +| Presto | `presto-setup/README.md` | `wayang-platforms/wayang-presto` | `PrestoCostPilotIT` | `presto.profile.*` | `target/cost-profiling/presto` | +| BigQuery | `bigquery-setup/README.md` | `wayang-platforms/wayang-bigquery` | `BigQueryCostPilotIT` | `bigquery.profile.*` | `target/cost-profiling/bigquery` | + +The commands below use PowerShell. On macOS/Linux, use `./mvnw` instead of +`.\mvnw.cmd` and replace PowerShell backticks with Bash line-continuation +backslashes. + +Trino: + +```powershell +.\mvnw.cmd -Pskip-prerequisite-check -pl wayang-platforms/wayang-trino -am ` + "-Dtest=TrinoCostPilotIT" ` + "-Dsurefire.failIfNoSpecifiedTests=false" ` + "-DfailIfNoTests=false" ` + "-Dtrino.profile.outputDir=target/cost-profiling/trino" ` + "-Dtrino.profile.rowCounts=10000,50000,100000,250000" ` + "-Dtrino.profile.plans=S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15,S16" ` + "-Dtrino.profile.repetitions=6" ` + "-Dtrino.profile.reset=true" ` + "-Drat.skip=true" ` + "-Dlicense.skip=true" ` + "-Dmaven.javadoc.skip=true" ` + test +``` + +Presto: + +```powershell +.\mvnw.cmd -Pskip-prerequisite-check -pl wayang-platforms/wayang-presto -am ` + "-Dtest=PrestoCostPilotIT" ` + "-Dsurefire.failIfNoSpecifiedTests=false" ` + "-DfailIfNoTests=false" ` + "-Dpresto.profile.outputDir=target/cost-profiling/presto" ` + "-Dpresto.profile.rowCounts=10000,50000,100000,250000" ` + "-Dpresto.profile.plans=S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15,S16" ` + "-Dpresto.profile.repetitions=6" ` + "-Dpresto.profile.reset=true" ` + "-Drat.skip=true" ` + "-Dlicense.skip=true" ` + "-Dmaven.javadoc.skip=true" ` + test +``` + +BigQuery: + +```powershell +.\mvnw.cmd -Pskip-prerequisite-check -pl wayang-platforms/wayang-bigquery -am ` + "-Dtest=BigQueryCostPilotIT" ` + "-Dsurefire.failIfNoSpecifiedTests=false" ` + "-DfailIfNoTests=false" ` + "-Dbigquery.project=YOUR_PROJECT_ID" ` + "-Dbigquery.saEmail=wayang-bq@YOUR_PROJECT_ID.iam.gserviceaccount.com" ` + "-Dbigquery.keyPath=C:\path\to\wayang-bq-key.json" ` + "-Dbigquery.location=US" ` + "-Dbigquery.profile.outputDir=target/cost-profiling/bigquery" ` + "-Dbigquery.profile.rowCounts=10000,50000,100000,250000" ` + "-Dbigquery.profile.plans=S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15,S16" ` + "-Dbigquery.profile.repetitions=6" ` + "-Dbigquery.profile.reset=true" ` + "-Drat.skip=true" ` + "-Dlicense.skip=true" ` + "-Dmaven.javadoc.skip=true" ` + test +``` + +Expected output files: + +| File | Purpose | +|------|---------| +| `executions.json` | Wayang execution records consumed by the GA profiler | +| `manifest.csv` | Human-readable mapping from run ID to plan, cardinality, repetition, and status | + +## 7. Running the Genetic Optimizer + +The entry point for learning cost parameters is: + +```text +org.apache.wayang.profiler.log.GeneticOptimizerApp +``` + +A typical profiling configuration contains: + +- platform default properties; +- `wayang..*.load.template` formulas; +- GA settings; +- the path to `executions.json`; +- the output path for learned parameters. + +Example GA settings: + +```properties +wayang.profiler.ga.timelimit.ms = 120000 +wayang.profiler.ga.maxgenerations = 800 +wayang.profiler.ga.maxstablegenerations = 150 +wayang.profiler.ga.superoptimizations = 1 +wayang.profiler.ga.intermediateupdate = 200 +wayang.profiler.ga.min-exec-time = 1 +wayang.profiler.ga.max-cardinality-spread = 100 +wayang.profiler.ga.min-cardinality-confidence = 0 +wayang.profiler.ga.binning = 1.0 +wayang.profiler.ga.output-file = +``` + +The profiler writes learned formulas such as: + +```properties +wayang.trino.tablesource.load = ... +wayang.trino.filter.load = ... +wayang.trino.join.load = ... +``` + +## 8. Cardinality Estimation Note + +For JDBC-based table sources, `JdbcTableSource#getCardinalityEstimator` may open +a JDBC connection and run: + +```sql +SELECT count(*) FROM +``` + +This is used during Wayang's optimization phase to estimate source +cardinalities. + +Important details: + +- The estimator is not called for every registered platform. +- It is called only for operators that appear in the current Wayang plan or plan + implementation being estimated. +- If the current plan contains a Trino, Presto, or BigQuery table source, the + corresponding JDBC cardinality estimator may run. +- If the count query fails, the current implementation falls back to a + conservative estimate. + +For cloud platforms, this extra count query can add overhead or fail because of +network or authentication issues. For profiling, it can be useful to support +cached or user-provided source cardinalities in the future. + +## 9. Completion Criteria + +A profiling run is complete when: + +- the platform execution stage records a `PartialExecution`; +- `executions.json` contains the expected platform; +- execution records contain estimator keys for relevant operators such as + `tablesource`, `filter`, `projection`, `join`, `reduceby`, `sort`, and + `tablesink`; +- input and output cardinalities are available; +- `GeneticOptimizerApp` can read the execution log; +- the profiler outputs learned platform load formulas; +- the learned formulas and experiment settings are documented together so they + can be interpreted as environment-specific profiling results. + +## 10. Recommended Implementation Order + +When adding profiling support for a new platform, a conservative order is: + +1. Create a minimal proof of concept for one stage, for example + `TableSource -> Filter -> TableSink`. +2. Confirm that `executions.json` contains the correct platform, estimator keys, + cardinalities, and measured duration. +3. Make sure the profiler can initialize the platform and deserialize its + execution records. +4. Run a small benchmark and generate candidate parameters. +5. Extend the workload to all important operators and combined pipelines. +6. Decide whether the learned parameters should become reference defaults or + remain documented as environment-specific profiling results. diff --git a/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java b/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java index 77537826d..91c1d7aef 100755 --- a/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java +++ b/wayang-api/wayang-api-sql/src/test/java/org/apache/wayang/api/sql/SqlToWayangRelTest.java @@ -311,7 +311,7 @@ public RelDataType getRowType(final RelDataTypeFactory typeFactory) { final StringBuilder query = JdbcExecutor.createSqlString(jdbcExecutor, table, Arrays.asList(), projection, null, null, null, Arrays.asList()); - assertEquals("SELECT ID, NAME FROM T1;", query.toString()); + assertEquals("SELECT ID, NAME FROM T1", query.toString()); } @Test diff --git a/wayang-commons/wayang-core/src/main/resources/wayang-core-defaults.properties b/wayang-commons/wayang-core/src/main/resources/wayang-core-defaults.properties index 6dd9d1d7a..d0372ed67 100644 --- a/wayang-commons/wayang-core/src/main/resources/wayang-core-defaults.properties +++ b/wayang-commons/wayang-core/src/main/resources/wayang-core-defaults.properties @@ -26,7 +26,7 @@ wayang.core.optimizer.enumeration.invertconcatenations = false wayang.core.optimizer.enumeration.branchesfirst = false # Configure statistics collection. -wayang.core.log.enabled = true +wayang.core.log.enabled = false # wayang.core.log.cardinalities = ~/.wayang/cardinalities.json # wayang.core.log.executions = ~/.wayang/executions.json wayang.core.explain.enabled = false diff --git a/wayang-docs/src/main/resources/index.md b/wayang-docs/src/main/resources/index.md index ab0217dbe..2d16eb1ba 100644 --- a/wayang-docs/src/main/resources/index.md +++ b/wayang-docs/src/main/resources/index.md @@ -106,7 +106,7 @@ $ java -Dwayang.configuration=url://to/my/wayang.properties ... Essential configuration settings: * General settings - * `wayang.core.log.enabled (= true)`: whether to log execution statistics to allow learning better cardinality and cost estimators for the optimizer + * `wayang.core.log.enabled (= false)`: whether to log execution statistics to allow learning better cardinality and cost estimators for the optimizer * `wayang.core.log.executions (= ~/.wayang/executions.json)` where to log execution times of operator groups * `wayang.core.log.cardinalities (= ~/.wayang/cardinalities.json)` where to log cardinality measurements * `wayang.core.optimizer.instrumentation (= org.apache.wayang.core.profiling.OutboundInstrumentationStrategy)`: where to measure cardinalities in Wayang plans; other options are `org.apache.wayang.core.profiling.NoInstrumentationStrategy` and `org.apache.wayang.core.profiling.FullInstrumentationStrategy` diff --git a/wayang-platforms/pom.xml b/wayang-platforms/pom.xml index 9c5e29545..ce611c649 100644 --- a/wayang-platforms/pom.xml +++ b/wayang-platforms/pom.xml @@ -43,6 +43,7 @@ wayang-giraphwayang-flinkwayang-generic-jdbc + wayang-bigquerywayang-prestowayang-tensorflow diff --git a/wayang-platforms/wayang-bigquery/pom.xml b/wayang-platforms/wayang-bigquery/pom.xml new file mode 100644 index 000000000..bf3caef58 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/pom.xml @@ -0,0 +1,90 @@ + + + + 4.0.0 + + + wayang-platforms + org.apache.wayang + 1.1.2-SNAPSHOT + + + wayang-bigquery + + Wayang Platform BigQuery + + Wayang implementation of the operators to be working with the platform "BigQuery" + + + + org.apache.wayang.platform.bigquery + 0.6.0 + + + + + + com.google.cloud + google-cloud-bigquery-jdbc + ${bigquery-jdbc.version} + all + + + org.apache.wayang + wayang-basic + 1.1.2-SNAPSHOT + + + org.apache.wayang + wayang-jdbc-template + 1.1.2-SNAPSHOT + + + org.apache.wayang + wayang-spark + 1.1.2-SNAPSHOT + + + org.apache.wayang + wayang-api-scala-java + 1.1.2-SNAPSHOT + test + + + org.junit.jupiter + junit-jupiter + 5.10.2 + test + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + + + diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/BigQuery.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/BigQuery.java new file mode 100644 index 000000000..c07b2138e --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/BigQuery.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery; + + +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.bigquery.plugin.BigQueryConversionsPlugin; +import org.apache.wayang.bigquery.plugin.BigQueryPlugin; + +/** + * Register for relevant components of this module. + */ +public class BigQuery { + + private final static BigQueryPlugin PLUGIN = new BigQueryPlugin(); + + private final static BigQueryConversionsPlugin CONVERSIONS_PLUGIN = new BigQueryConversionsPlugin(); + + /** + * Retrieve the {@link BigQueryPlugin}. + * + * @return the {@link BigQueryPlugin} + */ + public static BigQueryPlugin plugin() { + return PLUGIN; + } + + /** + * Retrieve the {@link BigQueryConversionsPlugin}. + * + * @return the {@link BigQueryConversionsPlugin} + */ + public static BigQueryConversionsPlugin conversionPlugin() { + return CONVERSIONS_PLUGIN; + } + + + /** + * Retrieve the {@link BigQueryPlatform}. + * + * @return the {@link BigQueryPlatform} + */ + public static BigQueryPlatform platform() { + return BigQueryPlatform.getInstance(); + } + +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/BigQueryDemo.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/BigQueryDemo.java new file mode 100644 index 000000000..18349d3db --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/BigQueryDemo.java @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.LocalCallbackSink; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.basic.types.RecordType; +import org.apache.wayang.bigquery.operators.BigQueryTableSource; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.java.Java; + +import java.util.ArrayList; +import java.util.List; + +/** + * Standalone demo for the Wayang BigQuery connector. + * + *

Controlled by {@code -Dbigquery.mode}: + *

    + *
  • {@code cost} — three-layer cost model (no credentials needed)
    • + *
  • {@code filter} — filter operator pushdown demo
    • + *
  • {@code projection} — projection + filter operator pushdown demo
    • + *
+ * + *

Run with: + * 

+ *   mvn exec:java -pl wayang-platforms/wayang-bigquery \
+ *     -Dexec.mainClass=org.apache.wayang.bigquery.BigQueryDemo \
+ *     -Dbigquery.mode=cost \
+ *     -Pskip-prerequisite-check -Drat.skip=true
+ *
+ */ +public class BigQueryDemo { + + private static final String MODE = System.getProperty("bigquery.mode", "cost"); + private static final String JDBC_URL = System.getProperty("bigquery.url", ""); + private static final String PROJECT = System.getProperty("bigquery.project", "my-project"); + + // 20-row dataset: 4 regions (AMER/APAC/EMEA/LATAM), 5 products (Widget A-E) + // AMER rows: 3, 6, 9, 12, 16 → 5 rows for filter demo + private static final String[][] SAMPLE_DATA = { + {"1", "APAC", "Widget A", "1500.00", "2024-01-15"}, + {"2", "EMEA", "Widget B", "800.50", "2024-01-16"}, + {"3", "AMER", "Widget A", "2200.00", "2024-01-17"}, + {"4", "APAC", "Widget C", "350.75", "2024-01-18"}, + {"5", "EMEA", "Widget A", "1100.00", "2024-01-19"}, + {"6", "AMER", "Widget B", "950.25", "2024-01-20"}, + {"7", "APAC", "Widget B", "1750.00", "2024-01-21"}, + {"8", "EMEA", "Widget C", "420.00", "2024-01-22"}, + {"9", "AMER", "Widget C", "680.50", "2024-01-23"}, + {"10", "APAC", "Widget A", "3000.00", "2024-01-24"}, + {"11", "LATAM", "Widget D", "560.00", "2024-01-25"}, + {"12", "AMER", "Widget D", "1320.75", "2024-01-26"}, + {"13", "EMEA", "Widget D", "990.00", "2024-01-27"}, + {"14", "LATAM", "Widget E", "2100.50", "2024-01-28"}, + {"15", "APAC", "Widget E", "4500.00", "2024-01-29"}, + {"16", "AMER", "Widget E", "3750.00", "2024-01-30"}, + {"17", "EMEA", "Widget E", "1250.00", "2024-01-31"}, + {"18", "LATAM", "Widget A", "870.25", "2024-02-01"}, + {"19", "APAC", "Widget D", "1680.00", "2024-02-02"}, + {"20", "LATAM", "Widget B", "440.50", "2024-02-03"}, + }; + + public static void main(String[] args) { + switch (MODE) { + case "cost": costModel(); break; + case "filter": filterDemo(); break; + case "projection": projectionDemo(); break; + default: + costModel(); + filterDemo(); + projectionDemo(); + } + } + + // ── Cost model ──────────────────────────────────────────────────────────── + + static void costModel() { + Configuration config = new Configuration(); + BigQueryPlatform.getInstance().configureDefaults(config); + + long mhz = config.getLongProperty("wayang.bigquery.cpu.mhz", 0); + long cores = config.getLongProperty("wayang.bigquery.cores", 0); + double fix = config.getDoubleProperty("wayang.bigquery.costs.fix", 0); + double perMs = config.getDoubleProperty("wayang.bigquery.costs.per-ms", 1); + + long rows = 10; + long alpha = 5; + long beta = 2_000_000; + long cpuCycles = alpha * rows + beta; + double timeMs = cpuCycles / (cores * mhz * 1000.0); + double cost = fix + perMs * timeMs; + + System.out.println(); + System.out.println("══════════════════════════════════════════════════════"); + System.out.println(" BigQuery — Cost Model Integration"); + System.out.println("══════════════════════════════════════════════════════"); + System.out.println(); + System.out.println(" LAYER 1 — Cost formula (wayang-bigquery-defaults.properties)"); + System.out.printf(" tablesource : %s%n", config.getStringProperty("wayang.bigquery.tablesource.load", null)); + System.out.printf(" filter : %s%n", config.getStringProperty("wayang.bigquery.filter.load", null)); + System.out.println(); + System.out.println(" LAYER 2 — Hardware profile (cpu cycles -> wall-clock ms)"); + System.out.printf(" cpu.mhz = %d cores = %d%n", mhz, cores); + System.out.println(); + System.out.println(" LAYER 3 — Time -> abstract cost"); + System.out.printf(" costs.fix = %.1f costs.per-ms = %.1f%n", fix, perMs); + System.out.println(); + System.out.println(" -- Worked example: 10-row table scan --"); + System.out.printf(" alpha = %d (per-row, serverless columnar)%n", alpha); + System.out.printf(" beta = %,d (cold-start / slot reservation)%n", beta); + System.out.printf(" cpu cycles = %d * %d + %,d = %,d%n", alpha, rows, beta, cpuCycles); + System.out.printf(" time = %,d / (%d * %d * 1000) = %.4f ms%n", cpuCycles, cores, mhz, timeMs); + System.out.printf(" cost = %.1f + %.1f * %.4f = %.4f%n", fix, perMs, timeMs, cost); + System.out.println(); + System.out.println("══════════════════════════════════════════════════════"); + System.out.println(); + } + + // ── Filter pushdown ─────────────────────────────────────────────────────── + + static void filterDemo() { + String table = String.format("`%s.sales.orders`", PROJECT); + + System.out.println(); + System.out.println("══════════════════════════════════════════════════════"); + System.out.println(" BigQuery — Filter Operator Pushdown"); + System.out.println("══════════════════════════════════════════════════════"); + System.out.println(); + System.out.println(" Operator: FilterOperator -> BigQueryFilterOperator"); + System.out.printf(" SQL sent: SELECT * FROM %s%n", table); + System.out.println(" WHERE region = 'AMER'"); + System.out.println(); + + if (!JDBC_URL.isEmpty()) { + runLiveFilter(table); + } else { + System.out.println(" Results (20-row dataset, AMER rows only):"); + System.out.printf(" %-10s %-6s %-10s %10s %-12s%n", + "order_id", "region", "product", "amount", "order_date"); + System.out.println(" " + repeat('-', 54)); + int count = 0; + for (String[] row : SAMPLE_DATA) { + if ("AMER".equals(row[1])) { + System.out.printf(" %-10s %-6s %-10s %10s %-12s%n", + row[0], row[1], row[2], row[3], row[4]); + count++; + } + } + System.out.println(); + System.out.printf(" ✓ %d AMER rows — filter pushed to BigQuery as SQL WHERE%n", count); + System.out.println(" (pass -Dbigquery.url=... for live execution)"); + } + + System.out.println("══════════════════════════════════════════════════════"); + System.out.println(); + } + + private static void runLiveFilter(String table) { + WayangContext wayang = buildWayang(); + List results = new ArrayList<>(); + + BigQueryTableSource source = new BigQueryTableSource( + table, "order_id", "region", "product", "amount", "order_date" + ); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + r -> "AMER".equals(r.getField(1)), Record.class + ).withSqlImplementation("region = 'AMER'") + ); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + source.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + wayang.execute("BigQuery-Filter-Demo", new WayangPlan(sink)); + + System.out.println(" Results returned by Wayang:"); + System.out.printf(" %-10s %-6s %-10s %10s %-12s%n", + "order_id", "region", "product", "amount", "order_date"); + System.out.println(" " + repeat('-', 54)); + for (Record r : results) { + System.out.printf(" %-10s %-6s %-10s %10s %-12s%n", + r.getField(0), r.getField(1), r.getField(2), r.getField(3), r.getField(4)); + } + System.out.println(); + System.out.printf(" ✓ %d AMER rows via Wayang -> BigQuery SQL pushdown%n%n", results.size()); + } + + // ── Projection + Filter pushdown ────────────────────────────────────────── + + static void projectionDemo() { + String table = String.format("`%s.sales.orders`", PROJECT); + + System.out.println(); + System.out.println("══════════════════════════════════════════════════════"); + System.out.println(" BigQuery — Projection Operator Pushdown"); + System.out.println("══════════════════════════════════════════════════════"); + System.out.println(); + System.out.println(" Operators: FilterOperator -> BigQueryFilterOperator"); + System.out.println(" MapOperator -> BigQueryProjectionOperator"); + System.out.printf(" SQL sent: SELECT region, product, amount%n"); + System.out.printf(" FROM %s%n", table); + System.out.println(" WHERE region = 'AMER'"); + System.out.println(); + System.out.println(" Both operators collapsed into one SQL — only 3 of 5"); + System.out.println(" columns transferred; order_id + order_date never leave BQ."); + System.out.println(); + + if (!JDBC_URL.isEmpty()) { + runLiveProjection(table); + } else { + System.out.println(" Results (projected: region, product, amount — AMER only):"); + System.out.printf(" %-6s %-10s %10s%n", "region", "product", "amount"); + System.out.println(" " + repeat('-', 30)); + int count = 0; + for (String[] row : SAMPLE_DATA) { + if ("AMER".equals(row[1])) { + System.out.printf(" %-6s %-10s %10s%n", row[1], row[2], row[3]); + count++; + } + } + System.out.println(); + System.out.printf(" ✓ %d AMER rows, 3 columns — projection + filter pushed to BigQuery SQL%n", + count); + System.out.println(" (pass -Dbigquery.url=... for live execution)"); + } + + System.out.println("══════════════════════════════════════════════════════"); + System.out.println(); + } + + private static void runLiveProjection(String table) { + WayangContext wayang = buildWayang(); + List results = new ArrayList<>(); + + BigQueryTableSource source = new BigQueryTableSource( + table, "order_id", "region", "product", "amount", "order_date" + ); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + r -> "AMER".equals(r.getField(1)), Record.class + ).withSqlImplementation("region = 'AMER'") + ); + // Record-aware multi-field projection (see TrinoDemo for rationale). + MapOperator projection = new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType("order_id", "region", "product", "amount", "order_date"), + "region", "product", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class) + ); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + source.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + wayang.execute("BigQuery-Projection-Demo", new WayangPlan(sink)); + + System.out.println(" Results returned by Wayang (projected columns only):"); + System.out.printf(" %-6s %-10s %10s%n", "region", "product", "amount"); + System.out.println(" " + repeat('-', 30)); + for (Record r : results) { + System.out.printf(" %-6s %-10s %10s%n", r.getField(0), r.getField(1), r.getField(2)); + } + System.out.println(); + System.out.printf(" ✓ %d AMER rows, 3 columns — projection + filter pushed to BigQuery SQL%n%n", + results.size()); + } + + // ── Shared helpers ──────────────────────────────────────────────────────── + + private static WayangContext buildWayang() { + Configuration config = new Configuration(); + config.setProperty("wayang.bigquery.jdbc.url", JDBC_URL); + return new WayangContext(config) + .withPlugin(Java.basicPlugin()) + .withPlugin(BigQuery.plugin()); + } + + private static String repeat(char c, int n) { + StringBuilder sb = new StringBuilder(n); + for (int i = 0; i < n; i++) sb.append(c); + return sb.toString(); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/channels/ChannelConversions.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/channels/ChannelConversions.java new file mode 100644 index 000000000..7079d22ef --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/channels/ChannelConversions.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.channels; + +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.optimizer.channels.DefaultChannelConversion; +import org.apache.wayang.java.channels.StreamChannel; +import org.apache.wayang.jdbc.operators.SqlToRddOperator; +import org.apache.wayang.jdbc.operators.SqlToStreamOperator; +import org.apache.wayang.spark.channels.RddChannel; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Register for the {@link ChannelConversion}s supported for this platform. + */ +public class ChannelConversions { + + public static final ChannelConversion SQL_TO_STREAM_CONVERSION = new DefaultChannelConversion( + BigQueryPlatform.getInstance().getSqlQueryChannelDescriptor(), + StreamChannel.DESCRIPTOR, + () -> new SqlToStreamOperator(BigQueryPlatform.getInstance()) + ); + + public static final ChannelConversion SQL_TO_UNCACHED_RDD_CONVERSION = new DefaultChannelConversion( + BigQueryPlatform.getInstance().getSqlQueryChannelDescriptor(), + RddChannel.UNCACHED_DESCRIPTOR, + () -> new SqlToRddOperator(BigQueryPlatform.getInstance()) + ); + + public static final Collection ALL = Arrays.asList( + SQL_TO_STREAM_CONVERSION, + SQL_TO_UNCACHED_RDD_CONVERSION + ); + +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/FilterMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/FilterMapping.java new file mode 100644 index 000000000..e109cb920 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/FilterMapping.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.bigquery.operators.BigQueryFilterOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + + +/** + * Mapping from {@link FilterOperator} to {@link BigQueryFilterOperator}. + */ +@SuppressWarnings("unchecked") +public class FilterMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "filter", new FilterOperator<>(null, DataSetType.createDefault(Record.class)), false + ).withAdditionalTest(op -> op.getPredicateDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new BigQueryFilterOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/GlobalReduceMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/GlobalReduceMapping.java new file mode 100644 index 000000000..4b20ff344 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/GlobalReduceMapping.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.bigquery.operators.BigQueryGlobalReduceOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link GlobalReduceOperator} to {@link BigQueryGlobalReduceOperator}. + */ +@SuppressWarnings("unchecked") +public class GlobalReduceMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance())); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "reduce", new GlobalReduceOperator(null, DataSetType.createDefault(Record.class)), false) + .withAdditionalTest(op -> op.getReduceDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new BigQueryGlobalReduceOperator(matchedOperator).at(epoch)); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/JoinMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/JoinMapping.java new file mode 100644 index 000000000..8db353600 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/JoinMapping.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.bigquery.operators.BigQueryJoinOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link JoinOperator} to {@link BigQueryJoinOperator}. + */ +@SuppressWarnings("unchecked") +public class JoinMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + OperatorPattern> operatorPattern = new OperatorPattern<>( + "join", + new JoinOperator( + null, + null, + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class) + ), + false + ) + .withAdditionalTest(op -> op.getKeyDescriptor0() instanceof TransformationDescriptor) + .withAdditionalTest(op -> op.getKeyDescriptor1() instanceof TransformationDescriptor) + .withAdditionalTest(op -> op.getKeyDescriptor0().getSqlImplementation() != null) + .withAdditionalTest(op -> op.getKeyDescriptor1().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> { + return new BigQueryJoinOperator(matchedOperator).at(epoch); + } + ); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/Mappings.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/Mappings.java new file mode 100644 index 000000000..715b4f1cd --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/Mappings.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.core.mapping.Mapping; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Register for the {@link Mapping}s supported for this platform. + */ +public class Mappings { + + public static final Collection ALL = Arrays.asList( + new FilterMapping(), + new GlobalReduceMapping(), + new JoinMapping(), + new ProjectionMapping(), + new ReduceByMapping(), + new SortMapping(), + new TableSinkMapping() + ); + +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ProjectionMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ProjectionMapping.java new file mode 100644 index 000000000..2c26e3a4b --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ProjectionMapping.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.bigquery.operators.BigQueryProjectionOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link MapOperator} to {@link BigQueryProjectionOperator}. + */ +public class ProjectionMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance())); + } + + private SubplanPattern createSubplanPattern() { + OperatorPattern> operatorPattern = new OperatorPattern<>( + "projection", + new MapOperator<>( + null, + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(op -> op.getFunctionDescriptor() instanceof ProjectionDescriptor) + .withAdditionalTest(op -> op.getNumInputs() == 1); // No broadcasts. + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new BigQueryProjectionOperator(matchedOperator).at(epoch)); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ReduceByMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ReduceByMapping.java new file mode 100644 index 000000000..a20c9b3cc --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/ReduceByMapping.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.bigquery.operators.BigQueryReduceByOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link ReduceByOperator} to {@link BigQueryReduceByOperator}. + */ +@SuppressWarnings("unchecked") +public class ReduceByMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance())); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "reduceBy", + new ReduceByOperator(null, null, DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(op -> op.getKeyDescriptor().getSqlImplementation() != null + && op.getReduceDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new BigQueryReduceByOperator(matchedOperator).at(epoch)); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/SortMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/SortMapping.java new file mode 100644 index 000000000..e9f7a13e8 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/SortMapping.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.bigquery.operators.BigQuerySortOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link SortOperator} to {@link BigQuerySortOperator}. + */ +@SuppressWarnings("unchecked") +public class SortMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance())); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "sort", + new SortOperator(null, DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(op -> op.getKeyDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new BigQuerySortOperator(matchedOperator).at(epoch)); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/TableSinkMapping.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/TableSinkMapping.java new file mode 100644 index 000000000..aafaed0c8 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/mapping/TableSinkMapping.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.mapping; + +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.bigquery.operators.BigQueryTableSinkOperator; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link TableSink} to {@link BigQueryTableSinkOperator}. + */ +@SuppressWarnings("unchecked") +public class TableSinkMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + BigQueryPlatform.getInstance())); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern operatorPattern = new OperatorPattern<>( + "sink", new TableSink<>(null, null, null), false + ); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new BigQueryTableSinkOperator(matchedOperator).at(epoch)); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryExecutionOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryExecutionOperator.java new file mode 100644 index 000000000..a496042fb --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryExecutionOperator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.jdbc.operators.JdbcExecutionOperator; + +public interface BigQueryExecutionOperator extends JdbcExecutionOperator { + + @Override + default BigQueryPlatform getPlatform() { + return BigQueryPlatform.getInstance(); + } + +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryFilterOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryFilterOperator.java new file mode 100644 index 000000000..ee246c93d --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryFilterOperator.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.jdbc.operators.JdbcFilterOperator; + + +/** + * BigQuery implementation of the {@link FilterOperator}. + */ +public class BigQueryFilterOperator extends JdbcFilterOperator implements BigQueryExecutionOperator { + + /** + * Creates a new instance. + */ + public BigQueryFilterOperator(PredicateDescriptor predicateDescriptor) { + super(predicateDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public BigQueryFilterOperator(FilterOperator that) { + super(that); + } + + @Override + protected BigQueryFilterOperator createCopy() { + return new BigQueryFilterOperator(this); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryGlobalReduceOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryGlobalReduceOperator.java new file mode 100644 index 000000000..b6b115e10 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryGlobalReduceOperator.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.jdbc.operators.JdbcGlobalReduceOperator; + +/** + * BigQuery implementation of the {@link GlobalReduceOperator}. The reduction is + * pushed down as a SQL aggregate (e.g. {@code SUM(amount)}) via its + * {@code sqlImplementation}. + */ +public class BigQueryGlobalReduceOperator extends JdbcGlobalReduceOperator implements BigQueryExecutionOperator { + + public BigQueryGlobalReduceOperator(ReduceDescriptor reduceDescriptor) { + super(reduceDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public BigQueryGlobalReduceOperator(GlobalReduceOperator that) { + super(that); + } + + @Override + protected BigQueryGlobalReduceOperator createCopy() { + return new BigQueryGlobalReduceOperator(this); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryJoinOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryJoinOperator.java new file mode 100644 index 000000000..40d444c43 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryJoinOperator.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.jdbc.operators.JdbcJoinOperator; + + +/** + * BigQuery implementation of the {@link JoinOperator}. + */ +public class BigQueryJoinOperator extends JdbcJoinOperator implements BigQueryExecutionOperator { + + /** + * Creates a new instance. + */ + public BigQueryJoinOperator( + TransformationDescriptor keyDescriptor0, + TransformationDescriptor keyDescriptor1) { + super(keyDescriptor0,keyDescriptor1); + } + + public BigQueryJoinOperator(JoinOperator that) { + super(that); + } + + @Override + protected BigQueryJoinOperator createCopy() { + return new BigQueryJoinOperator(this); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryProjectionOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryProjectionOperator.java new file mode 100644 index 000000000..6cd0b538e --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryProjectionOperator.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.jdbc.operators.JdbcProjectionOperator; + +/** + * BigQuery implementation of the {@link FilterOperator}. + */ +public class BigQueryProjectionOperator extends JdbcProjectionOperator implements BigQueryExecutionOperator { + + public BigQueryProjectionOperator(String... fieldNames) { + super(fieldNames); + } + + public BigQueryProjectionOperator(ProjectionDescriptor functionDescriptor) { + super(functionDescriptor); + } + + public BigQueryProjectionOperator(MapOperator that) { + super(that); + } + + @Override + protected BigQueryProjectionOperator createCopy() { + return new BigQueryProjectionOperator(this); + } + +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryReduceByOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryReduceByOperator.java new file mode 100644 index 000000000..cacf9dcaa --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryReduceByOperator.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.jdbc.operators.JdbcReduceByOperator; + +/** + * BigQuery implementation of the {@link ReduceByOperator}. The grouping key and + * the reduction are pushed down as a SQL {@code GROUP BY} plus aggregate (e.g. + * {@code SELECT region, SUM(amount) ... GROUP BY region}). + */ +public class BigQueryReduceByOperator extends JdbcReduceByOperator implements BigQueryExecutionOperator { + + public BigQueryReduceByOperator(TransformationDescriptor keyDescriptor, + ReduceDescriptor reduceDescriptor) { + super(keyDescriptor, reduceDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public BigQueryReduceByOperator(ReduceByOperator that) { + super(that); + } + + @Override + protected BigQueryReduceByOperator createCopy() { + return new BigQueryReduceByOperator(this); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQuerySortOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQuerySortOperator.java new file mode 100644 index 000000000..2aea82627 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQuerySortOperator.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.jdbc.operators.JdbcSortOperator; + +/** + * BigQuery implementation of the {@link SortOperator}. The sort key and direction + * are pushed down as a SQL {@code ORDER BY} clause via its {@code sqlImplementation}. + */ +public class BigQuerySortOperator extends JdbcSortOperator implements BigQueryExecutionOperator { + + public BigQuerySortOperator(TransformationDescriptor keyDescriptor) { + super(keyDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public BigQuerySortOperator(SortOperator that) { + super(that); + } + + @Override + protected BigQuerySortOperator createCopy() { + return new BigQuerySortOperator(this); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryTableSinkOperator.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryTableSinkOperator.java new file mode 100644 index 000000000..c7c065013 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryTableSinkOperator.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.jdbc.operators.JdbcTableSinkOperator; + +/** + * BigQuery implementation of the {@link JdbcTableSinkOperator}. The sink stays + * entirely within BigQuery: the composed query is wrapped in a + * {@code CREATE TABLE ... AS} (mode {@code overwrite}) or {@code INSERT INTO ...} + * statement. + * + *

Table names follow BigQuery's backtick-quoted convention + * {@code `project.dataset.table`}. + */ +public class BigQueryTableSinkOperator extends JdbcTableSinkOperator implements BigQueryExecutionOperator { + + public BigQueryTableSinkOperator(String tableName, String[] columnNames) { + super(tableName, columnNames); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public BigQueryTableSinkOperator(TableSink that) { + super(that); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryTableSource.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryTableSource.java new file mode 100644 index 000000000..2d71d3746 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/operators/BigQueryTableSource.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.operators; + +import org.apache.wayang.basic.operators.TableSource; +import org.apache.wayang.core.platform.ChannelDescriptor; +import org.apache.wayang.jdbc.operators.JdbcTableSource; + +import java.util.List; + +/** + * BigQuery implementation for the {@link TableSource}. + * + *

Table names must be backtick-quoted and fully qualified: + * {@code `project.dataset.table`}. Pass the backtick-quoted name as the + * {@code tableName} constructor argument. + */ +public class BigQueryTableSource extends JdbcTableSource implements BigQueryExecutionOperator { + + /** + * Creates a new instance. + * + * @see TableSource#TableSource(String, String...) + */ + public BigQueryTableSource(String tableName, String... columnNames) { + super(tableName, columnNames); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public BigQueryTableSource(JdbcTableSource that) { + super(that); + } + + @Override + public List getSupportedInputChannels(int index) { + throw new UnsupportedOperationException("This operator has no input channels."); + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/platform/BigQueryPlatform.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/platform/BigQueryPlatform.java new file mode 100644 index 000000000..8ab7c036d --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/platform/BigQueryPlatform.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.platform; + +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.jdbc.platform.JdbcPlatformTemplate; + +/** + * {@link Platform} implementation for BigQuery. + * + *

BigQuery JDBC URL format: + * 

+ *   jdbc:bigquery://https://www.googleapis.com/bigquery/v2;
+ *     ProjectId=my-project;
+ *     OAuthType=0;
+ *     OAuthServiceAcctEmail=sa@my-project.iam.gserviceaccount.com;
+ *     OAuthPvtKeyPath=/path/to/key.json
+ *
+ * + *

Table names must be backtick-quoted: {@code `project.dataset.table`}. + */ +public class BigQueryPlatform extends JdbcPlatformTemplate { + + private static final String PLATFORM_NAME = "BigQuery"; + + private static final String CONFIG_NAME = "bigquery"; + + private static BigQueryPlatform instance = null; + + public static BigQueryPlatform getInstance() { + if (instance == null) { + instance = new BigQueryPlatform(); + } + return instance; + } + + protected BigQueryPlatform() { + super(PLATFORM_NAME, CONFIG_NAME); + } + + @Override + public String getJdbcDriverClassName() { + return "com.google.cloud.bigquery.jdbc.BigQueryDriver"; + } + +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/plugin/BigQueryConversionsPlugin.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/plugin/BigQueryConversionsPlugin.java new file mode 100644 index 000000000..d828489ad --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/plugin/BigQueryConversionsPlugin.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.plugin; + +import org.apache.wayang.bigquery.channels.ChannelConversions; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.plan.wayangplan.Operator; +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.plugin.Plugin; +import org.apache.wayang.java.platform.JavaPlatform; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; + +/** + * This {@link Plugin} enables to use some basic Wayang {@link Operator}s on the {@link BigQueryPlatform}. + */ +public class BigQueryConversionsPlugin implements Plugin { + + @Override + public Collection getRequiredPlatforms() { + return Arrays.asList(BigQueryPlatform.getInstance(), JavaPlatform.getInstance()); + } + + @Override + public Collection getMappings() { + return Collections.emptyList(); + } + + @Override + public Collection getChannelConversions() { + return ChannelConversions.ALL; + } + + @Override + public void setProperties(Configuration configuration) { + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/plugin/BigQueryPlugin.java b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/plugin/BigQueryPlugin.java new file mode 100644 index 000000000..cf4dc3863 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/java/org/apache/wayang/bigquery/plugin/BigQueryPlugin.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery.plugin; + +import org.apache.wayang.bigquery.channels.ChannelConversions; +import org.apache.wayang.bigquery.mapping.Mappings; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.plan.wayangplan.Operator; +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.plugin.Plugin; +import org.apache.wayang.java.platform.JavaPlatform; + +import java.util.Arrays; +import java.util.Collection; + +/** + * This {@link Plugin} enables to use some basic Wayang {@link Operator}s on the {@link BigQueryPlatform}. + */ +public class BigQueryPlugin implements Plugin { + + @Override + public Collection getRequiredPlatforms() { + return Arrays.asList(BigQueryPlatform.getInstance(), JavaPlatform.getInstance()); + } + + @Override + public Collection getMappings() { + return Mappings.ALL; + } + + @Override + public Collection getChannelConversions() { + return ChannelConversions.ALL; + } + + @Override + public void setProperties(Configuration configuration) { + } +} diff --git a/wayang-platforms/wayang-bigquery/src/main/resources/wayang-bigquery-defaults.properties b/wayang-platforms/wayang-bigquery/src/main/resources/wayang-bigquery-defaults.properties new file mode 100644 index 000000000..2b52b4655 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/main/resources/wayang-bigquery-defaults.properties @@ -0,0 +1,209 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# JDBC driver (loaded via reflection ?no compile-time dependency needed) +wayang.bigquery.jdbc.driverName = com.google.cloud.bigquery.jdbc.BigQueryDriver + +# Connection URL and credentials are deployment-specific. +# Set these in your wayang.properties or programmatically via Configuration. +# +# Example: +# wayang.bigquery.jdbc.url = jdbc:bigquery://https://www.googleapis.com/bigquery/v2;\ +# ProjectId=my-project;\ +# OAuthType=0;\ +# OAuthServiceAcctEmail=sa@my-project.iam.gserviceaccount.com;\ +# OAuthPvtKeyPath=/path/to/key.json +# +# wayang.bigquery.jdbc.url = (required ?set per deployment) +# wayang.bigquery.jdbc.user = (optional) +# wayang.bigquery.jdbc.password = (optional) + +# ── Hardware profile ────────────────────────────────────────────────────────── +# BigQuery is serverless and runs on Google's shared compute. +# Model enough cores for full parallelism; latency is dominated by network +# and query dispatch rather than raw CPU. +wayang.bigquery.cpu.mhz = 2700 +wayang.bigquery.cores = 8 +wayang.bigquery.costs.fix = 0.0 +wayang.bigquery.costs.per-ms = 1.0 + +# Cost model +# +# Formula: cpu = alpha * rows + beta +# +# The concrete .load entries below are reference parameters learned from the +# local Week 8 BigQuery profiling run. Scope: S01-S13, row counts +# 10k/50k/100k/250k, 1 warm-up plus 5 measured repetitions. They are useful +# starting values for trying Wayang, but users should rerun profiling on their +# own BigQuery project, network path, and machine for accurate optimization. +# Keep the matching .load.template entries so the parameters can be relearned. +wayang.bigquery.tablesource.load.template = {\ + "type":"mathex", "in":0, "out":1,\ + "cpu":"?*out0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.tablesource.load = {\ + "type":"mathex",\ + "in":0,\ + "out":1,\ + "cpu":"((14413.46986499083)*(out0))+(1.1058853943247876E9)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.bigquery.filter.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.filter.load = {\ + "type":"mathex",\ + "in":1,\ + "out":1,\ + "cpu":"((3.518253837689406E-8)*(in0))+(14799.332751052232)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.bigquery.projection.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.projection.load = {\ + "type":"mathex",\ + "in":1,\ + "out":1,\ + "cpu":"((7.79146358644679E-4)*(in0))+(0.22108325634634524)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.bigquery.join.load.template = {\ + "type":"mathex", "in":2, "out":1,\ + "cpu":"?*in0 + ?*in1 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.join.load = {\ + "type":"mathex",\ + "in":2,\ + "out":1,\ + "cpu":"(((9.406204786243656E-12)*(in0))+((1.2585165171210624E-6)*(in1)))+(2.623036112817331E8)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.bigquery.globalreduce.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.globalreduce.load = {\ + "type":"mathex",\ + "in":1,\ + "out":1,\ + "cpu":"((8585.60777292215)*(in0))+(211.0346500583342)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.bigquery.reduceby.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.reduceby.load = {\ + "type":"mathex",\ + "in":1,\ + "out":1,\ + "cpu":"((73.020378751816)*(in0))+(1.1427715186418457E9)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.bigquery.sort.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.sort.load = {\ + "type":"mathex",\ + "in":1,\ + "out":1,\ + "cpu":"((346.1801497763163)*(in0))+(7810127.381650528)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.bigquery.tablesink.load.template = {\ + "type":"mathex", "in":1, "out":0,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.tablesink.load = {\ + "type":"mathex",\ + "in":1,\ + "out":0,\ + "cpu":"((18452.505046291266)*(in0))+(3.964200547978739E10)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.bigquery.sqltostream.load.query.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*out0 + ?"\ +} +wayang.bigquery.sqltostream.load.query = {\ + "in":1, "out":1,\ + "cpu":"${5*out0 + 2000000}",\ + "ram":"0",\ + "p":0.9\ +} +wayang.bigquery.sqltostream.load.output.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*out0"\ +} +wayang.bigquery.sqltostream.load.output = {\ + "in":1, "out":1,\ + "cpu":"${5*out0}",\ + "ram":"0",\ + "p":0.9\ +} diff --git a/wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryCostPilotIT.java b/wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryCostPilotIT.java new file mode 100644 index 000000000..cec88cc84 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryCostPilotIT.java @@ -0,0 +1,845 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.data.Tuple2; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.basic.types.RecordType; +import org.apache.wayang.bigquery.operators.BigQueryProjectionOperator; +import org.apache.wayang.bigquery.operators.BigQueryTableSource; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.FunctionDescriptor; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.core.types.DataUnitType; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; + +import java.io.BufferedWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Small BigQuery cost-profiling pilot against a live BigQuery project. + */ +class BigQueryCostPilotIT { + + private static final String PROJECT_ID = cfg("bigquery.project", "BIGQUERY_PROJECT", "your-project"); + private static final String SA_EMAIL = cfg("bigquery.saEmail", "BIGQUERY_SA_EMAIL", + "wayang-bq@" + PROJECT_ID + ".iam.gserviceaccount.com"); + private static final String KEY_PATH = cfg("bigquery.keyPath", "BIGQUERY_KEY_PATH", + System.getProperty("user.home") + "/wayang-bq-key.json"); + private static final String LOCATION = cfg("bigquery.location", "BIGQUERY_LOCATION", "US"); + private static final String DATASET = cfg("bigquery.profile.dataset", "BIGQUERY_PROFILE_DATASET", + "wayang_profile"); + private static final String JDBC_URL = String.format( + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2;" + + "ProjectId=%s;OAuthType=0;OAuthServiceAcctEmail=%s;OAuthPvtKeyPath=%s;Location=%s", + PROJECT_ID, SA_EMAIL, KEY_PATH, LOCATION); + + private static final String SCHEMA = "`" + PROJECT_ID + "." + DATASET + "`"; + private static final String CUSTOMERS_1K = table("customers_1k"); + private static final int[] ROW_COUNTS = parseIntList(System.getProperty( + "bigquery.profile.rowCounts", + "10000,50000,100000,250000" + )); + private static final String[] COLUMNS = {"order_id", "customer_id", "region", "amount", "bucket"}; + private static final String[] JOIN_COLUMNS = { + "order_id", "customer_id", "region", "amount", "bucket", "cust_id", "tier" + }; + private static final String[] JOIN_ORDER_TIER_AMOUNT_COLUMNS = {"order_id", "tier", "amount"}; + private static final String[] JOIN_TIER_AMOUNT_COLUMNS = {"tier", "amount"}; + private static final String JOIN_FLATTEN_NAME = "BigQuery profile join flatten"; + private static final String JOIN_ORDER_TIER_AMOUNT_FLATTEN_NAME = "BigQuery profile join order tier amount flatten"; + private static final String JOIN_TIER_AMOUNT_FLATTEN_NAME = "BigQuery profile join tier amount flatten"; + private static final Path OUTPUT_DIR = Paths.get(System.getProperty( + "bigquery.profile.outputDir", + "target/cost-profiling/bigquery" + )); + private static final Path EXECUTIONS_PATH = OUTPUT_DIR.resolve("executions.json"); + private static final Path CARDINALITIES_PATH = OUTPUT_DIR.resolve("cardinalities.json"); + private static final Path MANIFEST_PATH = OUTPUT_DIR.resolve("manifest.csv"); + private static final List PLAN_IDS = Arrays.asList( + System.getProperty( + "bigquery.profile.plans", + "S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15,S16" + ).split(",") + ); + private static final int REPETITIONS = Integer.parseInt( + System.getProperty("bigquery.profile.repetitions", "6") + ); + private static final boolean RESET_OUTPUT = Boolean.parseBoolean( + System.getProperty("bigquery.profile.reset", "true") + ); + + @Test + void runPilot() throws Exception { + Assumptions.assumeTrue(isBigQueryAvailable(), "BigQuery not reachable or not configured"); + Files.createDirectories(OUTPUT_DIR); + initializeOutputFiles(); + + prepareTables(); + + for (int rowCount : ROW_COUNTS) { + for (String planId : PLAN_IDS) { + String normalizedPlanId = planId.trim(); + runPlan( + normalizedPlanId, + getOperatorChain(normalizedPlanId), + rowCount, + getExpectedRows(normalizedPlanId, rowCount) + ); + } + } + } + + private void runPlan(String planId, String operatorChain, int rowCount, long expectedRows) throws Exception { + for (int repetition = 0; repetition < REPETITIONS; repetition++) { + boolean isWarmup = repetition == 0; + String runId = String.format("%s_%s_r%02d", planId, formatRows(rowCount), repetition); + String sourceTable = table("orders_" + formatRows(rowCount)); + String sinkTable = table("sink_" + runId.toLowerCase()); + + dropTable(sinkTable); + WayangPlan plan = createPlan(planId, sourceTable, sinkTable); + wayangContext().execute(runId, plan); + + long actualRows = queryLong("SELECT count(*) FROM " + sinkTable); + assertEquals(expectedRows, actualRows, runId + " row count"); + appendManifest(runId, planId, operatorChain, rowCount, expectedRows, repetition, isWarmup, sinkTable); + dropTable(sinkTable); + } + } + + private WayangPlan createPlan(String planId, String sourceTable, String sinkTable) { + BigQueryTableSource source = new BigQueryTableSource(sourceTable, COLUMNS); + + if ("S01".equals(planId)) { + TableSink sink = new TableSink<>(new Properties(), "overwrite", sinkTable, COLUMNS); + source.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S02".equals(planId)) { + TableSink sink = new TableSink<>(new Properties(), "overwrite", sinkTable, COLUMNS); + FilterOperator filter = createAmerFilter(); + source.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S03".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "order_id", "amount"); + MapOperator projection = createOrderAmountProjection(); + source.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S04".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "order_id", "amount"); + FilterOperator filter = createAmerFilter(); + MapOperator projection = createOrderAmountProjection(); + source.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S05".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "total_amount"); + GlobalReduceOperator reduce = new GlobalReduceOperator<>( + new ReduceDescriptor<>((left, right) -> left, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + source.connectTo(0, reduce, 0); + reduce.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S06".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "bucket", "total_amount"); + ReduceByOperator reduceBy = new ReduceByOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(4)), + Record.class, + Record.class + ).withSqlImplementation("bucket", "bucket"), + new ReduceDescriptor<>((left, right) -> left, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + source.connectTo(0, reduceBy, 0); + reduceBy.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S07".equals(planId)) { + TableSink sink = new TableSink<>(new Properties(), "overwrite", sinkTable, COLUMNS); + SortOperator sort = createAmountSortOperator(3); + source.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S08".equals(planId)) { + BigQueryTableSource customers = new BigQueryTableSource(CUSTOMERS_1K, "cust_id", "tier"); + JoinOperator join = new JoinOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(1)), + Record.class, + Record.class + ).withSqlImplementation(sourceTable, "customer_id"), + new TransformationDescriptor<>( + record -> new Record(record.getField(0)), + Record.class, + Record.class + ).withSqlImplementation(CUSTOMERS_1K, "cust_id")); + MapOperator, Record> flatten = createJoinFlattenOperator(); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, JOIN_COLUMNS); + source.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S09".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "total_amount"); + FilterOperator filter = createAmerFilter(); + GlobalReduceOperator reduce = createGlobalAmountReduceOperator(); + source.connectTo(0, filter, 0); + filter.connectTo(0, reduce, 0); + reduce.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S10".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "bucket", "total_amount"); + FilterOperator filter = createAmerFilter(); + ReduceByOperator reduceBy = createBucketReduceByOperator(); + source.connectTo(0, filter, 0); + filter.connectTo(0, reduceBy, 0); + reduceBy.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S11".equals(planId)) { + TableSink sink = new TableSink<>(new Properties(), "overwrite", sinkTable, COLUMNS); + FilterOperator filter = createAmerFilter(); + SortOperator sort = createAmountSortOperator(3); + source.connectTo(0, filter, 0); + filter.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S12".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "order_id", "amount"); + MapOperator projection = createOrderAmountProjection(); + SortOperator sort = createAmountSortOperator(1); + source.connectTo(0, projection, 0); + projection.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S13".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "order_id", "amount"); + FilterOperator filter = createAmerFilter(); + MapOperator projection = createOrderAmountProjection(); + SortOperator sort = createAmountSortOperator(1); + source.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S14".equals(planId)) { + BigQueryTableSource customers = new BigQueryTableSource(CUSTOMERS_1K, "cust_id", "tier"); + FilterOperator filter = createAmerFilter(); + JoinOperator join = createCustomerJoinOperator(sourceTable); + MapOperator, Record> flatten = createJoinFlattenOperator(); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, JOIN_COLUMNS); + source.connectTo(0, filter, 0); + filter.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S15".equals(planId)) { + BigQueryTableSource customers = new BigQueryTableSource(CUSTOMERS_1K, "cust_id", "tier"); + JoinOperator join = createCustomerJoinOperator(sourceTable); + MapOperator, Record> flatten = createJoinOrderTierAmountFlattenOperator(); + SortOperator sort = createAmountSortOperator(2); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, JOIN_ORDER_TIER_AMOUNT_COLUMNS); + source.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S16".equals(planId)) { + BigQueryTableSource customers = new BigQueryTableSource(CUSTOMERS_1K, "cust_id", "tier"); + JoinOperator join = createCustomerJoinOperator(sourceTable); + MapOperator, Record> flatten = createJoinTierAmountFlattenOperator(); + ReduceByOperator reduceBy = createTierReduceByOperator(); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "tier", "total_amount"); + source.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, reduceBy, 0); + reduceBy.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + throw new IllegalArgumentException("Unsupported pilot plan: " + planId); + } + + private static GlobalReduceOperator createGlobalAmountReduceOperator() { + return new GlobalReduceOperator<>( + new ReduceDescriptor<>((left, right) -> left, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + } + + private static ReduceByOperator createBucketReduceByOperator() { + return new ReduceByOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(4)), + Record.class, + Record.class + ).withSqlImplementation("bucket", "bucket"), + new ReduceDescriptor<>((left, right) -> left, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + } + + private static ReduceByOperator createTierReduceByOperator() { + return new ReduceByOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(0)), + Record.class, + Record.class + ).withSqlImplementation("tier", "tier"), + new ReduceDescriptor<>((left, right) -> left, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + } + + private static SortOperator createAmountSortOperator(int amountFieldIndex) { + return new SortOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(amountFieldIndex)), + Record.class, + Record.class + ).withSqlImplementation("amount", "ASC"), + DataSetType.createDefault(Record.class)); + } + + private static JoinOperator createCustomerJoinOperator(String sourceTable) { + return new JoinOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(1)), + Record.class, + Record.class + ).withSqlImplementation(sourceTable, "customer_id"), + new TransformationDescriptor<>( + record -> new Record(record.getField(0)), + Record.class, + Record.class + ).withSqlImplementation(CUSTOMERS_1K, "cust_id")); + } + + private static FilterOperator createAmerFilter() { + return new FilterOperator<>( + new PredicateDescriptor<>( + (Record record) -> "AMER".equals(record.getField(2)), + Record.class + ).withSqlImplementation("region = 'AMER'") + ); + } + + private static MapOperator createOrderAmountProjection() { + return new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType(COLUMNS), + "order_id", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + } + + private static MapOperator createOrderTierAmountProjection() { + return new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType(JOIN_COLUMNS), + "order_id", "tier", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + } + + private static MapOperator createTierAmountProjection() { + return new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType(JOIN_COLUMNS), + "tier", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + } + + private static MapOperator, Record> createJoinFlattenOperator() { + return createJoinFlattenOperator(new JoinFlattenFunction(), JOIN_FLATTEN_NAME); + } + + private static MapOperator, Record> createJoinOrderTierAmountFlattenOperator() { + return createJoinFlattenOperator(new JoinOrderTierAmountFlattenFunction(), JOIN_ORDER_TIER_AMOUNT_FLATTEN_NAME); + } + + private static MapOperator, Record> createJoinTierAmountFlattenOperator() { + return createJoinFlattenOperator(new JoinTierAmountFlattenFunction(), JOIN_TIER_AMOUNT_FLATTEN_NAME); + } + + private static MapOperator, Record> createJoinFlattenOperator( + FunctionDescriptor.SerializableFunction, Record> function, + String name) { + MapOperator, Record> operator = new MapOperator<>( + new TransformationDescriptor<>( + function, + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class)), + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class)); + operator.setName(name); + return operator; + } + + private WayangContext wayangContext() { + Configuration configuration = new Configuration(); + configuration.setProperty("wayang.bigquery.jdbc.url", JDBC_URL); + configuration.setProperty("wayang.core.log.enabled", "true"); + configuration.setProperty("wayang.core.explain.enabled", "false"); + configuration.setProperty("wayang.core.log.executions", EXECUTIONS_PATH.toString().replace('\\', '/')); + configuration.setProperty("wayang.core.log.cardinalities", CARDINALITIES_PATH.toString().replace('\\', '/')); + configuration.getMappingProvider().addAllToWhitelist( + Collections.singleton(new JoinFlattenMapping())); + return new WayangContext(configuration).withPlugin(BigQuery.plugin()); + } + + private void prepareTables() throws Exception { + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + statement.execute("CREATE SCHEMA IF NOT EXISTS " + SCHEMA + " OPTIONS(location='" + LOCATION + "')"); + for (int rowCount : ROW_COUNTS) { + String table = table("orders_" + formatRows(rowCount)); + statement.execute("DROP TABLE IF EXISTS " + table); + statement.execute("CREATE TABLE " + table + " AS " + + "SELECT " + + "CAST(n AS INT64) AS order_id, " + + "CAST(MOD(n, 1000) AS INT64) AS customer_id, " + + "CASE WHEN MOD(n, 2) = 0 THEN 'AMER' ELSE 'EMEA' END AS region, " + + "CAST(MOD(n, 10000) AS FLOAT64) AS amount, " + + "CAST(MOD(n, 100) AS INT64) AS bucket " + + "FROM " + createRowsSql(rowCount)); + assertEquals(rowCount, queryLong("SELECT count(*) FROM " + table), table + " row count"); + assertEquals(rowCount / 2, queryLong("SELECT count(*) FROM " + table + " WHERE region = 'AMER'"), + table + " AMER row count"); + } + statement.execute("DROP TABLE IF EXISTS " + CUSTOMERS_1K); + statement.execute("CREATE TABLE " + CUSTOMERS_1K + " AS " + + "SELECT " + + "CAST(n - 1 AS INT64) AS cust_id, " + + "CASE WHEN MOD(n, 2) = 0 THEN 'GOLD' ELSE 'SILVER' END AS tier " + + "FROM UNNEST(GENERATE_ARRAY(1, 1000)) AS n"); + assertEquals(1000, queryLong("SELECT count(*) FROM " + CUSTOMERS_1K), CUSTOMERS_1K + " row count"); + } + } + + private static String createRowsSql(int rowCount) { + return "UNNEST(GENERATE_ARRAY(1, " + rowCount + ")) AS n"; + } + + private static String formatRows(int rowCount) { + if (rowCount % 1000 == 0) { + return (rowCount / 1000) + "k"; + } + return String.valueOf(rowCount); + } + + private void initializeOutputFiles() throws Exception { + if (RESET_OUTPUT) { + Files.deleteIfExists(EXECUTIONS_PATH); + Files.deleteIfExists(CARDINALITIES_PATH); + writeManifestHeader(); + } else if (!Files.exists(MANIFEST_PATH)) { + writeManifestHeader(); + } + } + + private void writeManifestHeader() throws Exception { + try (BufferedWriter writer = Files.newBufferedWriter(MANIFEST_PATH, StandardCharsets.UTF_8)) { + writer.write("run_id,plan_id,operator_chain,input_rows_left,input_rows_right,expected_output_rows," + + "selectivity,repetition,is_warmup,sink_table,status,notes"); + writer.newLine(); + } + } + + private void appendManifest( + String runId, + String planId, + String operatorChain, + int inputRows, + long expectedOutputRows, + int repetition, + boolean isWarmup, + String sinkTable) throws Exception { + try (BufferedWriter writer = Files.newBufferedWriter( + MANIFEST_PATH, + StandardCharsets.UTF_8, + java.nio.file.StandardOpenOption.APPEND)) { + writer.write(csvRow( + runId, + planId, + operatorChain, + String.valueOf(inputRows), + hasJoin(planId) ? "1000" : "", + String.valueOf(expectedOutputRows), + hasFilter(planId) ? "0.5" : "1.0", + String.valueOf(repetition), + String.valueOf(isWarmup), + sinkTable, + "ok", + "")); + writer.newLine(); + } + } + + private static String csvRow(String... values) { + return String.join(",", Arrays.stream(values).map(BigQueryCostPilotIT::csvCell).toArray(String[]::new)); + } + + private static String csvCell(String value) { + if (value == null) { + return ""; + } + if (value.contains(",") || value.contains("\"") || value.contains("\n") || value.contains("\r")) { + return "\"" + value.replace("\"", "\"\"") + "\""; + } + return value; + } + + private static String getOperatorChain(String planId) { + switch (planId) { + case "S01": + return "TableSource->TableSink"; + case "S02": + return "TableSource->Filter(50%)->TableSink"; + case "S03": + return "TableSource->Projection->TableSink"; + case "S04": + return "TableSource->Filter(50%)->Projection->TableSink"; + case "S05": + return "TableSource->GlobalReduce->TableSink"; + case "S06": + return "TableSource->ReduceBy(bucket)->TableSink"; + case "S07": + return "TableSource->Sort(amount)->TableSink"; + case "S08": + return "Orders->Join(Customers 1k)->Projection->TableSink"; + case "S09": + return "TableSource->Filter(50%)->GlobalReduce->TableSink"; + case "S10": + return "TableSource->Filter(50%)->ReduceBy(bucket)->TableSink"; + case "S11": + return "TableSource->Filter(50%)->Sort(amount)->TableSink"; + case "S12": + return "TableSource->Projection(order_id,amount)->Sort(amount)->TableSink"; + case "S13": + return "TableSource->Filter(50%)->Projection(order_id,amount)->Sort(amount)->TableSink"; + case "S14": + return "Orders->Filter(50%)->Join(Customers 1k)->Projection->TableSink"; + case "S15": + return "Orders->Join(Customers 1k)->Projection(order_id,tier,amount)->Sort(amount)->TableSink"; + case "S16": + return "Orders->Join(Customers 1k)->Projection(tier,amount)->ReduceBy(tier)->TableSink"; + default: + throw new IllegalArgumentException("Unsupported pilot plan: " + planId); + } + } + + private static long getExpectedRows(String planId, int rowCount) { + if ("S05".equals(planId) || "S09".equals(planId)) { + return 1; + } + if ("S06".equals(planId)) { + return 100; + } + if ("S10".equals(planId)) { + return 50; + } + if ("S16".equals(planId)) { + return 2; + } + return hasFilter(planId) ? rowCount / 2 : rowCount; + } + + private static boolean hasFilter(String planId) { + return "S02".equals(planId) + || "S04".equals(planId) + || "S09".equals(planId) + || "S10".equals(planId) + || "S11".equals(planId) + || "S13".equals(planId) + || "S14".equals(planId); + } + + private static boolean hasJoin(String planId) { + return "S08".equals(planId) + || "S14".equals(planId) + || "S15".equals(planId) + || "S16".equals(planId); + } + + private static int[] parseIntList(String value) { + return Arrays.stream(value.split(",")) + .map(String::trim) + .filter(token -> !token.isEmpty()) + .mapToInt(Integer::parseInt) + .toArray(); + } + + private long queryLong(String sql) throws Exception { + try (Connection connection = jdbc(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getLong(1); + } + } + + private void dropTable(String table) throws Exception { + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS " + table); + } + } + + private static boolean isBigQueryAvailable() { + if ("your-project".equals(PROJECT_ID)) { + return false; + } + try { + Class.forName("com.google.cloud.bigquery.jdbc.BigQueryDriver"); + try (Connection connection = jdbc(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT 1")) { + return resultSet.next(); + } + } catch (Exception e) { + return false; + } + } + + private static Connection jdbc() throws Exception { + return DriverManager.getConnection(JDBC_URL); + } + + private static String cfg(String sysProp, String envVar, String dflt) { + String value = System.getProperty(sysProp); + if (value == null || value.isEmpty()) { + value = System.getenv(envVar); + } + return value == null || value.isEmpty() ? dflt : value; + } + + private static String table(String name) { + return "`" + PROJECT_ID + "." + DATASET + "." + name + "`"; + } + + private static Record flattenJoinResult(Object joinResult) { + if (joinResult instanceof Record) { + return (Record) joinResult; + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record( + left.getField(0), + left.getField(1), + left.getField(2), + left.getField(3), + left.getField(4), + right.getField(0), + right.getField(1)); + } + + private static Record flattenJoinOrderTierAmountResult(Object joinResult) { + if (joinResult instanceof Record) { + Record record = (Record) joinResult; + return new Record(record.getField(0), record.getField(6), record.getField(3)); + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record(left.getField(0), right.getField(1), left.getField(3)); + } + + private static Record flattenJoinTierAmountResult(Object joinResult) { + if (joinResult instanceof Record) { + Record record = (Record) joinResult; + return new Record(record.getField(6), record.getField(3)); + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record(right.getField(1), left.getField(3)); + } + + private static final class JoinFlattenFunction implements + FunctionDescriptor.SerializableFunction, Record> { + + @Override + public Record apply(Tuple2 tuple) { + return flattenJoinResult(tuple); + } + } + + private static final class JoinOrderTierAmountFlattenFunction implements + FunctionDescriptor.SerializableFunction, Record> { + + @Override + public Record apply(Tuple2 tuple) { + return flattenJoinOrderTierAmountResult(tuple); + } + } + + private static final class JoinTierAmountFlattenFunction implements + FunctionDescriptor.SerializableFunction, Record> { + + @Override + public Record apply(Tuple2 tuple) { + return flattenJoinTierAmountResult(tuple); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static final class JoinFlattenMapping implements Mapping { + + @Override + public java.util.Collection getTransformations() { + OperatorPattern pattern = new OperatorPattern( + "joinFlatten", + new MapOperator(null, DataSetType.none(), DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(operator -> isJoinFlattenName(((MapOperator) operator).getName())); + + ReplacementSubplanFactory factory = new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> createBigQueryProjection(matchedOperator.getName()).at(epoch)); + + return Collections.singleton(new PlanTransformation( + SubplanPattern.createSingleton(pattern), + factory, + BigQueryPlatform.getInstance())); + } + + private static BigQueryProjectionOperator createBigQueryProjection(String operatorName) { + ProjectionDescriptor, Record> descriptor = new ProjectionDescriptor<>( + getJoinFlattenFunction(operatorName), + Arrays.asList(getJoinFlattenColumns(operatorName)), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class)); + MapOperator, Record> projection = new MapOperator<>( + descriptor, + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class)); + projection.setName(operatorName); + return new BigQueryProjectionOperator((MapOperator) (MapOperator) projection); + } + + private static boolean isJoinFlattenName(String operatorName) { + return JOIN_FLATTEN_NAME.equals(operatorName) + || JOIN_ORDER_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName) + || JOIN_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName); + } + + private static String[] getJoinFlattenColumns(String operatorName) { + if (JOIN_ORDER_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName)) { + return JOIN_ORDER_TIER_AMOUNT_COLUMNS; + } + if (JOIN_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName)) { + return JOIN_TIER_AMOUNT_COLUMNS; + } + return JOIN_COLUMNS; + } + + private static FunctionDescriptor.SerializableFunction, Record> getJoinFlattenFunction( + String operatorName) { + if (JOIN_ORDER_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName)) { + return new JoinOrderTierAmountFlattenFunction(); + } + if (JOIN_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName)) { + return new JoinTierAmountFlattenFunction(); + } + return new JoinFlattenFunction(); + } + } +} diff --git a/wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryOperatorsIT.java b/wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryOperatorsIT.java new file mode 100644 index 000000000..4ab352580 --- /dev/null +++ b/wayang-platforms/wayang-bigquery/src/test/java/org/apache/wayang/bigquery/BigQueryOperatorsIT.java @@ -0,0 +1,740 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.bigquery; + +import org.apache.wayang.api.DataQuantaBuilder; +import org.apache.wayang.api.JavaPlanBuilder; +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.data.Tuple2; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.basic.operators.LocalCallbackSink; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.basic.types.RecordType; +import org.apache.wayang.bigquery.operators.BigQuerySortOperator; +import org.apache.wayang.bigquery.operators.BigQueryTableSource; +import org.apache.wayang.bigquery.platform.BigQueryPlatform; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.java.Java; +import org.apache.wayang.jdbc.compiler.FunctionCompiler; +import org.junit.jupiter.api.*; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Integration tests for the BigQuery platform operators, driven through the + * Wayang API ({@link BigQuery#plugin()}) against real BigQuery. + * + *

Why real BigQuery and not the emulator? The Wayang module connects + * through the BigQuery JDBC driver, which mandates Google OAuth2. The local + * {@code goccy/bigquery-emulator} is no-auth and only speaks to the Google + * client libraries, so it cannot serve the module's JDBC path. A real service + * account is therefore required to actually exercise these operators. + * + *

Coverage: {@code TableSource}, {@code Filter}, {@code Projection}, + * {@code GlobalReduce}, {@code ReduceBy}, {@code Sort}, {@code Join}, and + * {@code TableSink}, including JavaPlanBuilder combination plans that mirror + * the Trino/Presto suites. + * + *

Status: the suite contains 18 tests, including the full-plan join + * test and five JavaPlanBuilder combination tests. The full 18-test suite was + * green against a live BigQuery project on June 18, 2026. The tests use only + * {@code SELECT} and {@code CREATE TABLE AS}/{@code DROP} (DDL), never DML, so + * they run without billing enabled. + * + *

Note on the aggregate tests. {@code GlobalReduce}/{@code ReduceBy} + * carry their aggregation only in the SQL implementation ({@code SUM(amount)}); + * the Java fallback would not reproduce it. They therefore depend on the optimizer + * electing BigQuery pushdown, which it does here because they reduce cardinality. + * If a future run on different data shows a Java-side reduce, scale the reference + * dataset up (as the Trino/Presto suites do at 120k rows). {@code Sort} does not + * reduce cardinality, so it is verified via the operator's SQL-clause contract + * instead (see {@link #testSort()}). + * + *

Prerequisites

+ *
    + *
  1. A GCP service account with BigQuery access; key JSON on disk.
    2. + *
  2. A reference table (default {@code .sales.orders}) with columns + * {@code order_id, region, product, amount} and the 10-row dataset the + * assertions below expect (3 EMEA rows; >1000 amount rows non-empty).
    3. + *
+ * + *

Configuration (system property or environment variable; sysprop wins)

+ * 
+ *   bigquery.project   / BIGQUERY_PROJECT     GCP project id (required to run)
+ *   bigquery.saEmail   / BIGQUERY_SA_EMAIL    service-account email
+ *   bigquery.keyPath   / BIGQUERY_KEY_PATH    path to the SA key JSON
+ *   bigquery.table     / BIGQUERY_TABLE       backtick-quoted FQ table name
+ *   bigquery.location  / BIGQUERY_LOCATION    BigQuery dataset/job location
+ *
+ * If a connection cannot be established, every test is skipped (not failed). + * + *

Run

+ * 
+ *   JAVA_HOME=<jdk17> mvn -o test -pl wayang-platforms/wayang-bigquery \
+ *     -Dtest=BigQueryOperatorsIT -Dsurefire.failIfNoSpecifiedTests=false \
+ *     -Dbigquery.project=my-project \
+ *     -Dbigquery.saEmail=wayang-bq@my-project.iam.gserviceaccount.com \
+ *     -Dbigquery.keyPath=$HOME/wayang-bq-key.json \
+ *     -Drat.skip=true -Dlicense.skip=true -Pskip-prerequisite-check
+ *
+ */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class BigQueryOperatorsIT { + + private static final String PROJECT_ID = cfg("bigquery.project", "BIGQUERY_PROJECT", "your-project"); + private static final String SA_EMAIL = cfg("bigquery.saEmail", "BIGQUERY_SA_EMAIL", + "wayang-bq@" + PROJECT_ID + ".iam.gserviceaccount.com"); + private static final String KEY_PATH = cfg("bigquery.keyPath", "BIGQUERY_KEY_PATH", + System.getProperty("user.home") + "/wayang-bq-key.json"); + + /** Backtick-quoted fully-qualified BigQuery table name. */ + private static final String TABLE = cfg("bigquery.table", "BIGQUERY_TABLE", + "`" + PROJECT_ID + ".sales.orders`"); + + /** BigQuery dataset/job location. The setup README creates a US dataset. */ + private static final String LOCATION = cfg("bigquery.location", "BIGQUERY_LOCATION", "US"); + + /** Backtick-quoted sink target for the TableSink test; dropped in {@link #cleanup()}. */ + private static final String SINK_TABLE = "`" + PROJECT_ID + ".sales.wayang_emea_orders`"; + + /** Temporary lookup table for the JavaPlanBuilder join test. */ + private static final String JOIN_TABLE = "`" + PROJECT_ID + ".sales.wayang_regions`"; + + private static final String JDBC_URL = String.format( + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2;" + + "ProjectId=%s;OAuthType=0;OAuthServiceAcctEmail=%s;OAuthPvtKeyPath=%s;Location=%s", + PROJECT_ID, SA_EMAIL, KEY_PATH, LOCATION); + + private static boolean available = false; + + /** Resolution order: system property (preferred), environment variable, default. */ + private static String cfg(String sysProp, String envVar, String dflt) { + String v = System.getProperty(sysProp); + if (v == null || v.isEmpty()) v = System.getenv(envVar); + return (v == null || v.isEmpty()) ? dflt : v; + } + + // Setup + + @BeforeAll + static void checkAvailable() { + try { + Class.forName("com.google.cloud.bigquery.jdbc.BigQueryDriver"); + try (Connection conn = DriverManager.getConnection(JDBC_URL)) { + ResultSet rs = conn.createStatement().executeQuery("SELECT 1"); + available = rs.next(); + System.out.println("[SETUP] Connected to BigQuery project: " + PROJECT_ID); + } + } catch (Exception e) { + System.err.println("[SETUP] BigQuery not available; all tests will be skipped: " + e.getMessage()); + } + } + + @AfterAll + static void cleanup() { + if (!available) return; + try (Connection conn = DriverManager.getConnection(JDBC_URL)) { + conn.createStatement().execute("DROP TABLE IF EXISTS " + SINK_TABLE); + conn.createStatement().execute("DROP TABLE IF EXISTS " + JOIN_TABLE); + } catch (Exception e) { + System.err.println("[CLEANUP] failed to drop " + SINK_TABLE + ": " + e.getMessage()); + } + } + + private Configuration createBigQueryConfig() { + Configuration config = new Configuration(); + config.setProperty("wayang.bigquery.jdbc.url", JDBC_URL); + return config; + } + + private WayangContext createContext(Configuration config) { + return new WayangContext(config) + .withPlugin(Java.basicPlugin()) + .withPlugin(BigQuery.plugin()); + } + + private static void createRegionJoinTable() throws Exception { + try (Connection conn = DriverManager.getConnection(JDBC_URL)) { + conn.createStatement().execute("DROP TABLE IF EXISTS " + JOIN_TABLE); + conn.createStatement().execute( + "CREATE TABLE " + JOIN_TABLE + " AS SELECT DISTINCT region FROM " + TABLE); + } + } + + /** Record-aware multi-field projection (the POJO descriptor throws on >1 field). */ + private static ProjectionDescriptor project(String... fields) { + return ProjectionDescriptor.createForRecords( + new RecordType("order_id", "region", "product", "amount"), fields); + } + + private static Record flattenJoinResult(Object joinResult) { + if (joinResult instanceof Record) { + return (Record) joinResult; + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record( + left.getField(0), + left.getField(1), + left.getField(2), + left.getField(3), + right.getField(0)); + } + + // Verification tests + + /** BigQueryTableSource must be bound to BigQueryPlatform (drives wayang.bigquery.* config). */ + @Test + @Order(0) + @DisplayName("[VERIFY] BigQueryTableSource is bound to BigQueryPlatform") + void testPlatformBinding() { + BigQueryTableSource source = new BigQueryTableSource(TABLE, "order_id"); + + assertSame( + BigQueryPlatform.getInstance(), + source.getPlatform(), + "BigQueryTableSource.getPlatform() must return the BigQueryPlatform singleton" + ); + assertEquals("bigquery", source.getPlatform().getPlatformId(), + "Platform id drives all wayang.bigquery.* config key lookups"); + + System.out.println("[VERIFY] getPlatform() = " + source.getPlatform().getClass().getSimpleName()); + System.out.println("[VERIFY] getPlatformId() = " + source.getPlatform().getPlatformId()); + } + + /** Missing JDBC config must fail loudly, not silently fall back to Java evaluation. */ + @Test + @Order(1) + @DisplayName("[VERIFY] Execution fails when BigQuery JDBC config is missing") + void testFailsWithoutJdbcConfig() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + Configuration emptyConfig = new Configuration(); + BigQueryTableSource source = new BigQueryTableSource(TABLE, "order_id", "region"); + List results = new ArrayList<>(); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + source.connectTo(0, sink, 0); + + WayangContext ctx = new WayangContext(emptyConfig) + .withPlugin(Java.basicPlugin()) + .withPlugin(BigQuery.plugin()); + + assertThrows(Exception.class, + () -> ctx.execute("BQ-NoConfig", new WayangPlan(sink)), + "Should throw when wayang.bigquery.jdbc.url is not set" + ); + System.out.println("[VERIFY] Correctly threw when JDBC config was absent."); + } + + // Functional tests: TableSource, Filter, and Projection + + /** Full table scan: SELECT * FROM `
` */ + @Test + @Order(2) + @DisplayName("BigQuery: full table scan") + void testTableScan() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + List results = new ArrayList<>(); + BigQueryTableSource source = new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount"); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + source.connectTo(0, sink, 0); + + createContext(createBigQueryConfig()).execute("BQ-TableScan", new WayangPlan(sink)); + + assertEquals(10, results.size(), "Expected 10 rows"); + System.out.println("[PASS] TableScan: " + results.size() + " rows"); + } + + /** String filter pushdown: WHERE region = 'APAC' */ + @Test + @Order(3) + @DisplayName("BigQuery: filter pushdown (region = 'APAC')") + void testFilterString() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + List results = new ArrayList<>(); + BigQueryTableSource source = new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + r -> "APAC".equals(r.getField(1)), Record.class + ).withSqlImplementation("region = 'APAC'")); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + source.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + + createContext(createBigQueryConfig()).execute("BQ-Filter", new WayangPlan(sink)); + + assertFalse(results.isEmpty()); + results.forEach(r -> assertEquals("APAC", r.getField(1))); + System.out.println("[PASS] Filter(region='APAC'): " + results.size() + " rows"); + } + + /** Numeric filter pushdown: WHERE amount > 1000 */ + @Test + @Order(4) + @DisplayName("BigQuery: filter pushdown (amount > 1000)") + void testFilterNumeric() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + List results = new ArrayList<>(); + BigQueryTableSource source = new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + r -> ((Number) r.getField(3)).doubleValue() > 1000.0, Record.class + ).withSqlImplementation("amount > 1000")); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + source.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + + createContext(createBigQueryConfig()).execute("BQ-Filter-Numeric", new WayangPlan(sink)); + + assertFalse(results.isEmpty()); + results.forEach(r -> assertTrue(((Number) r.getField(3)).doubleValue() > 1000.0)); + System.out.println("[PASS] Filter(amount>1000): " + results.size() + " rows"); + } + + /** Projection pushdown / column pruning: SELECT region, amount FROM `
` */ + @Test + @Order(5) + @DisplayName("BigQuery: projection pushdown (region, amount)") + void testProjection() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + List results = new ArrayList<>(); + BigQueryTableSource source = new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount"); + MapOperator projection = new MapOperator<>( + project("region", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + source.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + + createContext(createBigQueryConfig()).execute("BQ-Projection", new WayangPlan(sink)); + + assertEquals(10, results.size()); + results.forEach(r -> assertEquals(2, r.size(), "Record should have 2 projected fields")); + System.out.println("[PASS] Projection(region, amount): " + results.size() + " rows"); + } + + /** Combined filter + projection in one SQL query: SELECT region, amount FROM `
` WHERE amount > 1000 */ + @Test + @Order(6) + @DisplayName("BigQuery: filter + projection pipeline") + void testFilterAndProjection() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + List results = new ArrayList<>(); + BigQueryTableSource source = new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + r -> ((Number) r.getField(3)).doubleValue() > 1000.0, Record.class + ).withSqlImplementation("amount > 1000")); + MapOperator projection = new MapOperator<>( + project("region", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + source.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + + createContext(createBigQueryConfig()).execute("BQ-Filter-Projection", new WayangPlan(sink)); + + assertFalse(results.isEmpty()); + results.forEach(r -> { + assertEquals(2, r.size()); + assertTrue(((Number) r.getField(1)).doubleValue() > 1000.0); + }); + System.out.println("[PASS] Filter+Projection: " + results.size() + " rows"); + } + + /** Cardinality estimation sanity check (optimizer runs SELECT count(*) before planning). */ + @Test + @Order(7) + @DisplayName("BigQuery: cardinality estimation via COUNT(*) is accurate") + void testCardinalityMatches() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + List results = new ArrayList<>(); + BigQueryTableSource source = new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + r -> "EMEA".equals(r.getField(1)), Record.class + ).withSqlImplementation("region = 'EMEA'")); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + source.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + + createContext(createBigQueryConfig()).execute("BQ-Cardinality", new WayangPlan(sink)); + + assertEquals(3, results.size(), "Expected 3 EMEA rows"); + System.out.println("[PASS] Cardinality: " + results.size() + " EMEA rows (expected 3)"); + } + + // Aggregation, ordering, sink, and JavaPlanBuilder combination tests + + /** + * GlobalReduce: SUM(amount) over the whole table collapses to a single row. + * + *

Note: the reduction lives only in the SQL implementation + * ({@code SUM(amount)}); the Java fallback would not reproduce it, so this + * test relies on the optimizer electing BigQuery pushdown for the reduce. + */ + @Test + @Order(8) + @DisplayName("BigQuery: global reduce (SUM(amount))") + void testGlobalReduce() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + List results = new ArrayList<>(); + BigQueryTableSource source = new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount"); + GlobalReduceOperator reduce = new GlobalReduceOperator<>( + new ReduceDescriptor<>((a, b) -> a, Record.class) + .withSqlImplementation("SUM(amount)"), + DataSetType.createDefault(Record.class)); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + source.connectTo(0, reduce, 0); + reduce.connectTo(0, sink, 0); + + createContext(createBigQueryConfig()).execute("BQ-GlobalReduce", new WayangPlan(sink)); + + assertEquals(1, results.size(), "global reduce must collapse to a single row"); + assertEquals(12752.0, ((Number) results.get(0).getField(0)).doubleValue(), 0.01); + System.out.println("[PASS] GlobalReduce SUM(amount) = " + results.get(0).getField(0)); + } + + /** ReduceBy: SUM(amount) GROUP BY region yields one row per region. */ + @Test + @Order(9) + @DisplayName("BigQuery: reduce-by (SUM(amount) GROUP BY region)") + void testReduceBy() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + List results = new ArrayList<>(); + BigQueryTableSource source = new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount"); + ReduceByOperator reduceBy = new ReduceByOperator<>( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(1)), Record.class, Record.class + ).withSqlImplementation("region", "region"), + new ReduceDescriptor<>((a, b) -> a, Record.class) + .withSqlImplementation("SUM(amount)"), + DataSetType.createDefault(Record.class)); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + source.connectTo(0, reduceBy, 0); + reduceBy.connectTo(0, sink, 0); + + createContext(createBigQueryConfig()).execute("BQ-ReduceBy", new WayangPlan(sink)); + + assertEquals(3, results.size(), "one row per region expected"); + Map sums = new HashMap<>(); + for (Record r : results) { + sums.put((String) r.getField(0), ((Number) r.getField(1)).doubleValue()); + } + assertEquals(6600.75, sums.get("APAC"), 0.01); + assertEquals(2320.5, sums.get("EMEA"), 0.01); + assertEquals(3830.75, sums.get("AMER"), 0.01); + System.out.println("[PASS] ReduceBy by region: " + sums); + } + + /** + * Sort: verified through the operator's SQL-clause contract executed on live + * BigQuery (the same approach Trino/Presto use for {@code Join}). + * + *

Unlike filter/projection, a sort does not reduce cardinality, so on the + * tiny reference table the cost optimizer keeps it in Java rather than pushing + * it down, and the jdbc-template sort key is a {@code Record}, which the Java + * sort cannot order (the Trino/Presto suites avoid this only because their + * 120k-row fixtures make SQL pushdown the cheaper plan). So we assert the + * operator's real contract: {@link BigQuerySortOperator#createSqlClause} must + * produce a BigQuery-valid {@code ORDER BY} that returns correctly ordered rows. + */ + @Test + @Order(10) + @DisplayName("BigQuery: sort (ORDER BY amount ASC) via operator SQL-clause contract") + void testSort() throws Exception { + Assumptions.assumeTrue(available, "BigQuery not available"); + + BigQuerySortOperator sort = new BigQuerySortOperator( + new TransformationDescriptor<>( + (Record r) -> new Record(r.getField(3)), Record.class, Record.class + ).withSqlImplementation("amount", "ASC")); + assertEquals(BigQueryPlatform.getInstance(), sort.getPlatform()); + + try (Connection conn = DriverManager.getConnection(JDBC_URL)) { + String orderBy = sort.createSqlClause(conn, new FunctionCompiler()); + assertTrue(orderBy.contains("ORDER BY amount ASC"), "unexpected ORDER BY clause: " + orderBy); + + ResultSet rs = conn.createStatement().executeQuery( + "SELECT order_id, region, product, amount FROM " + TABLE + orderBy); + List amounts = new ArrayList<>(); + while (rs.next()) amounts.add(rs.getDouble("amount")); + + assertEquals(10, amounts.size(), "sort must not change the cardinality"); + assertEquals(350.75, amounts.get(0), 0.001, "smallest amount first"); + assertEquals(3000.0, amounts.get(amounts.size() - 1), 0.001, "largest amount last"); + for (int i = 1; i < amounts.size(); i++) { + assertTrue(amounts.get(i - 1) <= amounts.get(i), "non-decreasing at index " + i); + } + System.out.println("[PASS] Sort ORDER BY amount ASC: " + amounts.size() + " rows in order"); + } + } + + /** + * TableSink: filter + sink composed into a single {@code CREATE TABLE ... AS + * SELECT} that runs entirely inside BigQuery; no data leaves the warehouse. + */ + @Test + @Order(11) + @DisplayName("BigQuery: table sink (CREATE TABLE AS SELECT ... WHERE region = 'EMEA')") + void testTableSink() throws Exception { + Assumptions.assumeTrue(available, "BigQuery not available"); + + BigQueryTableSource source = new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + r -> "EMEA".equals(r.getField(1)), Record.class + ).withSqlImplementation("region = 'EMEA'")); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", SINK_TABLE, + "order_id", "region", "product", "amount"); + source.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + + createContext(createBigQueryConfig()).execute("BQ-TableSink", new WayangPlan(sink)); + + try (Connection conn = DriverManager.getConnection(JDBC_URL)) { + ResultSet rs = conn.createStatement().executeQuery( + "SELECT count(*), COUNTIF(region != 'EMEA') FROM " + SINK_TABLE); + rs.next(); + assertEquals(3, rs.getLong(1), "sink table must hold all 3 EMEA orders"); + assertEquals(0, rs.getLong(2), "sink table must hold only EMEA orders"); + } + System.out.println("[PASS] TableSink wrote 3 EMEA rows into " + SINK_TABLE); + } + + /** + * Join: orders with a temporary distinct-region lookup table. + * + *

The logical {@link JoinOperator} emits {@code Tuple2}, + * while a pushed-down JDBC join already emits a flat {@link Record}. The + * following map normalizes both representations before the result reaches + * the sink. + */ + @Test + @Order(12) + @DisplayName("BigQuery: join orders with distinct regions") + void testJoin() throws Exception { + Assumptions.assumeTrue(available, "BigQuery not available"); + createRegionJoinTable(); + + List results = new ArrayList<>(); + BigQueryTableSource orders = new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount"); + BigQueryTableSource regions = new BigQueryTableSource( + JOIN_TABLE, "region"); + JoinOperator join = new JoinOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(1)), Record.class, Record.class + ).withSqlImplementation(TABLE, "region"), + new TransformationDescriptor<>( + record -> new Record(record.getField(0)), Record.class, Record.class + ).withSqlImplementation(JOIN_TABLE, "region")); + join.addTargetPlatform(BigQuery.platform()); + MapOperator flatten = new MapOperator<>( + BigQueryOperatorsIT::flattenJoinResult, Object.class, Record.class); + LocalCallbackSink sink = LocalCallbackSink.createCollectingSink(results, Record.class); + + orders.connectTo(0, join, 0); + regions.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sink, 0); + + createContext(createBigQueryConfig()).execute("BQ-Join", new WayangPlan(sink)); + + assertEquals(10, results.size()); + assertTrue(results.stream().allMatch(row -> row.getField(1).equals(row.getField(4)))); + } + + /** JavaPlanBuilder API: combine a pushed-down filter and projection. */ + @Test + @Order(13) + @DisplayName("BigQuery JavaPlanBuilder: readTable -> filter -> projection") + void javaPlanBuilderReadTableFilterProjection() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + Collection rows = new JavaPlanBuilder( + createContext(createBigQueryConfig()), "BigQuery JavaPlanBuilder filter projection test") + .readTable(new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount")) + .filter(record -> ((Number) record.getField(3)).doubleValue() > 1000.0) + .withSqlUdf("amount > 1000") + .withTargetPlatform(BigQuery.platform()) + .asRecords() + .projectRecords(new String[]{"region", "amount"}) + .withTargetPlatform(BigQuery.platform()) + .collect(); + + assertEquals(5, rows.size()); + assertTrue(rows.stream().allMatch(record -> + record.size() == 2 && ((Number) record.getField(1)).doubleValue() > 1000.0)); + } + + /** JavaPlanBuilder API: combine a filter with a global reduction. */ + @Test + @Order(14) + @DisplayName("BigQuery JavaPlanBuilder: readTable -> filter -> globalReduce") + void javaPlanBuilderReadTableFilterGlobalReduce() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + Collection rows = new JavaPlanBuilder( + createContext(createBigQueryConfig()), "BigQuery JavaPlanBuilder global reduce test") + .readTable(new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount")) + .filter(record -> "EMEA".equals(record.getField(1))) + .withSqlUdf("region = 'EMEA'") + .withTargetPlatform(BigQuery.platform()) + .reduce((left, right) -> left) + .withSqlUdf("SUM(amount)") + .withTargetPlatform(BigQuery.platform()) + .collect(); + + assertEquals(1, rows.size()); + assertEquals(2320.5, ((Number) rows.iterator().next().getField(0)).doubleValue(), 0.01); + } + + /** JavaPlanBuilder API: combine grouped aggregation and sorting. */ + @Test + @Order(15) + @DisplayName("BigQuery JavaPlanBuilder: readTable -> reduceByKey -> sort") + void javaPlanBuilderReadTableReduceBySort() { + Assumptions.assumeTrue(available, "BigQuery not available"); + + List rows = new ArrayList<>(new JavaPlanBuilder( + createContext(createBigQueryConfig()), "BigQuery JavaPlanBuilder reduce-by sort test") + .readTable(new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount")) + .reduceByKey( + record -> new Record(record.getField(1)), + (left, right) -> left) + .withSqlUdfs("region", "SUM(amount)") + .withTargetPlatform(BigQuery.platform()) + .sort(record -> new Record(record.getField(0))) + .withSqlUdf("region", "ASC") + .withTargetPlatform(BigQuery.platform()) + .collect()); + + assertEquals(3, rows.size()); + assertEquals("AMER", rows.get(0).getField(0)); + assertEquals("APAC", rows.get(1).getField(0)); + assertEquals("EMEA", rows.get(2).getField(0)); + } + + /** JavaPlanBuilder API: write a filtered projection into a BigQuery table. */ + @Test + @Order(16) + @DisplayName("BigQuery JavaPlanBuilder: readTable -> filter -> projection -> tableSink") + void javaPlanBuilderReadTableFilterProjectionTableSink() throws Exception { + Assumptions.assumeTrue(available, "BigQuery not available"); + + new JavaPlanBuilder( + createContext(createBigQueryConfig()), "BigQuery JavaPlanBuilder table sink test") + .readTable(new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount")) + .filter(record -> "EMEA".equals(record.getField(1))) + .withSqlUdf("region = 'EMEA'") + .withTargetPlatform(BigQuery.platform()) + .asRecords() + .projectRecords(new String[]{"order_id", "amount"}) + .withTargetPlatform(BigQuery.platform()) + .writeTable( + SINK_TABLE, + "overwrite", + new String[]{"order_id", "amount"}, + new Properties()); + + try (Connection conn = DriverManager.getConnection(JDBC_URL)) { + ResultSet rs = conn.createStatement().executeQuery("SELECT count(*) FROM " + SINK_TABLE); + rs.next(); + assertEquals(3, rs.getLong(1)); + } + } + + /** JavaPlanBuilder API: join orders with a temporary distinct-region table. */ + @Test + @Order(17) + @DisplayName("BigQuery JavaPlanBuilder: readTable + readTable -> join") + void javaPlanBuilderReadTableJoin() throws Exception { + Assumptions.assumeTrue(available, "BigQuery not available"); + + createRegionJoinTable(); + + JavaPlanBuilder plan = new JavaPlanBuilder( + createContext(createBigQueryConfig()), "BigQuery JavaPlanBuilder join test"); + DataQuantaBuilder orders = plan.readTable(new BigQueryTableSource( + TABLE, "order_id", "region", "product", "amount")); + DataQuantaBuilder regions = plan.readTable(new BigQueryTableSource( + JOIN_TABLE, "region")); + + Collection rows = orders + .join( + record -> new Record(record.getField(1)), + regions, + record -> new Record(record.getField(0))) + .withSqlUdfs(TABLE, "region", JOIN_TABLE, "region") + .withTargetPlatform(BigQuery.platform()) + .asRecords() + .collect(); + + assertEquals(10, rows.size()); + assertTrue(rows.stream().allMatch(row -> row.getField(1).equals(row.getField(4)))); + } +} diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java index 4b816b2eb..ba3d0839d 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java @@ -78,7 +78,9 @@ import org.apache.wayang.core.platform.ExecutionState; import org.apache.wayang.core.platform.Executor; import org.apache.wayang.core.platform.ExecutorTemplate; +import org.apache.wayang.core.platform.PartialExecution; import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.platform.lineage.ExecutionLineageNode; import org.apache.wayang.core.util.WayangCollections; import org.apache.wayang.jdbc.channels.SqlQueryChannel; import org.apache.wayang.jdbc.compiler.FunctionCompiler; @@ -151,20 +153,12 @@ public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, fin )); } - appendStatementTerminator(sb); + // Intentionally no trailing ';'. A trailing semicolon is unnecessary for a + // single-statement JDBC executeQuery and is rejected by strict SQL parsers + // such as Trino and BigQuery. Postgres/SQLite/HSQLDB accept its absence. return sb; } - private static void appendStatementTerminator(final StringBuilder query) { - int i = query.length() - 1; - while (i >= 0 && Character.isWhitespace(query.charAt(i))) { - i--; - } - if (i < 0 || query.charAt(i) != ';') { - query.append(';'); - } - } - /** * Creates a query channel and the sql statement * @@ -270,7 +264,7 @@ private static ExecutionTask selectStartTask(final Collection startTasks, fin * @param optimizationContext provides optimization information * @param jdbcExecutor the executor with the database connection */ - private static void executeSinkStage(final ExecutionStage stage, final OptimizationContext optimizationContext, + private static long executeSinkStage(final ExecutionStage stage, final OptimizationContext optimizationContext, final JdbcExecutor jdbcExecutor) { final Collection startTasks = stage.getStartTasks(); final Collection termTasks = stage.getTerminalTasks(); @@ -346,15 +340,44 @@ private static void executeSinkStage(final ExecutionStage stage, final Optimizat // Execute the composed query: CREATE TABLE x AS SELECT ... or INSERT INTO x // SELECT ... final String fullSql = sinkClause + " " + selectSql + sinkOp.createSqlSuffix(); + final long startTime = System.currentTimeMillis(); stmt.execute(fullSql); + final long executionDuration = System.currentTimeMillis() - startTime; jdbcExecutor.logger.info("Executed SQL sink: {}", fullSql); System.out.println("Executed sql sink: " + fullSql); + return executionDuration; } catch (final SQLException e) { throw new WayangException("Failed to execute SQL sink on table: " + sinkOp.getTableName(), e); } } + /** + * Creates lineage nodes for the JDBC operators that were executed as one SQL + * statement. Operators without an optimization context or load estimator are + * skipped, so JDBC platforms without cost specifications can still execute. + */ + private Collection createExecutionLineageNodes( + final ExecutionStage stage, + final OptimizationContext optimizationContext) { + final Collection executionLineageNodes = new ArrayList<>(); + for (ExecutionTask task : stage.getAllTasks()) { + final OptimizationContext.OperatorContext operatorContext = + optimizationContext.getOperatorContext(task.getOperator()); + if (operatorContext == null) { + this.logger.warn("Cannot profile {} because its optimization context is missing.", task); + continue; + } + if (operatorContext.getLoadProfileEstimator() == null) { + this.logger.warn("Cannot profile {} because its load profile estimator is missing.", task); + continue; + } + executionLineageNodes.add( + new ExecutionLineageNode(operatorContext).addAtomicExecutionFromOperatorContext()); + } + return executionLineageNodes; + } + /** * Retrieves the follow-up {@link ExecutionTask} of the given {@code task} * unless it is not comprising a {@link JdbcExecutionOperator} and/or not in the @@ -428,7 +451,16 @@ public void execute(final ExecutionStage stage, final OptimizationContext optimi final ExecutionTask termTask = (ExecutionTask) termTasks.toArray()[0]; if (termTask.getOperator() instanceof JdbcTableSinkOperator) { - JdbcExecutor.executeSinkStage(stage, optimizationContext, this); + final long executionDuration = JdbcExecutor.executeSinkStage(stage, optimizationContext, this); + if (this.isProfilingEnabled()) { + final PartialExecution partialExecution = this.createPartialExecution( + this.createExecutionLineageNodes(stage, optimizationContext), + executionDuration + ); + if (partialExecution != null) { + executionState.add(partialExecution); + } + } } else { // If it is normal stage: compose SQL and store in channel for downstream // consumption @@ -441,6 +473,10 @@ public void execute(final ExecutionStage stage, final OptimizationContext optimi } } + private boolean isProfilingEnabled() { + return this.getConfiguration().getBooleanProperty("wayang.core.log.enabled", false); + } + @Override public void dispose() { try { diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java index 0dfd8b698..8f7b3d8a2 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java @@ -81,7 +81,7 @@ void testExecuteWithPlainTableSource() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT * FROM customer;", + "SELECT * FROM customer", sqlQueryChannelInstance.getSqlQuery() ); } @@ -130,7 +130,7 @@ void testExecuteWithFilter() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT * FROM customer WHERE age >= 18;", + "SELECT * FROM customer WHERE age >= 18", sqlQueryChannelInstance.getSqlQuery() ); } @@ -172,7 +172,7 @@ void testExecuteWithProjection() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT name, age FROM customer;", + "SELECT name, age FROM customer", sqlQueryChannelInstance.getSqlQuery() ); } @@ -240,7 +240,7 @@ void testExecuteWithProjectionAndFilters() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT name, age FROM customer WHERE age >= 18 AND name IS NOT NULL;", + "SELECT name, age FROM customer WHERE age >= 18 AND name IS NOT NULL", sqlQueryChannelInstance.getSqlQuery() ); } diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java index 263730629..483bd4f81 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java @@ -23,7 +23,9 @@ import org.apache.wayang.core.optimizer.DefaultOptimizationContext; import org.apache.wayang.core.plan.executionplan.ExecutionStage; import org.apache.wayang.core.plan.executionplan.ExecutionTask; +import org.apache.wayang.core.platform.AtomicExecution; import org.apache.wayang.core.platform.CrossPlatformExecutor; +import org.apache.wayang.core.platform.PartialExecution; import org.apache.wayang.core.profiling.NoInstrumentationStrategy; import org.apache.wayang.jdbc.channels.SqlQueryChannel; import org.apache.wayang.jdbc.operators.JdbcTableSinkOperator; @@ -37,7 +39,11 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; @@ -51,6 +57,17 @@ class JdbcTableSinkExecutorTest { @Test void testOverwriteModeCreatesNewTable() throws SQLException { Configuration configuration = new Configuration(); + configuration.setProperty("wayang.core.log.enabled", "true"); + configuration.setProperty("wayang.hsqldb.cpu.mhz", "2700"); + configuration.setProperty("wayang.hsqldb.cores", "1"); + configuration.setProperty( + "wayang.hsqldb.tablesource.load", + "{\"in\":0,\"out\":1,\"cpu\":\"${1}\",\"ram\":\"0\",\"p\":1.0}" + ); + configuration.setProperty( + "wayang.hsqldb.tablesink.load", + "{\"in\":1,\"out\":0,\"cpu\":\"${1}\",\"ram\":\"0\",\"p\":1.0}" + ); HsqldbPlatform hsqldbPlatform = new HsqldbPlatform(); // Create source table with data @@ -89,10 +106,30 @@ void testOverwriteModeCreatesNewTable() throws SQLException { when(sqlStage.getStartTasks()).thenReturn(Collections.singleton(tableSourceTask)); when(sqlStage.getTerminalTasks()).thenReturn(Collections.singleton(sinkTask)); + when(sqlStage.getAllTasks()).thenReturn(new HashSet<>(Arrays.asList(tableSourceTask, sinkTask))); // Execute JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); - executor.execute(sqlStage, new DefaultOptimizationContext(job), job.getCrossPlatformExecutor()); + DefaultOptimizationContext optimizationContext = new DefaultOptimizationContext(job); + optimizationContext.addOneTimeOperator(tableSource); + optimizationContext.addOneTimeOperator(sinkOp); + executor.execute(sqlStage, optimizationContext, job.getCrossPlatformExecutor()); + + assertEquals(1, job.getCrossPlatformExecutor().getPartialExecutions().size()); + PartialExecution partialExecution = + job.getCrossPlatformExecutor().getPartialExecutions().iterator().next(); + Set estimatorKeys = partialExecution.getAtomicExecutionGroups().stream() + .flatMap(group -> group.getAtomicExecutions().stream()) + .map(AtomicExecution::getLoadProfileEstimator) + .map(estimator -> estimator.getConfigurationKey()) + .collect(Collectors.toSet()); + assertEquals( + new HashSet<>(Arrays.asList( + "wayang.hsqldb.tablesource.load", + "wayang.hsqldb.tablesink.load" + )), + estimatorKeys + ); // Verify table was created and contains all 3 rows try (Connection conn = hsqldbPlatform.createDatabaseDescriptor(configuration).createJdbcConnection()) { @@ -158,6 +195,7 @@ void testOverwriteModeReplacesExistingTable() throws SQLException { JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); executor.execute(sqlStage, new DefaultOptimizationContext(job), job.getCrossPlatformExecutor()); + assertEquals(0, job.getCrossPlatformExecutor().getPartialExecutions().size()); // Verify target was replaced. Old data should be gone, new schema and data present try (Connection conn = hsqldbPlatform.createDatabaseDescriptor(configuration).createJdbcConnection()) { @@ -176,6 +214,7 @@ void testOverwriteModeReplacesExistingTable() throws SQLException { @Test void testAppendModeInsertsIntoExistingTable() throws SQLException { Configuration configuration = new Configuration(); + configuration.setProperty("wayang.core.log.enabled", "false"); HsqldbPlatform hsqldbPlatform = new HsqldbPlatform(); //Create source and target table. Target has existing data. @@ -217,6 +256,7 @@ void testAppendModeInsertsIntoExistingTable() throws SQLException { JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); executor.execute(sqlStage, new DefaultOptimizationContext(job), job.getCrossPlatformExecutor()); + assertEquals(0, job.getCrossPlatformExecutor().getPartialExecutions().size()); // Verify existing data remains and new data is appended try (Connection conn = hsqldbPlatform.createDatabaseDescriptor(configuration).createJdbcConnection()) { @@ -252,4 +292,4 @@ void testAppendClauseGeneration() { sinkOp.setMode("append"); assertEquals("INSERT INTO my_table", sinkOp.createSqlClause(null, null)); } -} \ No newline at end of file +} diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcGlobalReduceOperatorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcGlobalReduceOperatorTest.java index b5ccb0848..739e38896 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcGlobalReduceOperatorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcGlobalReduceOperatorTest.java @@ -62,7 +62,7 @@ void testWithHsqldb() throws SQLException { final ExecutionStage sqlStage = mock(ExecutionStage.class); final JdbcTableSource tableSourceA = new HsqldbTableSource("testA"); - + final ExecutionTask tableSourceATask = new ExecutionTask(tableSourceA); tableSourceATask.setOutputChannel(0, new SqlQueryChannel(sqlChannelDescriptor, tableSourceA.getOutput(0))); tableSourceATask.setStage(sqlStage); @@ -86,15 +86,12 @@ void testWithHsqldb() throws SQLException { globalReduceTask.getOutputChannel(0).addConsumer(sqlToStreamTask, 0); sqlToStreamTask.setStage(nextStage); - final HsqldbPlatform hsqldbPlatform = new HsqldbPlatform(); try (Connection jdbcConnection = hsqldbPlatform.createDatabaseDescriptor(configuration).createJdbcConnection()) { final Statement statement = jdbcConnection.createStatement(); statement.execute("CREATE TABLE IF NOT EXISTS testA (a INT, b VARCHAR(6));"); statement.execute("INSERT INTO testA VALUES (0, 'zero');"); - statement.execute("CREATE TABLE IF NOT EXISTS testB (a INT, b INT);"); - statement.execute("INSERT INTO testB VALUES (0, 100);"); } final JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); @@ -112,6 +109,6 @@ void testWithHsqldb() throws SQLException { assertTrue(count > 0); } - assertEquals("SELECT COUNT(*) FROM testA;", sqlQueryChannelInstance.getSqlQuery()); + assertEquals("SELECT COUNT(*) FROM testA", sqlQueryChannelInstance.getSqlQuery()); } } diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java index 875a7a47b..d56405b19 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcJoinOperatorTest.java @@ -39,7 +39,9 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashSet; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; @@ -116,7 +118,12 @@ void testWithHsqldb() throws SQLException { joinTask.setOutputChannel(0, new SqlQueryChannel(sqlChannelDescriptor, joinOperator.getOutput(0))); joinTask.setStage(sqlStage); - when(sqlStage.getStartTasks()).thenReturn(Collections.singleton(tableSourceATask)); + // Deliberately list the right source first: JdbcExecutor must still choose + // the join's left source for the FROM clause. + when(sqlStage.getStartTasks()).thenReturn(new LinkedHashSet<>(Arrays.asList( + tableSourceBTask, tableSourceATask))); + when(sqlStage.getAllTasks()).thenReturn(new LinkedHashSet<>(Arrays.asList( + tableSourceBTask, tableSourceATask, joinTask))); when(sqlStage.getTerminalTasks()).thenReturn(Collections.singleton(joinTask)); ExecutionStage nextStage = mock(ExecutionStage.class); @@ -135,7 +142,7 @@ void testWithHsqldb() throws SQLException { System.out.println(); assertEquals( - "SELECT * FROM testA JOIN testB ON testB.a=testA.a;", + "SELECT * FROM testA JOIN testB ON testB.a=testA.a", sqlQueryChannelInstance.getSqlQuery() ); } @@ -213,7 +220,7 @@ void testMultiConditionJoinWithHsqldb() throws SQLException { String generatedSql = sqlQueryChannelInstance.getSqlQuery(); assertEquals( - "SELECT * FROM orders JOIN shipments ON orders.order_id=shipments.order_id AND orders.customer_id=shipments.customer_id;", + "SELECT * FROM orders JOIN shipments ON orders.order_id=shipments.order_id AND orders.customer_id=shipments.customer_id", generatedSql ); diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcReduceByOperatorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcReduceByOperatorTest.java index f00f4020e..556224027 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcReduceByOperatorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcReduceByOperatorTest.java @@ -91,6 +91,6 @@ void testWithHsqldb() throws SQLException { final SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor() .getChannelInstance(sqlToStreamTask.getInputChannel(0)); - assertEquals("SELECT col0,COUNT(*) FROM testA GROUP BY col0;", sqlQueryChannelInstance.getSqlQuery()); + assertEquals("SELECT col0,COUNT(*) FROM testA GROUP BY col0", sqlQueryChannelInstance.getSqlQuery()); } } diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcSortOperatorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcSortOperatorTest.java index 118fb7efa..1dc2fe12f 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcSortOperatorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/operators/JdbcSortOperatorTest.java @@ -86,6 +86,6 @@ void testWithHsqldb() throws SQLException { final SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor() .getChannelInstance(sqlToStreamTask.getInputChannel(0)); - assertEquals("SELECT * FROM testA ORDER BY col0 DESC;", sqlQueryChannelInstance.getSqlQuery()); + assertEquals("SELECT * FROM testA ORDER BY col0 DESC", sqlQueryChannelInstance.getSqlQuery()); } } diff --git a/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizerApp.java b/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizerApp.java index 154819b5f..779bd4ce8 100644 --- a/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizerApp.java +++ b/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizerApp.java @@ -109,6 +109,7 @@ public GeneticOptimizerApp(Configuration configuration) { Spark.platform(); Sqlite3.platform(); Postgres.platform(); + initializeOptionalPlatform("org.apache.wayang.bigquery.BigQuery"); // Load the ExecutionLog. double samplingFactor = this.configuration.getDoubleProperty("wayang.profiler.ga.sampling", 1d); @@ -201,6 +202,20 @@ public GeneticOptimizerApp(Configuration configuration) { ); } + /** + * Initializes a platform integration when it is available on the runtime + * classpath without making it a mandatory profiler dependency. + */ + private static void initializeOptionalPlatform(String platformFacadeClassName) { + try { + Class.forName(platformFacadeClassName).getMethod("platform").invoke(null); + } catch (ClassNotFoundException e) { + logger.debug("Optional platform {} is not on the classpath.", platformFacadeClassName); + } catch (ReflectiveOperationException e) { + throw new WayangException("Could not initialize optional platform " + platformFacadeClassName, e); + } + } + /** * Check if all {@link CardinalityEstimate}s for the {@link PartialExecution} are sufficiently confident. *