From 6c7e36c09f7fa2ab0c0be8dd5ca5428da708c4d8 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Tue, 11 Aug 2026 13:56:42 +0100 Subject: [PATCH 01/10] feat(kits): vendor incremental capture dataflow pipeline Beam/Java restoration pipeline copied verbatim from GoogleCloudPlatform/firebase-extensions@68ef3fa (firestore-incremental-capture-pipeline). The prebuilt target/restore-firestore.jar is left behind; the kit builds the jar from source. --- .../pipeline/.gitignore | 1 + .../pipeline/README.md | 23 +++ .../pipeline/pom.xml | 121 +++++++++++++ .../java/com/pipeline/FirestoreHelpers.java | 156 +++++++++++++++++ .../com/pipeline/FirestoreReconstructor.java | 164 ++++++++++++++++++ .../com/pipeline/IncrementalCaptureLog.java | 134 ++++++++++++++ .../ReadFromFirestoreWithTimestamp.java | 54 ++++++ .../com/pipeline/RestorationPipeline.java | 139 +++++++++++++++ .../src/main/java/com/pipeline/Utils.java | 50 ++++++ .../com/pipeline/WriteToFirestoreDoFn.java | 63 +++++++ 10 files changed, 905 insertions(+) create mode 100644 kits/firestore-incremental-capture/pipeline/.gitignore create mode 100644 kits/firestore-incremental-capture/pipeline/README.md create mode 100644 kits/firestore-incremental-capture/pipeline/pom.xml create mode 100644 kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/FirestoreHelpers.java create mode 100644 kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/FirestoreReconstructor.java create mode 100644 kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/IncrementalCaptureLog.java create mode 100644 kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/ReadFromFirestoreWithTimestamp.java create mode 100644 kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/RestorationPipeline.java create mode 100644 kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/Utils.java create mode 100644 kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/WriteToFirestoreDoFn.java diff --git a/kits/firestore-incremental-capture/pipeline/.gitignore b/kits/firestore-incremental-capture/pipeline/.gitignore new file mode 100644 index 000000000..2f7896d1d --- /dev/null +++ b/kits/firestore-incremental-capture/pipeline/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/kits/firestore-incremental-capture/pipeline/README.md b/kits/firestore-incremental-capture/pipeline/README.md new file mode 100644 index 000000000..cf7144acd --- /dev/null +++ b/kits/firestore-incremental-capture/pipeline/README.md @@ -0,0 +1,23 @@ +## Debug the pipeline locally + +To debug this pipeline locally, use the `DirectRunner`: + +Note: If your Cloud Storage bucket was provisioned after September 30, 2024 +the default bucket name will be suffixed with `.firebasestorage.app` instead of `.appspot.com` + +```bash +mvn compile exec:java \ + -Dexec.mainClass=com.pipeline.RestorationPipeline \ + -Dexec.args='--timestamp=1697740800 --firestoreCollectionId="test" --firestoreDb="test" --tempLocation="gs://PROJECT_ID.appspot.com" --project="PROJECT_ID"' +``` + +### Arguments + +- `timestamp`: The timestamp to restore the data to from a PITR, if it's further than 7 days in the past, it will be set to 7 days in the past. The timestamp is in UNIX seconds. +- `firestoreCollectionId`: The collection to restore, use `*` if you want the full database. + +## Compile JAR to run on Dataflow + +```bash +mvn clean package -DskipTests -Dexec.mainClass=com.pipeline.RestorationPipeline +``` \ No newline at end of file diff --git a/kits/firestore-incremental-capture/pipeline/pom.xml b/kits/firestore-incremental-capture/pipeline/pom.xml new file mode 100644 index 000000000..fc7d15ce0 --- /dev/null +++ b/kits/firestore-incremental-capture/pipeline/pom.xml @@ -0,0 +1,121 @@ + + + 4.0.0 + + com.pipeline + pipeline + 1.0 + + + 1.8 + 1.8 + + + restore-firestore + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + package + + shade + + + false + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + + java + + + + + com.pipeline.RestorationPipeline + + + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + + true + lib/ + com.pipeline.RestorationPipeline + + + + + + + + + + + org.slf4j + slf4j-simple + 2.0.9 + + + + + org.slf4j + slf4j-api + 2.0.9 + + + + + org.apache.beam + beam-sdks-java-core + 2.51.0 + + + org.apache.beam + beam-runners-google-cloud-dataflow-java + 2.51.0 + + + org.apache.beam + beam-sdks-java-io-google-cloud-platform + 2.51.0 + + + + org.apache.beam + beam-runners-direct-java + 2.51.0 + + + + + \ No newline at end of file diff --git a/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/FirestoreHelpers.java b/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/FirestoreHelpers.java new file mode 100644 index 000000000..2a2310977 --- /dev/null +++ b/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/FirestoreHelpers.java @@ -0,0 +1,156 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pipeline; + +import org.apache.beam.sdk.io.gcp.firestore.FirestoreIO; +import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1.BatchWriteWithSummary; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.firestore.v1.Document; +import com.google.firestore.v1.RunQueryRequest; +import com.google.firestore.v1.RunQueryResponse; +import com.google.firestore.v1.StructuredQuery; +import com.google.firestore.v1.StructuredQuery.CollectionSelector; +import com.google.firestore.v1.Write; + +public class FirestoreHelpers { + public static final class RunQuery extends BasePTransform { + private static final Logger LOG = LoggerFactory.getLogger(Utils.class); + + final String projectId; + + public RunQuery(String projectId, String database) { + super("projects/" + projectId + "/databases/" + database + "/documents"); + this.projectId = projectId; + } + + @Override + public PCollection expand(PCollection input) { + LOG.info(baseDocumentPath); + return input.apply( + ParDo.of( + new DoFn() { + @ProcessElement + public void processElement(ProcessContext c) { + final String collectionId = c.element(); + + if (collectionId.equals("*")) { + LOG.info("Querying all collections"); + RunQueryRequest runQueryRequest = RunQueryRequest.newBuilder() + .setParent(baseDocumentPath) + .setStructuredQuery(StructuredQuery.newBuilder().build()) + .build(); + + c.output(runQueryRequest); + return; + } + + CollectionSelector collection = CollectionSelector + .newBuilder() + .setCollectionId(collectionId) + .build(); + + RunQueryRequest runQueryRequest = RunQueryRequest.newBuilder() + .setParent(baseDocumentPath) + .setStructuredQuery( + com.google.firestore.v1.StructuredQuery.newBuilder() + .addFrom(collection) + .build()) + .build(); + + c.output(runQueryRequest); + } + })); + } + } + + public static final class RunQueryResponseToDocument extends BasePTransform { + + public RunQueryResponseToDocument() { + super(""); + } + + @Override + public PCollection expand(PCollection input) { + return input.apply( + ParDo.of( + new DoFn() { + @ProcessElement + public void processElement(ProcessContext c) { + RunQueryResponse response = c.element(); + c.output(response.getDocument()); + } + })); + } + } + + public static final class DocumentToWrite extends BasePTransform, Write> { + + final String projectId; + + public DocumentToWrite(String projectId, String database) { + super("projects/" + projectId + "/databases/" + database + "/documents"); + this.projectId = projectId; + } + + @Override + public PCollection expand(PCollection> input) { + return input.apply( + ParDo.of( + new DoFn, Write>() { + @ProcessElement + public void processElement(ProcessContext c) { + String changeType = c.element().getKey(); + Document document = c.element().getValue(); + + // LOG.info("STEP ONE >>>>>>> changeType: {}, documentName: {}", changeType, + // document.getName()); + + switch (changeType) { + case "DELETE": + c.output(Write.newBuilder() + .setDelete(document.getName()) + .build()); + + break; + + default: + c.output(Write.newBuilder() + .setUpdate(document) + .build()); + } + } + })); + } + } + + private abstract static class BasePTransform + extends PTransform, PCollection> { + + protected final String baseDocumentPath; + + private BasePTransform(String baseDocumentPath) { + this.baseDocumentPath = baseDocumentPath; + } + } +} diff --git a/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/FirestoreReconstructor.java b/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/FirestoreReconstructor.java new file mode 100644 index 000000000..8c801811d --- /dev/null +++ b/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/FirestoreReconstructor.java @@ -0,0 +1,164 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pipeline; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.protobuf.Timestamp; +import com.google.firestore.v1.ArrayValue; +import com.google.firestore.v1.MapValue; +import com.google.firestore.v1.Value; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.List; +import java.time.Instant; + +public class FirestoreReconstructor { + + public enum FirestoreType { + STRING, + NUMBER, + BOOLEAN, + NULL, + TIMESTAMP, + GEOPOINT, + REFERENCE, + } + + // This method recursively builds a Firestore map from a JSON object + // representing a Firestore document or map, according to our schema + public static Map buildFirestoreMap(JsonElement dataJson, String projectId, String databaseId) { + + JsonObject dataObject = dataJson.getAsJsonObject(); + Map fieldsMap = new HashMap<>(); + + for (Map.Entry entry : dataObject.entrySet()) { + JsonElement valueElem = entry.getValue(); + + if (valueElem.isJsonObject() && valueElem.getAsJsonObject().has("type") + && valueElem.getAsJsonObject().has("value")) { + JsonObject entryValueObject = valueElem.getAsJsonObject(); + String valueType = entryValueObject.get("type").getAsString().toUpperCase(); + + Value val; + switch (valueType) { + case "STRING": + val = Value.newBuilder().setStringValue(entryValueObject.get("value").getAsString()).build(); + break; + case "NUMBER": + val = Value.newBuilder().setDoubleValue(entryValueObject.get("value").getAsDouble()).build(); + break; + case "BOOLEAN": + String originalValue = entryValueObject.get("value").getAsString(); + + if (originalValue.equals("true")) { + val = Value.newBuilder().setBooleanValue(true).build(); + } else { + val = Value.newBuilder().setBooleanValue(false).build(); + } + break; + case "OBJECT": + val = Value.newBuilder().setMapValue( + MapValue.newBuilder().putAllFields( + buildFirestoreMap(entryValueObject.get("value"), projectId, databaseId))) + .build(); + break; + case "MAP": + val = Value.newBuilder().setMapValue( + MapValue.newBuilder().putAllFields( + buildFirestoreMap(entryValueObject.get("value"), projectId, databaseId))) + .build(); + break; + case "ARRAY": + val = Value.newBuilder().setArrayValue( + ArrayValue.newBuilder().addAllValues( + buildFirestoreList(entryValueObject.get("value").getAsJsonArray(), projectId, + databaseId))) + .build(); + break; + case "GEOPOINT": + JsonObject geopointValue = entryValueObject.get("value").getAsJsonObject(); + JsonObject latitude = geopointValue.get("latitude").getAsJsonObject(); + JsonObject longitude = geopointValue.get("longitude").getAsJsonObject(); + + Double latitudeValue = latitude.get("value").getAsDouble(); + Double longitudeValue = longitude.get("value").getAsDouble(); + + val = Value.newBuilder().setGeoPointValue( + com.google.type.LatLng.newBuilder().setLatitude(latitudeValue) + .setLongitude(longitudeValue) + .build()) + .build(); + break; + case "TIMESTAMP": + + // parse the timestamp value as an Instant + Instant instant = Instant.parse(entryValueObject.get("value").getAsString()); + + long epochSecond = instant.getEpochSecond(); + int nanoSecond = instant.getNano(); + + Timestamp timestamp = Timestamp.newBuilder().setSeconds(epochSecond).setNanos(nanoSecond) + .build(); + + // convert to seconds and nanoseconds + val = Value.newBuilder().setTimestampValue(timestamp).build(); + break; + + case "REFERENCE": + + String pathString = entryValueObject.get("value").getAsString(); + + String fullReferenceString = String.format( + "projects/%s/databases/%s/documents/%s", + projectId, + databaseId, + pathString); + + val = Value.newBuilder().setReferenceValue(fullReferenceString) + .build(); + break; + default: + val = null; + continue; + } + + fieldsMap.put(entry.getKey(), val); + } + } + + // log it + return fieldsMap; + } + + private static List buildFirestoreList(JsonArray arr, String projectId, String databaseId) { + + List lst = new ArrayList<>(); + for (JsonElement el : arr) { + Map mapData = buildFirestoreMap(el, projectId, databaseId); + Value val = Value.newBuilder().setMapValue( + MapValue.newBuilder().putAllFields(mapData)).build(); + + lst.add(val); + } + + return lst; + } +} diff --git a/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/IncrementalCaptureLog.java b/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/IncrementalCaptureLog.java new file mode 100644 index 000000000..ef8743efb --- /dev/null +++ b/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/IncrementalCaptureLog.java @@ -0,0 +1,134 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pipeline; + +import java.util.Map; + +import org.apache.avro.generic.GenericRecord; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; +import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.joda.time.DateTime; +import org.joda.time.Instant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.firestore.v1.Document; +import com.google.firestore.v1.Value; +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; + +public class IncrementalCaptureLog + extends PTransform, PCollection>> { + private static final Logger LOG = LoggerFactory.getLogger(Utils.class); + + final private String projectId; + final private String firestoreDbId; + final private Instant timestamp; + final private String datasetId; + final private String tableId; + + public IncrementalCaptureLog(String projectId, Instant timestamp, String firestoreDbId, String datasetId, + String tableId) { + this.projectId = projectId; + this.timestamp = timestamp; + this.firestoreDbId = firestoreDbId; + this.datasetId = datasetId; + this.tableId = tableId; + } + + @Override + public PCollection> expand(PCollection input) { + + String formattedTimestamp = new DateTime(Utils.adjustDate(timestamp)).toString("yyyy-MM-dd HH:mm:ss"); + + return input.getPipeline().apply("Read from BigQuery with Dynamic Query", + BigQueryIO.read(new SerializableFunction>() { + public KV apply(SchemaAndRecord schemaAndRecord) { + return convertToFirestoreValue(schemaAndRecord, projectId, firestoreDbId); + } + }).fromQuery(constructQuery(formattedTimestamp)).usingStandardSql().withTemplateCompatibility()); + } + + private String constructQuery(String timestamp) { + + LOG.info("Querying BigQuery for changes before timestamp: " + timestamp); + String query = "WITH RankedChanges AS (" + + " SELECT " + + " documentId," + + " documentPath," + + " changeType," + + " beforeData," + + " afterData," + + " timestamp," + + " ROW_NUMBER() OVER(PARTITION BY documentId ORDER BY timestamp DESC) as rank" + + " FROM `" + projectId + "." + datasetId + "." + tableId + "`" + + " WHERE timestamp < TIMESTAMP('" + (timestamp) + "') " + + ") " + + "SELECT " + + " documentId," + + " documentPath," + + " changeType," + + " beforeData," + + " afterData," + + " timestamp " + + "FROM RankedChanges " + + "WHERE rank = 1 " + + "ORDER BY documentId, timestamp DESC"; + + return query; + + } + + private static KV convertToFirestoreValue(SchemaAndRecord schemaAndRecord, String projectId, + String databaseId) { + + GenericRecord record = schemaAndRecord.getRecord(); + + String data = record.get("afterData").toString(); + String documentPath = createDocumentName(record.get("documentPath").toString(), projectId, databaseId); + String changeType = record.get("changeType").toString(); + + // this JsonElement has serialized data, e.g a string would be represented on + // the json tree as {type: "STRING", value: "some string"} + JsonElement dataJson = JsonParser.parseString(data); + + Map firestoreMap = FirestoreReconstructor.buildFirestoreMap(dataJson, projectId, databaseId); + + // using static methods as beam seems to error when passing an instance version + // of FirestoreReconstructor to the transform + Document doc = Document.newBuilder().putAllFields((Map) firestoreMap).setName(createDocumentName( + documentPath, projectId, databaseId)).build(); + + KV kv = KV.of(changeType, doc); + + return kv; + } + + private static String createDocumentName(String path, String projectId, String databaseId) { + String documentPath = String.format( + "projects/%s/databases/%s/documents", + projectId, + databaseId); + + return documentPath + "/" + path; + } + +} diff --git a/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/ReadFromFirestoreWithTimestamp.java b/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/ReadFromFirestoreWithTimestamp.java new file mode 100644 index 000000000..f20c7d02a --- /dev/null +++ b/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/ReadFromFirestoreWithTimestamp.java @@ -0,0 +1,54 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pipeline; + +import org.apache.beam.sdk.io.gcp.firestore.FirestoreIO; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PCollection; +import org.joda.time.Instant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.firestore.v1.RunQueryRequest; +import com.google.firestore.v1.RunQueryResponse; + +public class ReadFromFirestoreWithTimestamp + extends PTransform, PCollection> { + private static final Logger LOG = LoggerFactory.getLogger(Utils.class); + + final private Instant readTime; + + public ReadFromFirestoreWithTimestamp(Instant readTime) { + this.readTime = readTime; + } + + @Override + public PCollection expand(PCollection input) { + try { + Utils.adjustDate(readTime); + } catch (Exception e) { + LOG.error(e.getMessage()); + throw new IllegalArgumentException(e); + } + + LOG.info("Read time: " + readTime.toDateTime().toString()); + + return input.apply( + "Batch read from Firestore with Timestamp", + FirestoreIO.v1().read().runQuery().withReadTime(readTime).build()); + } +} diff --git a/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/RestorationPipeline.java b/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/RestorationPipeline.java new file mode 100644 index 000000000..6adcfcc56 --- /dev/null +++ b/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/RestorationPipeline.java @@ -0,0 +1,139 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pipeline; + +import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.PipelineResult; +import org.apache.beam.sdk.io.gcp.firestore.FirestoreIO; +import org.apache.beam.sdk.options.Description; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.PCollection; +import org.joda.time.Instant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.cloud.firestore.FirestoreOptions; +import com.google.firestore.v1.Document; +import com.google.firestore.v1.Write; + +public class RestorationPipeline { + private static final FirestoreOptions DEFAULT_FIRESTORE_OPTIONS = FirestoreOptions.getDefaultInstance(); + private static final Logger LOG = LoggerFactory.getLogger(Utils.class); + + public interface MyOptions extends DataflowPipelineOptions, + org.apache.beam.sdk.io.gcp.firestore.FirestoreOptions { + + @Description("The timestamp to read from Firestore") + Long getTimestamp(); + + void setTimestamp(Long value); + + @Description("The Firestore collection to read from, or '*' to read from all collections") + String getFirestoreCollectionId(); + + void setFirestoreCollectionId(String value); + + @Description("The BigQuery dataset Id to export the data from") + String getBigQueryDataset(); + + void setBigQueryDataset(String value); + + @Description("The BigQuery table Id to export the data from") + String getBigQueryTable(); + + void setBigQueryTable(String value); + + } + + public static void main(String[] args) { + + MyOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().as(MyOptions.class); + + Pipeline pipeline = Pipeline.create(options); + + String project = options.getProject(); + String collectionId = options.getFirestoreCollectionId(); + String secondaryDatabase = options.getFirestoreDb(); + String datasetId = options.getBigQueryDataset(); + String tableId = options.getBigQueryTable(); + String defaultDatabase = DEFAULT_FIRESTORE_OPTIONS.getDatabaseId(); + Instant readTime = Utils.adjustDate(Instant.ofEpochSecond(options.getTimestamp())); + + options.setFirestoreDb(secondaryDatabase); + + // Read from Firestore at the specified timestamp to form the baseline + // The returned PCollection contains the documents at the specified timestamp in + // Firestore + PCollection documentsAtReadTime = pipeline + .apply("Passing the collection ID " + collectionId, Create.of(collectionId)) + .apply("Prepare the PITR query", new FirestoreHelpers.RunQuery(project, defaultDatabase)) + .apply( + FirestoreIO.v1() + .read() + .runQuery() + .withReadTime(readTime) + .build()) + .apply(new FirestoreHelpers.RunQueryResponseToDocument()); + + // Write the documents to the secondary database + documentsAtReadTime + .apply("Create the write request", + ParDo.of(new DoFn() { + @ProcessElement + public void processElement(ProcessContext c) { + Document document = c.element(); + + // Replace the default database with the secondary database in the document id + String id = document.getName().replace(defaultDatabase, secondaryDatabase); + + Document newDocument = Document.newBuilder() + .setName(id) + .putAllFields(document.getFieldsMap()) + .build(); + + c.output(Write.newBuilder() + .setUpdate(newDocument) + .build()); + } + })) + .apply("Write to the Firestore database instance", FirestoreIO.v1().write().batchWrite().build()); + + // BigQuery read and subsequent Firestore write + pipeline + .apply(Create.of("")) + .apply("Read from BigQuery", + new IncrementalCaptureLog(project, readTime, secondaryDatabase, datasetId, tableId)) + .apply("Prepare write operations", + new FirestoreHelpers.DocumentToWrite(defaultDatabase, defaultDatabase)) + .apply("Write to the Firestore database instance (From BigQuery)", + FirestoreIO.v1().write().batchWrite().build()); + + PipelineResult result = pipeline.run(); + + // We try to identify if the pipeline is being run or a template is being + // created + if (options.as(DataflowPipelineOptions.class).getTemplateLocation() == null) { + // If template location is null, then, pipeline is being run, so we can wait + // until finish + result.waitUntilFinish(); + } + } +} diff --git a/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/Utils.java b/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/Utils.java new file mode 100644 index 000000000..3dfc36aec --- /dev/null +++ b/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/Utils.java @@ -0,0 +1,50 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pipeline; + +import org.joda.time.DateTime; +import org.joda.time.Days; +import org.joda.time.Instant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Utils { + private static final Logger LOG = LoggerFactory.getLogger(Utils.class); + + public static Instant adjustDate(Instant timestamp) { + DateTime providedDate = new DateTime(timestamp).withSecondOfMinute(0); + DateTime now = DateTime.now().withSecondOfMinute(0); + + int daysDiff = Days.daysBetween(providedDate.withTimeAtStartOfDay(), now.withTimeAtStartOfDay()).getDays(); + + LOG.info("The provided date is " + providedDate.toString("yyyy-MM-dd HH:mm:ss")); + LOG.info(daysDiff + " days difference between now and the provided date"); + + if (providedDate.isAfterNow()) { + // The provided date is in the future + throw new IllegalArgumentException("The provided date is in the future!"); + } + + if (daysDiff > 7) { + // Set the date representing 7 days before the "now" date + return Instant.parse(now.minusDays(7).toString("yyyy-MM-dd'T'HH:mm:ss.SSS")); + } + + return Instant.parse(providedDate.toString("yyyy-MM-dd'T'HH:mm:ss.SSS")); + } + +} diff --git a/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/WriteToFirestoreDoFn.java b/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/WriteToFirestoreDoFn.java new file mode 100644 index 000000000..e60d0d067 --- /dev/null +++ b/kits/firestore-incremental-capture/pipeline/src/main/java/com/pipeline/WriteToFirestoreDoFn.java @@ -0,0 +1,63 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pipeline; + +import org.apache.beam.sdk.io.gcp.firestore.FirestoreIO; +import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1.BatchWriteWithSummary; +import org.apache.beam.sdk.transforms.DoFn; + +import com.google.cloud.firestore.Firestore; +import com.google.cloud.firestore.FirestoreOptions; +import com.google.firestore.v1.Document; + +public class WriteToFirestoreDoFn extends DoFn { + private final String projectId; + private final String databaseId; + private transient Firestore firestore; + + public WriteToFirestoreDoFn(String projectId, String databaseId) { + this.projectId = projectId; + this.databaseId = databaseId; + } + + @Setup + public void setup() { + // Initialize the Firestore client with the specified projectId + firestore = FirestoreOptions.newBuilder() + .setProjectId(projectId) + .setDatabaseId(databaseId) + .build() + .getService(); + } + + @ProcessElement + public void processElement(ProcessContext c) { + // Write the document data to Firestore + c.output(FirestoreIO.v1().write().batchWrite().build()); + } + + @Teardown + public void teardown() { + if (firestore != null) { + try { + firestore.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } +} From fffcbc55f29e9abf4c5c26266f48464abe558d2c Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Tue, 11 Aug 2026 13:56:58 +0100 Subject: [PATCH 02/10] chore(kits): add incremental capture legacy reference Temporary reference copy of the firestore-incremental-capture extension from GoogleCloudPlatform/firebase-extensions@68ef3fa, formatted to repo prettier style. Removed once the kit is written, as with the other kits. --- .../legacy/CHANGELOG.md | 52 +++ .../legacy/POSTINSTALL.md | 104 +++++ .../legacy/PREINSTALL.md | 53 +++ .../legacy/README.md | 116 +++++ .../legacy/extension.yaml | 223 ++++++++++ .../legacy/functions/.gitignore | 9 + .../__tests__/backupDatabase.test.ts | 75 ++++ .../__tests__/firestoreSerializer.test.ts | 418 ++++++++++++++++++ .../functions/__tests__/functions.test.ts | 126 ++++++ .../legacy/functions/__tests__/helpers.ts | 54 +++ .../__tests__/manualTesting/backup-test.js | 72 +++ .../__tests__/manualTesting/createTestData.js | 74 ++++ .../__tests__/manualTesting/exportfromBQ.js | 30 ++ .../legacy/functions/__tests__/tsconfig.json | 4 + .../legacy/functions/__tests__/types.ts | 24 + .../legacy/functions/jest.config.js | 32 ++ .../legacy/functions/package.json | 48 ++ .../legacy/functions/src/config.ts | 71 +++ .../src/constants/bq_backup_schema.ts | 24 + .../src/dataflow/build_flex_template.ts | 54 +++ .../functions/src/dataflow/cloud_build.ts | 95 ++++ .../src/dataflow/on_complete_handler.ts | 30 ++ .../src/dataflow/trigger_dataflow_job.ts | 71 +++ .../legacy/functions/src/index.ts | 75 ++++ .../legacy/functions/src/logs.ts | 49 ++ .../src/tasks/on_backup_restore_handler.ts | 143 ++++++ .../tasks/on_firestore_backup_init_handler.ts | 77 ++++ .../tasks/on_http_run_restoration_handler.ts | 53 +++ .../src/tasks/on_run_initial_setup_handler.ts | 44 ++ .../src/tasks/on_run_restoration_handler.ts | 30 ++ .../src/tasks/on_sync_data_handler.ts | 64 +++ .../src/tasks/sync_data_task_handler.ts | 35 ++ .../legacy/functions/src/utils/big_query.ts | 124 ++++++ .../legacy/functions/src/utils/database.ts | 45 ++ .../src/utils/firestore_serializer.ts | 149 +++++++ .../functions/src/utils/import_export.ts | 77 ++++ .../legacy/functions/src/utils/serialize.ts | 41 ++ .../legacy/functions/tsconfig.dev.json | 3 + .../legacy/functions/tsconfig.json | 14 + .../functions/build_dataflow_template.sh | 14 + .../functions/download_restore_firestore.sh | 42 ++ .../legacy/install/functions/enable_pitr.sh | 9 + .../functions/setup_artifact_registry.sh | 18 + .../install/functions/setup_firestore.sh | 18 + .../functions/setup_service_account.sh | 71 +++ .../legacy/install/run.sh | 83 ++++ 46 files changed, 3137 insertions(+) create mode 100644 kits/firestore-incremental-capture/legacy/CHANGELOG.md create mode 100644 kits/firestore-incremental-capture/legacy/POSTINSTALL.md create mode 100644 kits/firestore-incremental-capture/legacy/PREINSTALL.md create mode 100644 kits/firestore-incremental-capture/legacy/README.md create mode 100644 kits/firestore-incremental-capture/legacy/extension.yaml create mode 100644 kits/firestore-incremental-capture/legacy/functions/.gitignore create mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/backupDatabase.test.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/firestoreSerializer.test.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/functions.test.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/helpers.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/backup-test.js create mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/createTestData.js create mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/exportfromBQ.js create mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/tsconfig.json create mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/types.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/jest.config.js create mode 100644 kits/firestore-incremental-capture/legacy/functions/package.json create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/config.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/constants/bq_backup_schema.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/dataflow/build_flex_template.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/dataflow/cloud_build.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/dataflow/on_complete_handler.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/dataflow/trigger_dataflow_job.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/index.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/logs.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/tasks/on_backup_restore_handler.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/tasks/on_firestore_backup_init_handler.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/tasks/on_http_run_restoration_handler.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_initial_setup_handler.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_restoration_handler.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/tasks/on_sync_data_handler.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/tasks/sync_data_task_handler.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/utils/big_query.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/utils/database.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/utils/firestore_serializer.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/utils/import_export.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/src/utils/serialize.ts create mode 100644 kits/firestore-incremental-capture/legacy/functions/tsconfig.dev.json create mode 100644 kits/firestore-incremental-capture/legacy/functions/tsconfig.json create mode 100644 kits/firestore-incremental-capture/legacy/install/functions/build_dataflow_template.sh create mode 100644 kits/firestore-incremental-capture/legacy/install/functions/download_restore_firestore.sh create mode 100644 kits/firestore-incremental-capture/legacy/install/functions/enable_pitr.sh create mode 100644 kits/firestore-incremental-capture/legacy/install/functions/setup_artifact_registry.sh create mode 100644 kits/firestore-incremental-capture/legacy/install/functions/setup_firestore.sh create mode 100644 kits/firestore-incremental-capture/legacy/install/functions/setup_service_account.sh create mode 100755 kits/firestore-incremental-capture/legacy/install/run.sh diff --git a/kits/firestore-incremental-capture/legacy/CHANGELOG.md b/kits/firestore-incremental-capture/legacy/CHANGELOG.md new file mode 100644 index 000000000..502298cff --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/CHANGELOG.md @@ -0,0 +1,52 @@ +## Version 0.0.12 + +chore: complete runtime migration to Node.js 22 + +## Version 0.0.11 + +chore: bump runtime to Node.js 22 +chore: npm run audit + +## Version 0.0.10 + +chore: bump dependencies to fix vulnerabilities + +## Version 0.0.9 + +chore: bump dependencies + +## Version 0.0.8 + +chore: update and audit packages + +## Version 0.0.7 + +fixed: bump to nodejs20 runtime in functions and run npm audit fix + +fixed: support new default bucket suffix + +## Version 0.0.6 + +fixed - deployment, documentation and scripting updates + +## Version 0.0.5 + +docs: fix POSTINSTALL instruction scripts, improve backup instance id param and regexes + +## Version 0.0.4 + +docs: update PREINSTALL, display name, and icon. + +refactor: removed legacy code + +## Version 0.0.3 + +docs: Add author and contributors field, add license headers + +## Version 0.0.2 + +docs: Add to the PREINSTALL.md and generate README.md + +## Version 0.0.1 + +Initial release of the firestore-incremental-capture extension. diff --git a/kits/firestore-incremental-capture/legacy/POSTINSTALL.md b/kits/firestore-incremental-capture/legacy/POSTINSTALL.md new file mode 100644 index 000000000..e178caf38 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/POSTINSTALL.md @@ -0,0 +1,104 @@ +## Enable PITR in the Google Cloud Console + +Follow the guidelines here [here](https://firebase.google.com/docs/firestore/use-pitr#gcloud) to enable PITR on your current database. + +## Creating a secondary Firestore database + +```bash + gcloud alpha firestore databases create --database=DATABASE_ID --location=LOCATION --type=firestore-native --project=${param:PROJECT_ID} +``` + +More information on this can be found [here](https://cloud.google.com/sdk/gcloud/reference/alpha/firestore/databases/create) + +## Building the Dataflow Flex Template + +Before this extension can run restoration jobs from BigQuery to Firestore, you must build the Dataflow Flex Template. This is a one-time process that you must perform before you can use the extension. + +We have detailed the steps below, or there is a single script you can run which will perform all the steps for you [here](https://github.com/GoogleCloudPlatform/firebase-extensions/blob/main/firestore-incremental-capture/install/run.sh). + +1. Find your extensions's service account email: + + ```bash + gcloud iam service-accounts list --format="value(EMAIL)" --filter="displayName='Firebase Extensions ${param:EXT_INSTANCE_ID} service account' AND DISABLED=False" --project="${param:PROJECT_ID}" + ``` + + You can also do this through the console, by navigating to https://console.cloud.google.com/iam-admin/serviceaccounts?authuser=0&project=${param:PROJECT_ID} + +2. [Configure the Artificat Registery](https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates?hl=en#configure): + +```bash + gcloud artifacts repositories create ${param:EXT_INSTANCE_ID} \ + --repository-format=docker \ + --location=${param:LOCATION} \ + --project=${param:PROJECT_ID} \ + --async +``` + +Configure Docker to authenticate requests for Artifact Registry: + +```bash +gcloud auth configure-docker ${param:LOCATION}-docker.pkg.dev +``` + +3. Add required policy binding for the repository: + +```bash + gcloud artifacts repositories add-iam-policy-binding ${param:EXT_INSTANCE_ID} \ + --location=${param:LOCATION} \ + --project=${param:PROJECT_ID} \ + --member=serviceAccount:SERVICE_ACCOUNT_EMAIL \ + --role=roles/artifactregistry.writer +``` + +4. Add the required role for the extension service account to trigger Dataflow: + + ```bash + gcloud projects add-iam-policy-binding ${param:PROJECT_ID} \ + --project ${param:PROJECT_ID} \ + --member=serviceAccount:SA_EMAIL \ + --role=roles/dataflow.developer + ``` + +5. Add the required role for the extension service account to trigger Dataflow: + + ```bash + gcloud projects add-iam-policy-binding ${param:PROJECT_ID} \ + --project ${param:PROJECT_ID} \ + --member=serviceAccount:SA_EMAIL \ + --role=roles/iam.serviceAccountUser + ``` + +6. Add the required role for the extension service account to trigger Dataflow: + + ```bash + gcloud projects add-iam-policy-binding ${param:PROJECT_ID} \ + --project ${param:PROJECT_ID} \ + --member=serviceAccount:SA_EMAIL \ + --role=roles/artifactregistry.writer + ``` + +7. Download the JAR file for the Dataflow Flex Template [here](https://github.com/GoogleCloudPlatform/firebase-extensions/tree/main/firestore-incremental-capture-pipeline/target/restore-firestore.jar). +8. Run the following command to build the Dataflow Flex Template. Note that Cloud Storage buckets provisioned after September 30th 2024 are suffixed by `.firebasestorage.app` rather than `.appspot.com` and you should change the following command accordingly: + +```bash + gcloud dataflow flex-template build gs://${param:PROJECT_ID}.appspot.com/${param:EXT_INSTANCE_ID}-dataflow-restore \ + --image-gcr-path ${param:LOCATION}-docker.pkg.dev/${param:PROJECT_ID}/${param:EXT_INSTANCE_ID}/dataflow/restore:latest \ + --sdk-language JAVA \ + --flex-template-base-image JAVA11 \ + --jar /path/to/restore-firestore.jar \ + --env FLEX_TEMPLATE_JAVA_MAIN_CLASS="com.pipeline.RestorationPipeline" \ + --project ${param:PROJECT_ID} +``` + +## Triggering a restoration job + +You can trigger a restoration job by calling the `restoreFirestore` function [here](https://${LOCATION}-${PROJECT_ID}.cloudfunctions.net/${EXT_INSTANCE_ID}). + +Here is an example that will run from one hour ago: + +```bash +curl -m 70 -X POST https://us-central1-${PROJECT_ID}.cloudfunctions.net/ext-firestore-incremental-capture-onHttpRunRestoration \ +-H "Authorization: bearer $(gcloud auth print-identity-token)" \ +-H "Content-Type: application/json" \ +-d "{\"timestamp\":$(date -u -v-1H +%s)}" +``` diff --git a/kits/firestore-incremental-capture/legacy/PREINSTALL.md b/kits/firestore-incremental-capture/legacy/PREINSTALL.md new file mode 100644 index 000000000..914f90252 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/PREINSTALL.md @@ -0,0 +1,53 @@ +This extension provides an automated, incremental backup solution that extends native Firestore capabilities. Generally, you should consider [Firestore’s native Point in Time Recovery](https://firebase.google.com/docs/firestore/use-pitr) and [Scheduled Backups](https://cloud.google.com/firestore/docs/backups) solutions as a first option. However, if those features don’t meet your needs, this extension can be a more flexible alternative. + +This extension provides an automated, incremental backup solution that extends native Firestore capabilities. Generally, you should consider Firestore’s native Point in Time Recovery and Scheduled Backups solution as a first option. However, if those features don’t meet your needs, this extension can be a more flexible alternative. + +With this extension, you can capture and retain incremental changes in Firestore for up to 30 days or more, allowing for point-in-time recovery well beyond the default 7-day window. + +The extension captures changes on every Firestore write and stores the change incrementally in BigQuery. This data capture mechanism ensures a complete history is maintained, enabling recovery to any point within the configured backup period. + +You can choose to incrementally capture a single collection, a collection group using wildcards, or an entire Firestore database. + +The extension also provides a Dataflow connector that can incrementally restore data from BigQuery to Firestore. Installation is done through a simple script that needs to be executed by you, and instructions to do this are provided upon installation. After installation, triggering the restoration is as simple as calling a Cloud Function. + +This extension is subject to [BigQuery write throughput limitations and availability limitations](https://cloud.google.com/bigquery/quotas), as well as [Cloud Functions at-least-once delivery guarantee](https://cloud.google.com/functions/docs/concepts/execution-environment). Since data is mirrored into BigQuery through Cloud Events, it is recommended to restore to timestamp prior to the current time to prevent missing data. + +## Additional Setup + +Before this extension can run restoration jobs from BigQuery to Firestore, you’ll need to: + +- [Set up Cloud Firestore in your Firebase project](https://firebase.google.com/docs/firestore/quickstart). +- [Enable PiTR in your Firestore database instance](https://firebase.google.com/docs/firestore/use-pitr) +- Ensure that a separate Firestore instance exists. A valid database must exist for the restoration to backup to. Ensure that a separate Firestore instance exists If one does not exist, you can create one with the following script: + +```bash + gcloud alpha firestore databases create \ + --database=DATABASE_ID \ + --location=LOCATION \ + --type=firestore-native \ + --project=PROJECT_ID +``` + +(Note that this extension currently only works on database instances in `firestore-native` mode). + +Further instructions are provided upon installation. + +### Billing + +To install an extension, your project must be on the Blaze (pay as you go) plan. You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). +This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service's no-cost tier: + +- Dataflow +- BigQuery +- Artifact Registry +- Cloud EventArc +- Cloud Functions (See [FAQs](https://firebase.google.com/support/faq#extensions-pricing)) + +[Learn more about Firebase billing](https://firebase.google.com/pricing). + +### Additional Uninstall Steps + +> ⚠️ The extension does not delete various resources automatically on uninstall. + +After you have uninstalled this extension, you will be required to remove the dataflow pipeline which was set up. You can do this through the +Google Cloud Console [here](https://console.cloud.google.com/dataflow/pipelines). This extension will also create artifacts stored in the Artifact Registry, which you can also manage from the console [here](https://console.cloud.google.com/artifacts). diff --git a/kits/firestore-incremental-capture/legacy/README.md b/kits/firestore-incremental-capture/legacy/README.md new file mode 100644 index 000000000..d43f3374f --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/README.md @@ -0,0 +1,116 @@ +# Firestore Incremental Backup Stream + +**Author**: Google Cloud (**[https://cloud.google.com/](https://cloud.google.com/)**) + +**Description**: Offers a cost-effective, flexible disaster recovery mechanism for Firestore. + + + +**Details**: This extension provides an automated, incremental backup solution that extends native Firestore capabilities. Generally, you should consider [Firestore’s native Point in Time Recovery](https://firebase.google.com/docs/firestore/use-pitr) and [Scheduled Backups](https://cloud.google.com/firestore/docs/backups) solutions as a first option. However, if those features don’t meet your needs, this extension can be a more flexible alternative. + +This extension provides an automated, incremental backup solution that extends native Firestore capabilities. Generally, you should consider Firestore’s native Point in Time Recovery and Scheduled Backups solution as a first option. However, if those features don’t meet your needs, this extension can be a more flexible alternative. + +With this extension, you can capture and retain incremental changes in Firestore for up to 30 days or more, allowing for point-in-time recovery well beyond the default 7-day window. + +The extension captures changes on every Firestore write and stores the change incrementally in BigQuery. This data capture mechanism ensures a complete history is maintained, enabling recovery to any point within the configured backup period. + +You can choose to incrementally capture a single collection, a collection group using wildcards, or an entire Firestore database. + +The extension also provides a Dataflow connector that can incrementally restore data from BigQuery to Firestore. Installation is done through a simple script that needs to be executed by you, and instructions to do this are provided upon installation. After installation, triggering the restoration is as simple as calling a Cloud Function. + +This extension is subject to [BigQuery write throughput limitations and availability limitations](https://cloud.google.com/bigquery/quotas), as well as [Cloud Functions at-least-once delivery guarantee](https://cloud.google.com/functions/docs/concepts/execution-environment). Since data is mirrored into BigQuery through Cloud Events, it is recommended to restore to timestamp prior to the current time to prevent missing data. + +## Additional Setup + +Before this extension can run restoration jobs from BigQuery to Firestore, you’ll need to: + +- [Set up Cloud Firestore in your Firebase project](https://firebase.google.com/docs/firestore/quickstart). +- [Enable PiTR in your Firestore database instance](https://firebase.google.com/docs/firestore/use-pitr) +- Ensure that a separate Firestore instance exists. A valid database must exist for the restoration to backup to. Ensure that a separate Firestore instance exists If one does not exist, you can create one with the following script: + +```bash + gcloud alpha firestore databases create \ + --database=DATABASE_ID \ + --location=LOCATION \ + --type=firestore-native \ + --project=PROJECT_ID +``` + +(Note that this extension currently only works on database instances in `firestore-native` mode). + +Further instructions are provided upon installation. + +### Billing + +To install an extension, your project must be on the Blaze (pay as you go) plan. You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). +This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service's no-cost tier: + +- Dataflow +- BigQuery +- Artifact Registry +- Cloud EventArc +- Cloud Functions (See [FAQs](https://firebase.google.com/support/faq#extensions-pricing)) + +[Learn more about Firebase billing](https://firebase.google.com/pricing). + +### Additional Uninstall Steps + +> ⚠️ The extension does not delete various resources automatically on uninstall. + +After you have uninstalled this extension, you will be required to remove the dataflow pipeline which was set up. You can do this through the +Google Cloud Console [here](https://console.cloud.google.com/dataflow/pipelines). This extension will also create artifacts stored in the Artifact Registry, which you can also manage from the console [here](https://console.cloud.google.com/artifacts). + + + + +**Configuration Parameters:** + +* Cloud Functions location: Where do you want to deploy the functions created for this extension? You usually want a location close to your database. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). + +* Collection path: What is the path to the collection that contains the strings that you want to capture all changes of? Use `{document=**}` to capture all collections. + + +* Bigquery dataset Id: The id of the Bigquery dataset to sync data to. + + +* Bigquery table Id: The id of the Bigquery table to sync data to. + + +* Backup instance Id: The name of the Firestore instance to backup the database to. + + + + +**Cloud Functions:** + +* **runInitialSetup:** Creates the backup BigQuery database if it does not exist + +* **syncData:** Enqueues a task to sync data to BigQuery + +* **syncDataTask:** Distributed cloud task for syncing data to BigQuery + +* **onHttpRunRestoration:** Starts a new restoration task + +* **onBackupRestore:** Exports data from storage to a pre-defined Firestore instance. + + + +**APIs Used**: + +* eventarc.googleapis.com (Reason: Powers all events and triggers) + +* bigquery.googleapis.com (Reason: Running queries) + +* dataflow.googleapis.com (Reason: Running dataflow jobs) + + + +**Access Required**: + + + +This extension will operate with the following project IAM roles: + +* datastore.user (Reason: Allows the extension to write updates to the database.) + +* bigquery.dataEditor (Reason: Allows the creation of BQ jobs to import Firestore backups.) diff --git a/kits/firestore-incremental-capture/legacy/extension.yaml b/kits/firestore-incremental-capture/legacy/extension.yaml new file mode 100644 index 000000000..885b96667 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/extension.yaml @@ -0,0 +1,223 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: firestore-incremental-capture +version: 0.0.12 +specVersion: v1beta + +icon: icon.png + +displayName: Firestore Incremental Backup Stream +description: + Offers a cost-effective, flexible disaster recovery mechanism for Firestore. + +license: Apache-2.0 + +author: + authorName: Google Cloud + url: https://cloud.google.com/ + +contributors: + - authorName: Invertase + email: oss@invertase.io + url: https://github.com/invertase + +sourceUrl: https://github.com/GoogleCloudPlatform/firebase-extensions/tree/main/ +releaseNotesUrl: https://github.com/GoogleCloudPlatform/firebase-extensions/tree/main/ + +apis: + - apiName: eventarc.googleapis.com + reason: Powers all events and triggers + + - apiName: bigquery.googleapis.com + reason: Running queries + + - apiName: dataflow.googleapis.com + reason: Running dataflow jobs + +roles: + - role: datastore.user + reason: Allows the extension to write updates to the database. + + - role: bigquery.dataEditor + reason: Allows the creation of BQ jobs to import Firestore backups. + + # - role: dataflow.developer + # reason: Allows this extension to create and run dataflow jobs. + + # - role: artifactregistry.writer + # reason: Allows this extension to write to the artifact registry. + +billingRequired: true + +resources: + - name: runInitialSetup + type: firebaseextensions.v1beta.function + description: >- + Creates the backup BigQuery database if it does not exist + properties: + availableMemoryMb: 512 + location: ${LOCATION} + runtime: nodejs22 + timeout: 540s + taskQueueTrigger: {} + + - name: syncData + type: firebaseextensions.v1beta.function + description: Enqueues a task to sync data to BigQuery + properties: + runtime: nodejs22 + location: ${LOCATION} + eventTrigger: + eventType: providers/cloud.firestore/eventTypes/document.write + resource: projects/${param:PROJECT_ID}/databases/(default)/documents/${param:SYNC_COLLECTION_PATH}/{docId} + + - name: syncDataTask + type: firebaseextensions.v1beta.function + description: >- + Distributed cloud task for syncing data to BigQuery + properties: + availableMemoryMb: 512 + location: ${LOCATION} + runtime: nodejs22 + timeout: 540s + taskQueueTrigger: {} + + - name: onHttpRunRestoration + type: firebaseextensions.v1beta.function + description: >- + Starts a new restoration task + properties: + location: ${LOCATION} + runtime: nodejs22 + httpsTrigger: {} + + # TODO change to a Firestore trigger + - name: onBackupRestore + type: firebaseextensions.v1beta.function + description: >- + Exports data from storage to a pre-defined Firestore instance. + properties: + location: ${LOCATION} + runtime: nodejs22 + availableMemoryMb: 1024 + taskQueueTrigger: {} + +params: + - param: LOCATION + label: Cloud Functions location + description: >- + Where do you want to deploy the functions created for this extension? You + usually want a location close to your database. For help selecting a + location, refer to the [location selection + guide](https://firebase.google.com/docs/functions/locations). + type: select + options: + - label: Iowa (us-central1) + value: us-central1 + - label: South Carolina (us-east1) + value: us-east1 + - label: Northern Virginia (us-east4) + value: us-east4 + - label: Los Angeles (us-west2) + value: us-west2 + - label: Salt Lake City (us-west3) + value: us-west3 + - label: Las Vegas (us-west4) + value: us-west4 + - label: Warsaw (europe-central2) + value: europe-central2 + - label: Belgium (europe-west1) + value: europe-west1 + - label: London (europe-west2) + value: europe-west2 + - label: Frankfurt (europe-west3) + value: europe-west3 + - label: Zurich (europe-west6) + value: europe-west6 + - label: Taiwan (asia-east1) + value: asia-east1 + - label: Hong Kong (asia-east2) + value: asia-east2 + - label: Tokyo (asia-northeast1) + value: asia-northeast1 + - label: Osaka (asia-northeast2) + value: asia-northeast2 + - label: Seoul (asia-northeast3) + value: asia-northeast3 + - label: Mumbai (asia-south1) + value: asia-south1 + - label: Singapore (asia-southeast1) + value: asia-southeast1 + - label: Jakarta (asia-southeast2) + value: asia-southeast2 + - label: Montreal (northamerica-northeast1) + value: northamerica-northeast1 + - label: Sao Paulo (southamerica-east1) + value: southamerica-east1 + - label: Sydney (australia-southeast1) + value: australia-southeast1 + default: us-central1 + required: true + immutable: true + + - param: SYNC_COLLECTION_PATH + label: Collection path + description: > + What is the path to the collection that contains the strings that you want + to capture all changes of? Use `{document=**}` to capture all collections. + example: users + validationRegex: "^[^/]+(/[^/]+/[^/]+)*$" + validationErrorMessage: Must be a valid Cloud Firestore Collection + required: true + + - param: SYNC_DATASET + label: Bigquery dataset Id + description: > + The id of the Bigquery dataset to sync data to. + example: backup_dataset + default: backup_dataset + validationRegex: "^[a-zA-Z0-9_]+$" + validationErrorMessage: > + BigQuery dataset IDs must be alphanumeric (plus underscores) and must be + no more than 1024 characters. + required: true + + - param: SYNC_TABLE + label: Bigquery table Id + description: > + The id of the Bigquery table to sync data to. + example: backup_table + default: backup_table + required: true + + - param: BACKUP_INSTANCE_ID + label: Backup instance Id + description: > + The name of the Firestore instance to backup the database to. + example: my-backup + validationRegex: "^[a-zA-Z][a-zA-Z0-9-]{2,61}[a-zA-Z0-9]$" + validationErrorMessage: Enter a valid instance id + required: true + +lifecycleEvents: + onInstall: + function: runInitialSetup + processingMessage: Creates the backup BigQuery database if it does not exist + onUpdate: + function: runInitialSetup + processingMessage: Creates the backup BigQuery database if it does not exist + onConfigure: + function: runInitialSetup + processingMessage: Creates the backup BigQuery database if it does not exist diff --git a/kits/firestore-incremental-capture/legacy/functions/.gitignore b/kits/firestore-incremental-capture/legacy/functions/.gitignore new file mode 100644 index 000000000..65b4c06ec --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/.gitignore @@ -0,0 +1,9 @@ +# Compiled JavaScript files +lib/**/*.js +lib/**/*.js.map + +# TypeScript v1 declaration files +typings/ + +# Node.js dependency directory +node_modules/ diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/backupDatabase.test.ts b/kits/firestore-incremental-capture/legacy/functions/__tests__/backupDatabase.test.ts new file mode 100644 index 000000000..0ec017c7a --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/__tests__/backupDatabase.test.ts @@ -0,0 +1,75 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { runInitialSetup } from "../src/index"; + +const mockQueue = jest.fn(); + +const getFunctionsMock = () => ({ + taskQueue: (functionName: string, instanceId: string) => ({ + enqueue: (data: any) => { + mockQueue(data); + return Promise.resolve(); + }, + }), +}); + +const mockSetProcessingState = jest.fn(); + +const getExtensionsMock = () => ({ + runtime: () => ({ + setProcessingState: (state: string, message: string) => + mockSetProcessingState(state, message), + }), +}); + +jest.mock("firebase-admin/functions", () => ({ + ...jest.requireActual("firebase-admin/functions"), + getFunctions: () => getFunctionsMock(), +})); + +jest.mock("firebase-admin/extensions", () => ({ + ...jest.requireActual("firebase-admin/extensions"), + getExtensions: () => getExtensionsMock(), +})); + +jest.mock("../src/config", () => ({ + default: { + table: "", + dataset: "", + datasetLocation: "us", + collectionName: "27062023", + runInitialBackup: true, + bucketName: "dev-extensions-testing.appspot.com", + }, +})); + +/** Setup project config */ +process.env.FIRESTORE_EMULATOR_HOST = "127.0.0.1:8080"; +process.env.FIREBASE_FIRESTORE_EMULATOR_ADDRESS = "127.0.0.1:8080"; +process.env.FIREBASE_AUTH_EMULATOR_HOST = "127.0.0.1:9099"; +process.env.PUBSUB_EMULATOR_HOST = "127.0.0.1:8085"; +process.env.GOOGLE_CLOUD_PROJECT = "demo-test"; +process.env.FIREBASE_STORAGE_EMULATOR_HOST = "127.0.0.1:9199"; + +/** Global vars */ + +describe("backupDatabase", () => { + test("Can backup database", async () => { + /** Run the function */ + await runInitialSetup(); + }); +}); diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/firestoreSerializer.test.ts b/kits/firestore-incremental-capture/legacy/functions/__tests__/firestoreSerializer.test.ts new file mode 100644 index 000000000..3e5783a8d --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/__tests__/firestoreSerializer.test.ts @@ -0,0 +1,418 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as admin from "firebase-admin"; + +const { Timestamp, GeoPoint } = require("@google-cloud/firestore"); + +import { verifySchema } from "./helpers"; + +process.env.FIRESTORE_EMULATOR_HOST = "127.0.0.1:8080"; +process.env.FIREBASE_FIRESTORE_EMULATOR_ADDRESS = "127.0.0.1:8080"; +process.env.FIREBASE_AUTH_EMULATOR_HOST = "127.0.0.1:9099"; +process.env.PUBSUB_EMULATOR_HOST = "127.0.0.1:8085"; +process.env.GOOGLE_CLOUD_PROJECT = "demo-project"; + +admin.initializeApp({ projectId: "demo-project" }); + +const db = admin.firestore(); + +/** + * TODO: Handle binary examples + */ + +describe("generateSchema", () => { + test("should handle an string value", async () => { + const documentPath = "products/stringExample"; + const sampleDocData = { + stringValue: "Hello, Firestore!", + }; + + await db.doc(documentPath).set(sampleDocData); + + await verifySchema(documentPath, { + stringValue: { type: "string", value: "Hello, Firestore!" }, + }); + }, 12000); + + test("should handle an boolean value", async () => { + const documentPath = "products/booleanExample"; + const sampleDocData = { + booleanExample: true, + }; + + await db.doc(documentPath).set(sampleDocData); + + await verifySchema(documentPath, { + booleanExample: { type: "boolean", value: true }, + }); + }, 12000); + + test("should handle a geopoint value", async () => { + const documentPath = "products/geoPointExample"; + const sampleDocData = { + geopointValue: new GeoPoint(52.379189, 4.899431), + }; + + await db.doc(documentPath).set(sampleDocData); + + await verifySchema(documentPath, { + geopointValue: { + type: "geopoint", + value: { + latitude: { + type: "number", + value: 52.379189, + }, + longitude: { + type: "number", + value: 4.899431, + }, + }, + }, + }); + }, 12000); + + test("should handle a document reference value", async () => { + const documentPath = "products/documentReferenceExample"; + const ref = db.doc("products/stringExample"); + const sampleDocData = { + documentReferenceValue: ref, + }; + + await db.doc(documentPath).set(sampleDocData); + + await verifySchema(documentPath, { + documentReferenceValue: { + type: "documentReference", + value: ref.path, // Assuming you want to store the path of the document reference + }, + }); + }, 12000); + + test("should handle a timestamp reference value", async () => { + const documentPath = "products/timestampExample"; + const timestampValue = Timestamp.now(); + const sampleDocData = { + timestampValue, + }; + + await db.doc(documentPath).set(sampleDocData); + + await verifySchema(documentPath, { + timestampValue: { + type: "timestamp", + value: timestampValue.toDate().toISOString(), + }, + }); + }, 12000); + + test("should handle an objectValue value", async () => { + const documentPath = "products/objectValueExample"; + const sampleDocData = { + objectValue: { foo: "bar" }, + }; + + await db.doc(documentPath).set(sampleDocData); + + await verifySchema(documentPath, { + objectValue: { + type: "map", + value: { + foo: { + type: "string", + value: "bar", + }, + }, + }, + }); + }, 12000); + + test("should handle an multiple objectValue values", async () => { + const documentPath = "products/objectValueExample"; + const sampleDocData = { + objectValue: { foo: "bar", foo2: "bar2" }, + }; + + await db.doc(documentPath).set(sampleDocData); + + await verifySchema(documentPath, { + objectValue: { + type: "map", + value: { + foo: { + type: "string", + value: "bar", + }, + foo2: { + type: "string", + value: "bar2", + }, + }, + }, + }); + }, 12000); + + test("should handle an empty array value", async () => { + const documentPath = "products/arrayValueExample"; + const sampleDocData = { + arrayValue: [], + }; + + await db.doc(documentPath).set(sampleDocData); + + await verifySchema(documentPath, { + arrayValue: { + type: "array", + value: [], + }, + }); + }, 12000); + + test("should handle an array with basic data types", async () => { + const documentPath = "products/complexExample"; + + const sampleDocData = { + arrayValue: [ + { + stringValue: "test", + integerValue: 42, + floatValue: 42.42, + booleanValue: true, + nullValue: null, + }, + ], + }; + + await db.doc(documentPath).set(sampleDocData); + + // Define the expected result according to the behavior of the flattenData function. + const expectedData = { + arrayValue: { + type: "array", + value: [ + { + stringValue: { + type: "string", + value: "test", + }, + integerValue: { + type: "number", + value: 42, + }, + floatValue: { + type: "number", + value: 42.42, + }, + booleanValue: { + type: "boolean", + value: true, + }, + nullValue: { + type: "null", + value: null, + }, + }, + ], + }, + }; + + await verifySchema(documentPath, expectedData); + }, 12000); + + test("should handle an array with complex data types", async () => { + const documentPath = "products/complexArrayExample"; + const timestampValue = Timestamp.now(); + + const sampleDocData = { + arrayValue: [ + { + nestedString: "nestedTest", + nestedNumber: 42, + nestedObject: { + deepNestedValue: "deepValue", + }, + geoPointValue: new GeoPoint(52.379189, 4.899431), + timestampValue, + }, + ], + }; + + await db.doc(documentPath).set(sampleDocData); + + // Define the expected result according to the behavior of the flattenData function. + const expectedData = { + arrayValue: { + type: "array", + value: [ + { + nestedString: { + type: "string", + value: "nestedTest", + }, + nestedNumber: { + type: "number", + value: 42, + }, + nestedObject: { + type: "map", + value: { + deepNestedValue: { + type: "string", + value: "deepValue", + }, + }, + }, + geoPointValue: { + type: "geopoint", + value: { + latitude: { + type: "number", + value: 52.379189, + }, + longitude: { + type: "number", + value: 4.899431, + }, + }, + }, + timestampValue: { + type: "timestamp", + value: timestampValue.toDate().toISOString(), + }, + }, + ], + }, + }; + + await verifySchema(documentPath, expectedData); + }, 12000); + + test("should handle arrays with mixed data types", async () => { + const documentPath = "products/mixedArrayExample"; + const sampleDocData = { + mixedArray: ["string", 42, true], + }; + + await db.doc(documentPath).set(sampleDocData); + + await verifySchema(documentPath, { + mixedArray: { + type: "array", + value: [ + { type: "string", value: "string" }, + { type: "number", value: 42 }, + { type: "boolean", value: true }, + ], + }, + }); + }, 12000); + + test("should handle standalone numbers", async () => { + const documentPath = "products/numberExample"; + const sampleDocData = { + numberValue: 42.42, + }; + + await db.doc(documentPath).set(sampleDocData); + + await verifySchema(documentPath, { + numberValue: { + type: "number", + value: 42.42, + }, + }); + }, 12000); + + test("should handle binary blob data", async () => { + const documentPath = "products/blobExample"; + + const sampleDocData = { + blobValue: Buffer.from("some sample data", "utf8"), + }; + + db.doc(documentPath).set(sampleDocData); + await db.doc(documentPath).set(sampleDocData); + + await verifySchema(documentPath, { + blobValue: { + type: "binary", + value: Buffer.from("some sample data").toString("base64"), // Modified this line to directly use a Buffer + }, + }); + }, 12000); + + test("should handle an integer value", async () => { + const documentPath = "products/integerExample"; + + // Sample data with an integer value + const sampleDocData = { + integerValue: 12345, + }; + + // Set the data in Firestore + await db.doc(documentPath).set(sampleDocData); + + // Verify if the value in the schema (or retrieved value) matches the set value + await verifySchema(documentPath, { + integerValue: { + type: "number", + value: 12345, + }, + }); + }, 12000); + + test("should handle a floating point value", async () => { + const documentPath = "products/floatingPointExample"; + + // Sample data with a floating point value + const sampleDocData = { + floatValue: 123.45, + }; + + // Set the data in Firestore + await db.doc(documentPath).set(sampleDocData); + + // Verify if the value in the schema (or retrieved value) matches the set value + await verifySchema(documentPath, { + floatValue: { + type: "number", + value: 123.45, + }, + }); + }, 12000); + + test("should handle a null value", async () => { + const documentPath = "products/nullValueExample"; + const sampleDocData = { + nullableField: null, + }; + + // Set the data in Firestore + await db.doc(documentPath).set(sampleDocData); + + // Prepare the update using the helper function (assuming this is how you're setting up your other tests) + await db.doc(documentPath).set(sampleDocData); + + // Check against the expected schema + await verifySchema(documentPath, { + nullableField: { + type: "null", // null is a type of object in JavaScript + value: null, + }, + }); + }, 12000); +}); diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/functions.test.ts b/kits/firestore-incremental-capture/legacy/functions/__tests__/functions.test.ts new file mode 100644 index 000000000..0dd80b91c --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/__tests__/functions.test.ts @@ -0,0 +1,126 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as admin from "firebase-admin"; +import * as functions from "firebase-functions-test"; +import { syncData } from "../src/index"; +import { getTable, initialize } from "../src/bigquery"; + +import config from "../src/config"; +import { Table } from "@google-cloud/bigquery"; +import { clearBQTables } from "./helpers"; + +jest.mock("../src/config", () => ({ + default: { + table: "", + dataset: "", + datasetLocation: "us", + syncCollectionPath: "testing", + }, +})); + +/** Setup project config */ +const projectId = "dev-extensions-testing"; +const fft = functions({ projectId }); + +process.env.FIRESTORE_EMULATOR_HOST = "127.0.0.1:8080"; +process.env.FIREBASE_FIRESTORE_EMULATOR_ADDRESS = "127.0.0.1:8080"; +process.env.FIREBASE_AUTH_EMULATOR_HOST = "127.0.0.1:9099"; +process.env.PUBSUB_EMULATOR_HOST = "127.0.0.1:8085"; +process.env.GOOGLE_CLOUD_PROJECT = "demo-test"; +process.env.FIREBASE_STORAGE_EMULATOR_HOST = "127.0.0.1:9199"; + +/** Global vars */ +const { makeDocumentSnapshot } = fft.firestore; + +const db = admin.firestore(); +const collection = db.collection(config.syncCollectionPath); +let randomId = ""; + +xdescribe("functions", () => { + beforeAll(async () => { + /** clear all datasets */ + await clearBQTables(); + }); + beforeEach(async () => { + /** generate random id */ + randomId = (Math.random() + 1).toString(36).substring(7); + + config.table = randomId; + config.dataset = randomId; + + await initialize(); + }); + + xit("Can sync data with BQ", async () => { + /** Set document data */ + const doc = await collection.add({}); + const path = `${config.syncCollectionPath}/${doc.id}`; + const snap = makeDocumentSnapshot({ foo: "bar" }, path); + + /** Run the function */ + const wrapped = fft.wrap(syncData); + await wrapped(snap); + + /** check data has synced */ + const table: Table = await getTable(config.dataset, config.table); + + /** wait for 2 seconds */ + await new Promise((resolve) => setTimeout(resolve, 2000)); + + const [query] = await table.createQueryJob({ + query: `Select * from ${config.dataset}.${config.table}`, + }); + + const [results] = await query.getQueryResults(); + const { foo } = JSON.parse(results[0].data); + + expect(foo).toEqual("bar"); + }); + + it("Can replay data", async () => { + /** Set document data */ + const doc = await collection.add({}); + const path = `${config.syncCollectionPath}/${doc.id}`; + + /** Make an array of 10 items */ + const snapshots = Array.from(Array(10).keys()); + + /** Write snapshots to the database */ + const wrapped = fft.wrap(syncData); + for await (const snapshot of snapshots) { + /** Run the function */ + const bs = makeDocumentSnapshot({}, path); + const as = makeDocumentSnapshot({ foo: snapshot }, path); + const change = fft.makeChange(bs, as); + await wrapped(change); + } + + /** check data has synced */ + const table: Table = await getTable(config.dataset, config.table); + + const [query] = await table.createQueryJob({ + query: `Select * from ${config.dataset}.${config.table}`, + }); + + const [results] = await query.getQueryResults(); + const $ = JSON.parse(results[0].data); + + expect($).toEqual({ foo: 0 }); + + /** */ + }); +}); diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/helpers.ts b/kits/firestore-incremental-capture/legacy/functions/__tests__/helpers.ts new file mode 100644 index 000000000..5bfbe4434 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/__tests__/helpers.ts @@ -0,0 +1,54 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DocumentReference, DocumentSnapshot } from "firebase-admin/firestore"; +import { WrappedFirebaseFunction } from "./types"; +import { FeaturesList } from "firebase-functions-test/lib/features"; + +const { BigQuery } = require("@google-cloud/bigquery"); +const bq = new BigQuery({ projectId: "dev-extensions-testing" }); + +export const simulateFunctionTriggered = + ( + module: FeaturesList, + wrappedFunction: WrappedFirebaseFunction, + collectionName: string + ) => + async (ref: DocumentReference, before?: DocumentSnapshot) => { + const data = (await ref.get()).data() as { [key: string]: any }; + const beforeFunctionExecution = module.firestore.makeDocumentSnapshot( + data, + `${collectionName}/${ref.id}` + ) as DocumentSnapshot; + const change = module.makeChange(before, beforeFunctionExecution); + await wrappedFunction(change); + return beforeFunctionExecution; + }; + +export const clearBQTables = async () => { + const [datasets] = await bq.getDatasets({ + projectId: "dev-extensions-testing", + }); + + for await (const dataset of datasets) { + try { + await dataset.delete({ force: true }); + console.log(`Dataset ${dataset.id} deleted.`); + } catch (ex) { + console.log((ex as Error).message); + } + } +}; diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/backup-test.js b/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/backup-test.js new file mode 100644 index 000000000..ef2d19f39 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/backup-test.js @@ -0,0 +1,72 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const { Spanner } = require("@google-cloud/spanner"); +const { PreciseDate } = require("@google-cloud/precise-date"); + +(async () => { + const projectId = "dev-extensions-testing"; + const instanceId = "my-instance"; + const databaseId = "my-database"; + const backupId = "my-backup"; + const versionTime = Date.now() - 1000 * 60 * 60 * 24; // One day ago + + const spanner = new Spanner({ + projectId: projectId, + }); + + // Gets a reference to a Cloud Spanner instance and database + const instance = spanner.instance(instanceId); + const database = instance.database(databaseId); + + const backup = instance.backup(backupId); + + // Creates a new backup of the database + try { + console.log(`Creating backup of database ${database.formattedName_}.`); + const databasePath = database.formattedName_; + // Expire backup 14 days in the future + const expireTime = Date.now() + 1000 * 60 * 60 * 24 * 14; + // Create a backup of the state of the database at the current time. + const [, operation] = await backup.create({ + databasePath: databasePath, + expireTime: expireTime, + versionTime: versionTime, + }); + + console.log(`Waiting for backup ${backup.formattedName_} to complete...`); + await operation.promise(); + + // Verify backup is ready + const [backupInfo] = await backup.getMetadata(); + if (backupInfo.state === "READY") { + console.log( + `Backup ${backupInfo.name} of size ` + + `${backupInfo.sizeBytes} bytes was created at ` + + `${new PreciseDate(backupInfo.createTime).toISOString()} ` + + "for version of database at " + + `${new PreciseDate(backupInfo.versionTime).toISOString()}` + ); + } else { + console.error("ERROR: Backup is not ready."); + } + } catch (err) { + console.error("ERROR:", err); + } finally { + // Close the database when finished. + await database.close(); + } +})(); diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/createTestData.js b/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/createTestData.js new file mode 100644 index 000000000..286fe69c5 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/createTestData.js @@ -0,0 +1,74 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const admin = require("firebase-admin"); + +admin.initializeApp({ projectId: "dev-extensions-testing" }); + +/** add a root level collections */ +const rootCollection = admin.firestore().collection("sync"); + +(async () => { + const rootCollectionDocument = await rootCollection.add({ + name: "Sample Document", + description: "This is a sample document for reference.", + }); + + /** Set the reference and Wait 5 seconds */ + const subCollectionRef = rootCollection.doc().collection("subCollection"); + await new Promise((resolve) => setTimeout(resolve, 5000)); + + /** Add a sub collection document */ + let i = "first update"; + + await subCollectionRef.add({ + stringField: `This is a string ${i}`, + numberField: 12345 + i, + booleanField: i % 2 === 0, // Will alternate between true and false + arrayField: ["apple", "banana", "cherry"], + dateField: new Date(), + nullField: null, + objectField: { + subString: `Sub object string ${i}`, + subNumber: 67890 + i, + }, + geopointField: new admin.firestore.GeoPoint(34.0522, -118.2437), // This represents LA latitude and longitude + referenceField: rootCollectionDocument, + }); + + /** Wait 5 seconds */ + await new Promise((resolve) => setTimeout(resolve, 5000)); + + i = "second update"; + + await subCollectionRef.add({ + stringField: `This is a string ${i}`, + numberField: 12345 + i, + booleanField: i % 2 === 0, // Will alternate between true and false + arrayField: ["apple", "banana", "cherry"], + dateField: new Date(), + nullField: null, + objectField: { + subString: `Sub object string ${i}`, + subNumber: 67890 + i, + }, + geopointField: new admin.firestore.GeoPoint(34.0522, -118.2437), // This represents LA latitude and longitude + referenceField: rootCollectionDocument, + }); + + /** Wait 30 seconds */ + await new Promise((resolve) => setTimeout(resolve, 5000)); +})(); diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/exportfromBQ.js b/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/exportfromBQ.js new file mode 100644 index 000000000..8231920c8 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/exportfromBQ.js @@ -0,0 +1,30 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const { BigQuery } = require("@google-cloud/bigquery"); + +const bq = new BigQuery({ projectId: "dev-extensions-testing" }); + +(async () => { + /** Get all the records from before 2023-08-22 13:23 */ + const query = + "SELECT * FROM `dev-extensions-testing.syncData.syncData` WHERE timestamp < TIMESTAMP('2023-08-22 13:23:00')"; + + /** Execute the query */ + await bq.query(query).then((data) => { + console.log(data); + }); +})(); diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/tsconfig.json b/kits/firestore-incremental-capture/legacy/functions/__tests__/tsconfig.json new file mode 100644 index 000000000..379a994d8 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/__tests__/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../tsconfig.json", + "include": ["."] +} diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/types.ts b/kits/firestore-incremental-capture/legacy/functions/__tests__/types.ts new file mode 100644 index 000000000..655f93a1d --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/__tests__/types.ts @@ -0,0 +1,24 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DocumentSnapshot } from "firebase-admin/firestore"; +import { WrappedFunction } from "firebase-functions-test/lib/v1"; +import { Change } from "firebase-functions/v1"; + +export type WrappedFirebaseFunction = WrappedFunction< + Change, + void +>; diff --git a/kits/firestore-incremental-capture/legacy/functions/jest.config.js b/kits/firestore-incremental-capture/legacy/functions/jest.config.js new file mode 100644 index 000000000..d936ac9c5 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/jest.config.js @@ -0,0 +1,32 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const packageJson = require("./package.json"); + +module.exports = { + name: packageJson.name, + displayName: packageJson.name, + rootDir: "./", + globals: { + "ts-jest": { + tsConfig: "/__tests__/tsconfig.json", + }, + }, + testMatch: ["**/__tests__/*.test.ts"], + testPathIgnorePatterns: ["manualTesting"], + testEnvironment: "node", + preset: "ts-jest", +}; diff --git a/kits/firestore-incremental-capture/legacy/functions/package.json b/kits/firestore-incremental-capture/legacy/functions/package.json new file mode 100644 index 000000000..f646de75d --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/package.json @@ -0,0 +1,48 @@ +{ + "name": "functions", + "scripts": { + "prepare": "npm run build", + "lint": "eslint --ext .js,.ts .", + "build": "tsc", + "build:watch": "tsc --watch", + "serve": "npm run build && firebase emulators:start --only functions", + "shell": "npm run build && firebase functions:shell", + "start": "npm run shell", + "deploy": "firebase deploy --only functions", + "logs": "firebase functions:log", + "generate-readme": "firebase ext:info .. --markdown > ../README.md", + "publish-from-main": "firebase ext:dev:upload googlecloud/firestore-incremental-capture --repo=https://github.com/googlecloudplatform/firebase-extensions --root=firestore-incremental-capture --ref=main --project pub-ext-gcloud" + }, + "engines": { + "node": "22" + }, + "main": "lib/index.js", + "dependencies": { + "@google-cloud/bigquery": "^7.1.1", + "@google-cloud/cloudbuild": "^4.0.1", + "@google-cloud/dataflow": "^3.0.1", + "@types/traverse": "^0.6.37", + "firebase-admin": "^12.2.0", + "firebase-functions": "^4.3.1", + "jest": "^29.6.2", + "traverse": "^0.6.11", + "ts-jest": "^29.4.0" + }, + "devDependencies": { + "@google-cloud/firestore": "^7.11.2", + "@google-cloud/precise-date": "^4.0.0", + "@google-cloud/spanner": "^7.0.0", + "@typescript-eslint/eslint-plugin": "^8.54.0", + "@typescript-eslint/parser": "^8.54.0", + "dotenv": "^16.3.1", + "eslint": "^9.39.2", + "eslint-config-google": "^0.14.0", + "eslint-plugin-import": "^2.32.0", + "firebase-functions-test": "^3.4.1", + "typescript": "^4.9.0" + }, + "overrides": { + "fast-xml-parser": "^5.3.4" + }, + "private": true +} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/config.ts b/kits/firestore-incremental-capture/legacy/functions/src/config.ts new file mode 100644 index 000000000..7ab342130 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/config.ts @@ -0,0 +1,71 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as admin from "firebase-admin"; + +admin.initializeApp(); + +const projectId = process.env.PROJECT_ID!; +const instanceId = process.env.EXT_INSTANCE_ID!; +const location = process.env.LOCATION!; +const backupInstance = process.env.BACKUP_INSTANCE_ID!; +const backupInstanceFullId = `projects/${projectId}/databases/${backupInstance}`; + +const getDefaultBucket = (): string => { + try { + // Try to get the default bucket from Firebase Admin + const defaultBucket = admin.storage().bucket().name; + console.log(`Using detected default bucket: ${defaultBucket}`); + return process.env.BUCKET_NAME || defaultBucket; + } catch (error) { + // Fallback to the environment variable or construct using project ID + console.log( + `Could not detect default bucket, using fallback: ${projectId}.appspot.com` + ); + return process.env.BUCKET_NAME || `${projectId}.appspot.com`; + } +}; + +const bucketName = getDefaultBucket(); + +export { admin }; // Export admin to use in other files + +export default { + projectId, + instanceId, + bucketName, + location, + bucketPath: "backups", + datasetLocation: "us", + runInitialBackup: true, + + instanceCollection: `_ext-${process.env.EXT_INSTANCE_ID!}`, + statusDoc: `_ext-${process.env.EXT_INSTANCE_ID!}/status`, + backupDoc: `_ext-${process.env.EXT_INSTANCE_ID!}/backups`, + restoreDoc: `_ext-${process.env.EXT_INSTANCE_ID!}/restore`, + cloudBuildDoc: `_ext-${process.env.EXT_INSTANCE_ID!}/cloudBuild`, + syncCollectionPath: process.env.SYNC_COLLECTION_PATH!, + + bqDataset: process.env.SYNC_DATASET!, + bqtable: process.env.SYNC_TABLE!, + + backupInstanceName: backupInstanceFullId, + + stagingLocation: `gs://${bucketName}/${instanceId}/staging`, + templateLocation: `gs://${bucketName}/${instanceId}/templates/myTemplate`, + dataflowRegion: + process.env.DATAFLOW_REGION || process.env.LOCATION || "us-central1", +}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/constants/bq_backup_schema.ts b/kits/firestore-incremental-capture/legacy/functions/src/constants/bq_backup_schema.ts new file mode 100644 index 000000000..c1fb6934e --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/constants/bq_backup_schema.ts @@ -0,0 +1,24 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const bqBackupSchema = [ + { name: "documentId", type: "STRING", mode: "REQUIRED" }, + { name: "documentPath", type: "STRING", mode: "REQUIRED" }, + { name: "beforeData", type: "JSON" }, + { name: "afterData", type: "JSON" }, + { name: "changeType", type: "STRING", mode: "REQUIRED" }, + { name: "timestamp", type: "TIMESTAMP", mode: "REQUIRED" }, +]; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/dataflow/build_flex_template.ts b/kits/firestore-incremental-capture/legacy/functions/src/dataflow/build_flex_template.ts new file mode 100644 index 000000000..c3ac1d932 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/dataflow/build_flex_template.ts @@ -0,0 +1,54 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import config from "../config"; + +import { exec } from "child_process"; + +/** + * This function builds the flex template for the dataflow job, + * but it is not used in the current implementation. + * The reason is that gcloud CLI is not available in the cloud functions runtime, + * hence the build process cannot be automated. + * + * It is included here for reference purposes. + */ +export async function buildFlexTemplateHandler() { + const projectId = config.projectId; + const bucketName = config.bucketName; + const location = config.location; + const instanceId = config.instanceId; + + // Building JAR: mvn clean package -DskipTests -Dexec.mainClass=com.pipeline.RestorationPipeline + + exec( + `gcloud dataflow flex-template build gs://${bucketName}/dataflow-templates/${instanceId} \ + --image-gcr-path "${location}-docker.pkg.dev/${projectId}/${instanceId}/dataflow/restore:latest" \ + --sdk-language "JAVA" \ + --flex-template-base-image JAVA11 \ + --jar "path/to/pipeline.jar" \ + --env FLEX_TEMPLATE_JAVA_MAIN_CLASS="com.pipeline.RestorationPipeline" \ + --project ${projectId}`, + (err, stdout) => { + if (err) { + console.log(err); + Promise.reject(err); + } + + Promise.resolve(stdout); + } + ); +} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/dataflow/cloud_build.ts b/kits/firestore-incremental-capture/legacy/functions/src/dataflow/cloud_build.ts new file mode 100644 index 000000000..4bfc3d57c --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/dataflow/cloud_build.ts @@ -0,0 +1,95 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { logger } from "firebase-functions/v1"; +import config from "../config"; +import { CloudBuildClient } from "@google-cloud/cloudbuild"; + +const cloneStep = { + name: "gcr.io/cloud-builders/git", + args: [ + "clone", + "https://github.com/GoogleCloudPlatform/firebase-extensions.git", + ], +}; + +const checkoutStep = { + name: "gcr.io/cloud-builders/git", + args: ["checkout", "@invertase/firestore-incremental-capture"], + dir: "firebase-extensions", +}; + +const buildStep = { + name: "maven:3.8.1-openjdk-11", + args: [ + "mvn", + "compile", + "exec:java", + "-Dexec.mainClass=com.pipeline.RestorationPipeline", + `-Dexec.args=--runner=DataflowRunner --project=${config.projectId} --stagingLocation=${config.stagingLocation} --templateLocation=${config.templateLocation} --region=${config.dataflowRegion}`, + ], + dir: "firebase-extensions/firestore-incremental-capture/functions/pipeline", +}; + +// const notifyStep = { +// name: 'gcr.io/cloud-builders/curl', +// entrypoint: 'bash', +// args: [ +// '-c', +// `curl -X POST -H "Content-Type: application/json" -d \'{"status": "$BUILD_STATUS", "build_id": "$BUILD_ID"}\' https://${config.instanceId}/onCloudBuildComplete`, +// ], +// }; + +const client = new CloudBuildClient(); + +/** + * Builds the template for the dataflow pipeline, return the LROperation name + */ +export const stageTemplate = async () => { + logger.info("Staging template"); + const build_id = `${config.instanceId}-dataflow-template-${Date.now()}`; + + const [operation] = await client.createBuild({ + projectId: config.projectId, + build: { + name: build_id, + id: build_id, + steps: [cloneStep, checkoutStep, buildStep], + }, + }); + + logger.info(`Build created: ${operation.name}`); + + if (operation.error) { + throw new Error(operation.error.message); + } + return operation; +}; + +/** + * Regularly ping the import operation to check for completion + */ +export async function WaitForCreateBuildCompletion(name: string) { + logger.log("Checking for create build progress: ", name); + const response = await client.checkCreateBuildProgress(name); + if (!response.done) { + // Wait for 1 minute retrying + await new Promise((resolve) => setTimeout(resolve, 60000)); + /** try again */ + await WaitForCreateBuildCompletion(name); + } + return Promise.resolve(response); +} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/dataflow/on_complete_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/dataflow/on_complete_handler.ts new file mode 100644 index 000000000..08681ea87 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/dataflow/on_complete_handler.ts @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as admin from "firebase-admin"; +import { logger } from "firebase-functions/v1"; + +import config from "../config"; + +export async function onCompleteHandler(payload: any) { + logger.info("build event completed!"); + logger.info(`Message ===> ${JSON.stringify(payload)}`); + + await admin + .firestore() + .doc(config.cloudBuildDoc) + .update({ status: "staged", ...payload }); +} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/dataflow/trigger_dataflow_job.ts b/kits/firestore-incremental-capture/legacy/functions/src/dataflow/trigger_dataflow_job.ts new file mode 100644 index 000000000..d32016fff --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/dataflow/trigger_dataflow_job.ts @@ -0,0 +1,71 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as admin from "firebase-admin"; +import { logger } from "firebase-functions/v1"; +import { FlexTemplatesServiceClient } from "@google-cloud/dataflow"; +import { Timestamp } from "firebase-admin/firestore"; + +import config from "../config"; + +const dataflowClient = new FlexTemplatesServiceClient(); + +export async function launchJob(timestamp: number) { + const projectId = config.projectId; + const serverTimestamp = Timestamp.now().toMillis(); + const { syncCollectionPath } = config; + + const runId = `${config.instanceId}-dataflow-run-${serverTimestamp}`; + + logger.info(`Launching job ${runId}`, { + labels: { run_id: runId }, + }); + + const runDoc = admin.firestore().doc(`restore/${runId}`); + + // Extract the database name from the backup instance name + const values = config.backupInstanceName.split("/"); + const firestoreDb = values[values.length - 1]; + + /** Select the correct collection Id for apache beam */ + const firestoreCollectionId = + syncCollectionPath === "{document=**}" ? "*" : syncCollectionPath; + + const [response] = await dataflowClient.launchFlexTemplate({ + projectId, + location: config.location, + launchParameter: { + jobName: runId, + parameters: { + timestamp: timestamp.toString(), + firestoreCollectionId, + firestoreDb, + bigQueryDataset: config.bqDataset, + bigQueryTable: config.bqtable, + }, + containerSpecGcsPath: `gs://${config.bucketName}/${config.instanceId}-dataflow-restore`, + }, + }); + + await runDoc.set({ status: "export triggered", runId: runId }); + + logger.info(`Launched job named ${response.job?.name} successfully`, { + job_response: response, + labels: { run_id: runId }, + }); + + return response; +} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/index.ts b/kits/firestore-incremental-capture/legacy/functions/src/index.ts new file mode 100644 index 000000000..dc1045424 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/index.ts @@ -0,0 +1,75 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as functions from "firebase-functions"; + +import config from "./config"; + +import { syncDataHandler } from "./tasks/on_sync_data_handler"; +import { onCompleteHandler } from "./dataflow/on_complete_handler"; +import { syncDataTaskHandler } from "./tasks/sync_data_task_handler"; +import { buildFlexTemplateHandler } from "./dataflow/build_flex_template"; +import { onBackupRestoreHandler } from "./tasks/on_backup_restore_handler"; +import { runInitialSetupHandler } from "./tasks/on_run_initial_setup_handler"; +import { onHttpRunRestorationHandler } from "./tasks/on_http_run_restoration_handler"; +import { onFirestoreBackupInitHandler } from "./tasks/on_firestore_backup_init_handler"; + +/** + * Sync data to BigQuery, triggered by any change to a Firestore document + * */ +export const syncData = functions.firestore + .document(config.syncCollectionPath) + .onWrite(syncDataHandler); + +/** + * Cloud task to handle data sync + * */ +export const syncDataTask = functions.tasks + .taskQueue() + .onDispatch(syncDataTaskHandler); + +/** + * Backup the entire database on initial deployment + * */ +export const runInitialSetup = async () => await runInitialSetupHandler(); + +/** + * Run a backup restoration. + * */ +export const onHttpRunRestoration = functions.https.onRequest( + onHttpRunRestorationHandler +); + +export const onBackupRestore = functions.tasks + .taskQueue() + .onDispatch(onBackupRestoreHandler); + +/** + * Cloud task for handling database restoration + * */ +export const onFirestoreBackupInit = functions.tasks + .taskQueue() + .onDispatch(onFirestoreBackupInitHandler); + +/** + * Cloud task for staging the dataflow template + * */ +export const buildFlexTemplate = functions.tasks + .taskQueue() + .onDispatch(buildFlexTemplateHandler); + +export const onCloudBuildComplete = + functions.https.onRequest(onCompleteHandler); diff --git a/kits/firestore-incremental-capture/legacy/functions/src/logs.ts b/kits/firestore-incremental-capture/legacy/functions/src/logs.ts new file mode 100644 index 000000000..7cc69c452 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/logs.ts @@ -0,0 +1,49 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { logger } from "firebase-functions"; + +export const bigQueryDatasetExists = (dataset: string) => { + logger.log(`${dataset} already exists`); +}; + +export const bigQueryTableExists = (dataset: string) => { + logger.log(`${dataset} already exists`); +}; + +export const bigQueryDatasetCreating = (dataset: string) => { + logger.log(`Creating dataset: ${dataset}`); +}; + +export const bigQueryTableCreating = (table: string) => { + logger.log(`Creating table: ${table}`); +}; + +export const bigQueryDatasetCreated = (dataset: string) => { + logger.log(`successfully created dataset: ${dataset}`); +}; + +export const bigQueryTableCreated = (table: string) => { + logger.log(`successfully created table: ${table}`); +}; + +export const tableCreationError = (table: string, message: string) => { + logger.log(`error creating table: ${table}, ${message}`); +}; + +export const datasetCeationError = (dataset: string) => { + logger.log(`error creatign dataset: ${dataset}`); +}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_backup_restore_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_backup_restore_handler.ts new file mode 100644 index 000000000..75941ca70 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_backup_restore_handler.ts @@ -0,0 +1,143 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { logger } from "firebase-functions/v1"; + +import { launchJob } from "../dataflow/trigger_dataflow_job"; + +export const onBackupRestoreHandler = async (data: any) => { + const timestamp = data.timestamp as number; + + if (!isValidUnixTimestamp(timestamp)) { + logger.error( + '"timestamp" field is missing, please ensure that you are sending a valid timestamp in the request body, is in seconds since epoch and is not in the future.' + ); + return Promise.resolve(); + } + + logger.info(`Running backup restoration at PIT ${timestamp}`); + + // const importDoc = await db + // .doc(config.backupDoc) + // .collection('imports') + // .add({}); + + // // Get the latest backup collection + // const backupExportsCollection = db + // .doc(config.backupDoc) + // .collection('exports'); + + // const completedExports = await backupExportsCollection + // .where('status', '==', 'Completed') + // .get(); + + // const documents = completedExports.docs.map(doc => ({ + // id: doc.id, + // data: doc.data(), + // })); + + // // Sort documents by timestamp in descending order + // const sortedDocuments = documents.sort( + // (a, b) => b.data.timestamp.toDate() - a.data.timestamp.toDate() + // ); + + // Get the most recent document + // const doc = sortedDocuments.length > 0 ? sortedDocuments[0] : null; + + //TODO: use this version in the future, index creation is needed. + // const backupDocuments = await db + // .doc(config.backupDoc) + // .collection('exports') + // .where('status', '==', 'Completed') + // .orderBy('timestamp', 'desc') + // .limit(1) + // .get(); + + // Get the latest backup + // const backupId = doc?.id; + + // // If no backup + // if (!backupId) { + // logger.info('No backup found'); + // return Promise.resolve(); + // } + + try { + // // Export the Firestore db to storage + // const {id, operation} = await createImport(backupId); + + // // Update Firestore for tracking + // await importDoc.set({ + // id, + // status: 'Running import...', + // operation: operation.name, + // timestamp: FieldValue.serverTimestamp(), + // }); + + // // Wait for import completion + // await waitForImportCompletion(operation.name || ''); + + // await importDoc.set({ + // id, + // status: 'Initial backup restored, replaying final updates...', + // operation: operation.name, + // timestamp: FieldValue.serverTimestamp(), + // }); + + // Run DataFLow updates + await launchJob(timestamp); + + // await importDoc.set({ + // id, + // status: 'Completed', + // operation: operation.name, + // timestamp: FieldValue.serverTimestamp(), + // }); + } catch (ex: any) { + logger.error("Error restoring backup", ex); + + // await db.doc(config.backupDoc).collection('exports').add({ + // error: ex.message, + // status: 'Failed', + // timestamp: FieldValue.serverTimestamp(), + // }); + + return Promise.resolve(); + } +}; + +/** + * Checks if a long integer is a valid UNIX timestamp in seconds. + * + * @param timestamp The timestamp to check. + * @returns Whether the timestamp is valid. + */ +function isValidUnixTimestamp(timestamp: number): boolean { + // Ensure it's a non-negative integer + if (!timestamp || timestamp < 0 || !Number.isInteger(timestamp)) { + return false; + } + + // Get the current UNIX timestamp + const currentTimestamp: number = Math.floor(Date.now() / 1000); + + // Ensure the timestamp isn't in the future + if (timestamp > currentTimestamp) { + return false; + } + + return true; +} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_firestore_backup_init_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_firestore_backup_init_handler.ts new file mode 100644 index 000000000..a2bb7648e --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_firestore_backup_init_handler.ts @@ -0,0 +1,77 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getExtensions } from "firebase-admin/extensions"; + +import { logger } from "firebase-functions/v1"; +import { updateBackup, updateStatus } from "../utils/database"; + +import { waitForExportCompletion } from "../utils/import_export"; +import { FieldValue } from "firebase-admin/firestore"; + +export const onFirestoreBackupInitHandler = async (data: any) => { + const { id, name } = data; + const runtime = getExtensions().runtime(); + + // Update the status + await runtime.setProcessingState( + "NONE", + "Waiting for the export to be completed" + ); + + try { + // Update the Firestore status + await updateStatus(id, { + status: "Exporting initial backup", + }); + + // Start polling for updates + await waitForExportCompletion(name); + + // Set status to completed + await updateStatus(id, { + status: "Completed", + }); + + // Update the current backup + updateBackup(id, { + status: "Completed", + timestamp: FieldValue.serverTimestamp(), + }); + + // Update the status + await runtime.setProcessingState( + "PROCESSING_COMPLETE", + "Successfully backed up to Firestore" + ); + } catch (ex: any) { + logger.error("Error backing up to BQ", ex); + + await updateStatus(id, { + status: "Error", + error: ex.message, + }); + + await runtime.setProcessingState( + "PROCESSING_FAILED", + "Error backing up to Firestore" + ); + + return Promise.resolve(); + } + + return Promise.resolve(); +}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_http_run_restoration_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_http_run_restoration_handler.ts new file mode 100644 index 000000000..ad02288e1 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_http_run_restoration_handler.ts @@ -0,0 +1,53 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getFunctions } from "firebase-admin/functions"; +import { Request, Response, logger } from "firebase-functions/v1"; + +import config from "../config"; + +export const onHttpRunRestorationHandler = async ( + request: Request, + response: Response +) => { + const timestamp = request.body.timestamp; + if (!timestamp) { + logger.error( + '"timestamp" field is missing, please ensure that you are sending a valid timestamp in the request body' + ); + return Promise.resolve(); + } + + const now = new Date().getTime(); + + if (timestamp >= now) { + logger.error("The timestamp is in the future, aborting"); + return Promise.resolve(); + } + + const taskName = `projects/${config.projectId}/locations/${config.location}/functions/onBackupRestore`; + + const queue = getFunctions().taskQueue(taskName, config.instanceId); + + logger.log( + `Enqueuing task ${taskName} with timestamp ${timestamp}`, + request.body + ); + + // Queue a restoration task + await queue.enqueue(request.body); + response.status(200).send("Restoration task enqueued"); +}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_initial_setup_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_initial_setup_handler.ts new file mode 100644 index 000000000..c4698f77a --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_initial_setup_handler.ts @@ -0,0 +1,44 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getExtensions } from "firebase-admin/extensions"; + +import config from "../config"; + +import { initialize } from "../utils/big_query"; +import { bqBackupSchema } from "../constants/bq_backup_schema"; + +export async function runInitialSetupHandler() { + // Setup runtime + const runtime = getExtensions().runtime(); + + await runtime.setProcessingState( + "NONE", + `Creating/updating dataset and table ${config.bqDataset}.${config.bqtable}` + ); + + // Setup sync dataset and tables + const [syncDataset, syncTable] = await initialize( + config.bqDataset, + config.bqtable, + bqBackupSchema + ); + + return runtime.setProcessingState( + "PROCESSING_COMPLETE", + `Initialized dataset and table ${syncDataset.id}.${syncTable.id}` + ); +} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_restoration_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_restoration_handler.ts new file mode 100644 index 000000000..edab916f5 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_restoration_handler.ts @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getFunctions } from "firebase-admin/functions"; + +import config from "../config"; +import { onBackupRestore } from "../index"; + +export const onRunRestorationHandler = async () => { + const queue = getFunctions().taskQueue( + `locations/${config.location}/functions/${onBackupRestore.name}`, + config.instanceId + ); + + // Queue a restoration task + return queue.enqueue({}); +}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_sync_data_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_sync_data_handler.ts new file mode 100644 index 000000000..387736e9e --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_sync_data_handler.ts @@ -0,0 +1,64 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as functions from "firebase-functions"; +import { getFunctions } from "firebase-admin/functions"; + +import config from "../config"; +import { firestoreSerializer } from "../utils/firestore_serializer"; + +const getState = ( + change: functions.Change +) => { + // return if created + if (!change.before?.exists) return "CREATE"; + + // return if deleted + if (!change.after?.exists) return "DELETE"; + + //else return updated + return "UPDATE"; +}; + +export const syncDataHandler = async ( + change: functions.Change, + ctx: functions.EventContext +) => { + const queue = getFunctions().taskQueue( + `locations/${config.location}/functions/syncDataTask`, + config.instanceId + ); + + //state whether the update is an CREATE, UPDATE or DELETE + const changeType = getState(change); + + // format data + const beforeData = change.before ? change.before.data() : null; + const afterData = change.after ? change.after.data() : null; + + // serialize data + const serializedBeforeData = await firestoreSerializer(beforeData); + const serializedAfterData = await firestoreSerializer(afterData); + + return queue.enqueue({ + beforeData: JSON.stringify(serializedBeforeData), + afterData: JSON.stringify(serializedAfterData), + documentId: change.before?.id || change.after.id, + documentPath: change.before?.ref?.path || change.after.ref.path, + timestamp: ctx.timestamp, + changeType, + }); +}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/tasks/sync_data_task_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/tasks/sync_data_task_handler.ts new file mode 100644 index 000000000..b5d619282 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/tasks/sync_data_task_handler.ts @@ -0,0 +1,35 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { logger } from "firebase-functions"; + +import config from "../config"; +import { getTable } from "../utils/big_query"; + +export async function syncDataTaskHandler( + data: Record +): Promise { + const table = await getTable(config.bqDataset, config.bqtable); + + // Write the data to the database + await table.insert(data).catch((ex: any) => { + for (const error of ex.errors) { + for (const err of error.errors) { + logger.error(err); + } + } + }); +} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/utils/big_query.ts b/kits/firestore-incremental-capture/legacy/functions/src/utils/big_query.ts new file mode 100644 index 000000000..8bb7a3138 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/utils/big_query.ts @@ -0,0 +1,124 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { storage } from "firebase-admin"; +import { BigQuery, Dataset } from "@google-cloud/bigquery"; + +import config from "../config"; +import * as logs from "../logs"; + +const bq = new BigQuery({ projectId: config.projectId }); + +function bigqueryDataset(databaseId: string) { + return bq.dataset(databaseId, { + location: config.datasetLocation, + }); +} + +async function initializeDataset(databaseId: string) { + let dataset: Dataset = bigqueryDataset(databaseId); + const [datasetExists] = await dataset.exists(); + + if (datasetExists) { + logs.bigQueryDatasetExists(databaseId); + return dataset; + } + + /** Create table if does not exisst */ + try { + logs.bigQueryDatasetCreating(databaseId); + [dataset] = await bq.createDataset(databaseId, { + location: config.datasetLocation, + }); + logs.bigQueryDatasetCreated(databaseId); + return dataset; + } catch (ex: any) { + logs.datasetCeationError(databaseId); + return dataset; + } +} + +async function initializeTable( + databaseId: string, + tableId: string, + schema: Record[] | null = null +) { + let table; + const dataset: Dataset = bigqueryDataset(databaseId); + table = dataset.table(tableId); + const [tableExists] = await table.exists(); + + /** Return if table exists */ + logs.bigQueryTableExists(tableId); + if (tableExists) return table; + + /** Create a new table and return */ + try { + logs.bigQueryTableCreating(tableId); + + if (!dataset.id || !schema) + throw new Error("Dataset ID and schema must not be undefined"); + + /** + * TODO: Add time partitioning + * TODO: Include expirationMs for partitioning based on config + */ + [table] = await bq.dataset(dataset.id).createTable(tableId, { + schema, + location: config.datasetLocation, + }); + + logs.bigQueryTableCreated(tableId); + return table; + } catch (ex: any) { + logs.tableCreationError(config.bqDataset, ex.message); + return dataset; + } +} + +export async function initialize( + databaseId: string, + tableId: string, + schema: Record[] | null = null +) { + const dataset = await initializeDataset(databaseId); + const table = await initializeTable(databaseId, tableId, schema); + + return [dataset, table]; +} + +export async function getTable(datasetId: string, tableId: string) { + return bq.dataset(datasetId).table(tableId); +} + +/** + * Export the Firestore db to storage + * TODO: This may now be obsolete. We can restore a database, and then replay the data through dataflow. + */ +export const exportToBQ = async (id: string) => { + const name = config.instanceId; + const filename = `${config.bucketPath}/${id}/all_namespaces/kind_${name}/all_namespaces_kind_${name}.export_metadata`; + const bucket = storage().bucket(`gs://${config.bucketName}`); + const file = bucket.file(filename); + + /** + * writeDisposition to overwire the table, if exists + */ + + return bq.dataset(config.bqDataset).table(config.bqtable).load(file, { + writeDisposition: "WRITE_TRUNCATE", + }); +}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/utils/database.ts b/kits/firestore-incremental-capture/legacy/functions/src/utils/database.ts new file mode 100644 index 000000000..dfc6f6997 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/utils/database.ts @@ -0,0 +1,45 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as admin from "firebase-admin"; +import config from "../config"; +import { logger } from "firebase-functions/v1"; + +export const updateStatus = (id: string, data: any) => { + // log update + logger.info(`Updating status for ${id}`, data); + + // Get the backup collection document + const document = admin.firestore().doc(config.statusDoc); + + // Update the document + return document.set({ ...data }, { merge: true }); +}; + +export const updateBackup = (id: string, data: any) => { + // log update + logger.info(`Updating backup for ${id}`, data); + + // Get the backup collection document + const document = admin + .firestore() + .doc(config.backupDoc) + .collection("exports") + .doc(`${id}`); + + // Update the document + return document.set({ ...data }, { merge: true }); +}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/utils/firestore_serializer.ts b/kits/firestore-incremental-capture/legacy/functions/src/utils/firestore_serializer.ts new file mode 100644 index 000000000..44efc9c89 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/utils/firestore_serializer.ts @@ -0,0 +1,149 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as traverse from "traverse"; + +import { + DocumentReference, + GeoPoint, + Timestamp, +} from "firebase-admin/firestore"; + +export const firestoreSerializer = (data: any) => { + return traverse(data).reduce(function (acc, property) { + if (this.isRoot) return acc; + + if (Buffer.isBuffer(property)) { + if (this.key) { + acc[this.key] = { + type: "binary", + value: property.toString("base64"), + }; + } + + this.delete(true); + return acc; + } + + /** Handle array types */ + if (Array.isArray(property)) { + if (this.key) + acc[this.key] = { + type: "array", + value: property.map((item) => { + // If the item is a primitive, return the serialized format + if (typeof item !== "object" || item === null) { + return { type: typeof item, value: item }; + } + // If the item is an object (including array), recursively serialize it + return firestoreSerializer(item); + }), + }; + this.delete(true); + return acc; + } + + // Handle GeoPoint special type + if (property instanceof GeoPoint) { + if (this.key) + acc[this.key] = { + type: "geopoint", + value: { + latitude: { + type: "number", + value: property.latitude, + }, + longitude: { + type: "number", + value: property.longitude, + }, + }, + }; + this.delete(true); // Delete this node and halt further traversal for its children + return acc; + } + + // Handle Timestamp special type + if (property instanceof Timestamp) { + const date = property.toDate(); // Convert Timestamp to JavaScript Date + if (this.key) + acc[this.key] = { + type: "timestamp", + value: date.toISOString(), // Convert Date to ISO string + }; + this.delete(true); + return acc; + } + + // Handle DocumentReference special type + if (property instanceof DocumentReference) { + if (this.key) + acc[this.key] = { + type: "documentReference", + value: property.path, // Assuming DocumentReference has a 'path' property + }; + this.delete(true); + return acc; + } + + if (property === null) { + if (this.key) + acc[this.key] = { + type: "null", // Set the type as 'null' + value: null, + }; + return acc; + } + + // Handle object type nodes + if (!this.isLeaf) { + /** Handle array types */ + if (Array.isArray(property)) { + if (this.key) + acc[this.key] = { + type: "array", + value: property.map((item) => firestoreSerializer(item)), + }; + this.delete(true); + return acc; + } + + // If it's an object but not a special type, serialize it + else if (typeof property === "object" && property !== null) { + if (this.key) + acc[this.key] = { + type: "map", + value: firestoreSerializer(property), // Recursive serialization + }; + this.delete(true); + return acc; + } + + return acc; + } + + // Decide the accumulator context based on the parent node type + const context = + this.parent?.node && this.parent.node.type === "object" + ? this.parent?.node.value + : acc; + + if (this.key) + context[this.key] = { type: typeof property, value: property }; + + return acc; + }, {}); +}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/utils/import_export.ts b/kits/firestore-incremental-capture/legacy/functions/src/utils/import_export.ts new file mode 100644 index 000000000..58de329d8 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/utils/import_export.ts @@ -0,0 +1,77 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// eslint-disable-next-line node/no-unpublished-import +import * as firestore from "@google-cloud/firestore"; +import config from "../config"; +import { logger } from "firebase-functions/v1"; + +const client = new firestore.v1.FirestoreAdminClient({ + projectId: config.projectId, +}); + +/** + * Regularly ping the export operation to check for completion + */ +export async function waitForExportCompletion(name: string) { + logger.log("Checking for export progress: ", name); + const response = await client.checkExportDocumentsProgress(name); + if (!response.done) { + // Wait for 1 minute retrying + await new Promise((resolve) => setTimeout(resolve, 60000)); + //try again + await waitForExportCompletion(name); + } + return Promise.resolve(response); +} + +/** + * Regularly ping the import operation to check for completion + */ +export async function waitForImportCompletion(name: string) { + logger.log("Checking for import progress: ", name); + const response = await client.checkImportDocumentsProgress(name); + if (!response.done) { + // Wait for 1 minute retrying + await new Promise((resolve) => setTimeout(resolve, 60000)); + // try again + await waitForImportCompletion(name); + } + return Promise.resolve(response); +} + +/** + * Imports data from GCS to the specified Firestore backup instance + */ +export async function createImport(id: string) { + const { projectId, syncCollectionPath, bucketName } = config; + + // Extract the database name from the backup instance name + const values = config.backupInstanceName.split("/"); + const database = values[values.length - 1]; + + const name = client.databasePath(projectId, database); + + // Start backup + const [operation] = await client.importDocuments({ + name, + inputUriPrefix: `gs://${bucketName}/backups/${id}`, + collectionIds: + syncCollectionPath === "**" ? [] : syncCollectionPath.split(","), + }); + + return { id, operation }; +} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/utils/serialize.ts b/kits/firestore-incremental-capture/legacy/functions/src/utils/serialize.ts new file mode 100644 index 000000000..e2146b1c2 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/src/utils/serialize.ts @@ -0,0 +1,41 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DocumentReference } from "firebase-admin/firestore"; + +import * as traverse from "traverse"; + +export const serializeData = (eventData: any) => { + if (typeof eventData === "undefined") { + return undefined; + } + + const data = traverse>(eventData).map(function ( + property: any + ) { + if (property && property.constructor) { + if (property.constructor.name === "Buffer") { + this.remove(); + } + + if (property.constructor.name === DocumentReference.name) { + this.update(property.path); + } + } + }); + + return data; +}; diff --git a/kits/firestore-incremental-capture/legacy/functions/tsconfig.dev.json b/kits/firestore-incremental-capture/legacy/functions/tsconfig.dev.json new file mode 100644 index 000000000..c0f990d78 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/tsconfig.dev.json @@ -0,0 +1,3 @@ +{ + "include": [".eslintrc.js"] +} diff --git a/kits/firestore-incremental-capture/legacy/functions/tsconfig.json b/kits/firestore-incremental-capture/legacy/functions/tsconfig.json new file mode 100644 index 000000000..2e24641f7 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/functions/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "module": "commonjs", + "noImplicitReturns": true, + "noUnusedLocals": false, + "outDir": "lib", + "skipLibCheck": true, + "sourceMap": true, + "strict": true, + "target": "es2017" + }, + "compileOnSave": true, + "include": ["src"] +} diff --git a/kits/firestore-incremental-capture/legacy/install/functions/build_dataflow_template.sh b/kits/firestore-incremental-capture/legacy/install/functions/build_dataflow_template.sh new file mode 100644 index 000000000..bcaa9d834 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/install/functions/build_dataflow_template.sh @@ -0,0 +1,14 @@ +echo -e "${YELLOW}Step 6: Building Dataflow Flex Template...${NC}" +if gcloud dataflow flex-template build gs://$BUCKET_NAME/$EXT_INSTANCE_ID-dataflow-restore \ + --image-gcr-path $LOCATION-docker.pkg.dev/$PROJECT_ID/$EXT_INSTANCE_ID/dataflow/restore:latest \ + --sdk-language JAVA \ + --flex-template-base-image JAVA11 \ + --jar $JAR_PATH \ + --env FLEX_TEMPLATE_JAVA_MAIN_CLASS="com.pipeline.RestorationPipeline" \ + --project $PROJECT_ID; then + echo -e "${GREEN}Dataflow Flex Template built successfully.${NC}" + SUCCESS_TASKS+=("${GREEN}${TICK} Dataflow Flex Template built successfully.") +else + echo -e "${RED}Failed to build Dataflow Flex Template.${NC}" + FAILED_TASKS+=("${RED}${CROSS} Failed to build Dataflow Flex Template.") +fi \ No newline at end of file diff --git a/kits/firestore-incremental-capture/legacy/install/functions/download_restore_firestore.sh b/kits/firestore-incremental-capture/legacy/install/functions/download_restore_firestore.sh new file mode 100644 index 000000000..640094ec7 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/install/functions/download_restore_firestore.sh @@ -0,0 +1,42 @@ +echo -e "${YELLOW}Downloading the JAR file...${NC}" + +# Use the correct URL +if curl -L -o restore-firestore.jar "https://github.com/GoogleCloudPlatform/firebase-extensions/raw/main/firestore-incremental-capture-pipeline/target/restore-firestore.jar"; then + # Check if the file is actually a JAR and not HTML + if file restore-firestore.jar | grep -q "HTML"; then + echo -e "${YELLOW}The downloaded file appears to be an HTML page, not a JAR file. The file may not exist at that location.${NC}" + + # Try alternative sources + echo -e "${YELLOW}Trying alternative locations...${NC}" + + # Alternative 1: Try Firebase Extensions GitHub repo directly + if curl -L -o restore-firestore.jar "https://github.com/firebase/extensions/raw/main/firestore-incremental-capture-pipeline/target/restore-firestore.jar"; then + if ! file restore-firestore.jar | grep -q "HTML"; then + echo -e "${GREEN}JAR file downloaded successfully from alternative location.${NC}" + SUCCESS_TASKS+=("${GREEN}${TICK} Successfully downloaded assets.") + exit 0 + fi + fi + + # Alternative 2: Try checking Google Cloud Storage + echo -e "${YELLOW}Trying to download from Cloud Storage...${NC}" + if gcloud storage cp gs://firebase-preview-drop/extension-builds/firestore-incremental-capture/restore-firestore.jar ./restore-firestore.jar 2>/dev/null; then + echo -e "${GREEN}JAR file downloaded successfully from Cloud Storage.${NC}" + SUCCESS_TASKS+=("${GREEN}${TICK} Successfully downloaded assets.") + exit 0 + fi + + # If all attempts fail + echo -e "${RED}Failed to download a valid JAR file from all known locations.${NC}" + echo -e "${YELLOW}You may need to build the JAR from source or contact Firebase support.${NC}" + FAILED_TASKS+=("${RED}${CROSS} Failed to download assets.") + exit 1 + else + echo -e "${GREEN}JAR file downloaded successfully.${NC}" + SUCCESS_TASKS+=("${GREEN}${TICK} Successfully downloaded assets.") + fi +else + echo -e "${RED}Failed to download JAR file.${NC}" + FAILED_TASKS+=("${RED}${CROSS} Failed to download assets.") + exit 1 +fi \ No newline at end of file diff --git a/kits/firestore-incremental-capture/legacy/install/functions/enable_pitr.sh b/kits/firestore-incremental-capture/legacy/install/functions/enable_pitr.sh new file mode 100644 index 000000000..d60033560 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/install/functions/enable_pitr.sh @@ -0,0 +1,9 @@ +echo -e "${YELLOW}Step 1: Enabling PITR as per Google Cloud Console guide${NC}" + +if gcloud alpha firestore databases update --project=$PROJECT_ID --enable-pitr; then + echo -e "${GREEN}PITR enabled successfully on (default) database.${NC}" + SUCCESS_TASKS+=("${GREEN}${TICK} Enabled PiTR on (default) database.") +else + echo -e "${RED}Failed to enable PITR on (default) database.${NC}" + FAILED_TASKS+=("${RED}${CROSS} Failed to enable PiTR on (default) database.") +fi \ No newline at end of file diff --git a/kits/firestore-incremental-capture/legacy/install/functions/setup_artifact_registry.sh b/kits/firestore-incremental-capture/legacy/install/functions/setup_artifact_registry.sh new file mode 100644 index 000000000..5b26e89db --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/install/functions/setup_artifact_registry.sh @@ -0,0 +1,18 @@ +# Configure Artifact Registry +echo -e "${YELLOW}Step 3: Configuring Artifact Registry...${NC}" + +ARTIFACT_EXISTS=$(gcloud artifacts repositories list --location=$LOCATION --project=$PROJECT_ID --format="value(name)") + +if echo "$ARTIFACT_EXISTS" | grep -q "$EXT_INSTANCE_ID"; then + echo -e "${YELLOW}Artifact Registry already exists, skipping creation.${NC}" + SUCCESS_TASKS+=("${GREEN}${TICK} Artifact Registry already exists, configuration skipped.") +else + if gcloud artifacts repositories create $EXT_INSTANCE_ID --repository-format=docker --location=$LOCATION --project=$PROJECT_ID --async && \ + gcloud auth configure-docker $LOCATION-docker.pkg.dev; then + echo -e "${GREEN}Artifact Registry configured successfully.${NC}" + SUCCESS_TASKS+=("${GREEN}${TICK} Artifact Registry configured successfully.") + else + echo -e "${RED}Failed to configure Artifact Registry.${NC}" + FAILED_TASKS+=("${RED}${CROSS} Failed to configure Artifact Registry.") + fi +fi \ No newline at end of file diff --git a/kits/firestore-incremental-capture/legacy/install/functions/setup_firestore.sh b/kits/firestore-incremental-capture/legacy/install/functions/setup_firestore.sh new file mode 100644 index 000000000..c80c5feb9 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/install/functions/setup_firestore.sh @@ -0,0 +1,18 @@ +# Check if Firestore database already exists +echo -e "${YELLOW}Step 2: Setting up Firestore database${NC}" +DB_EXISTS=$(gcloud alpha firestore databases list --project=$PROJECT_ID --format="value(name)") + +if echo "$DB_EXISTS" | grep -q "projects/$PROJECT_ID/databases/$DATABASE_ID"; then + echo -e "${GREEN}Firestore database already exists, skipping creation.${NC}" + SUCCESS_TASKS+=("${GREEN}${TICK} Database already exists, setup skipped.") +else + # Create secondary Firestore database + echo -e "${YELLOW}Creating secondary Firestore database...${NC}" + if gcloud alpha firestore databases create --database=$DATABASE_ID --location=$DATABASE_LOCATION --type=firestore-native --project=$PROJECT_ID; then + echo -e "${GREEN}Firestore database created successfully.${NC}" + SUCCESS_TASKS+=("${GREEN}${TICK} Database created successfully.") + else + echo -e "${RED}Failed to create Firestore database.${NC}" + FAILED_TASKS+=("${RED}${CROSS} Failed to create Firestore database.") + fi +fi \ No newline at end of file diff --git a/kits/firestore-incremental-capture/legacy/install/functions/setup_service_account.sh b/kits/firestore-incremental-capture/legacy/install/functions/setup_service_account.sh new file mode 100644 index 000000000..445f21099 --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/install/functions/setup_service_account.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# Find extension's service account email +echo -e "${YELLOW}Finding extension's service account email...${NC}" + +# Get all service accounts that match our filter +SA_EMAILS=$(gcloud iam service-accounts list --format="value(EMAIL)" --filter="displayName~'Firebase Extensions $EXT_INSTANCE_ID service account' AND DISABLED=False" --project="$PROJECT_ID") + +# Check if we found any service accounts +if [ -z "$SA_EMAILS" ]; then + echo -e "${RED}Failed to find extension's service account email.${NC}" + FAILED_TASKS+=("${RED}${CROSS} Failed to find extension's service account.") + exit 1 # Exit if no service account is found as the next steps need it +else + # Use the first service account from the list + export SA_EMAIL=$(echo "$SA_EMAILS" | head -n 1) + echo -e "${GREEN}Service account email found: $SA_EMAIL${NC}" + + # Show all found service accounts for debugging + echo "$SA_EMAILS" + + SUCCESS_TASKS+=("${GREEN}${TICK} Found extension's service account.") +fi + +# Add required policy binding for Artifact Registry +echo -e "${YELLOW}Step 4: Adding IAM policy binding for Artifact Registry...${NC}" +if gcloud artifacts repositories add-iam-policy-binding $EXT_INSTANCE_ID \ + --location=$LOCATION \ + --project=$PROJECT_ID \ + --member="serviceAccount:$SA_EMAIL" \ + --role=roles/artifactregistry.writer \ + --condition=None; then + echo -e "${GREEN}Policy binding added successfully.${NC}" + SUCCESS_TASKS+=("${GREEN}${TICK} Policy binding added successfully.") +else + echo -e "${RED}Failed to add policy binding.${NC}" + FAILED_TASKS+=("${RED}${CROSS} Failed to add policy binding.") +fi + +# Add roles for extension service account to trigger Dataflow +echo -e "${YELLOW}Step 5: Adding roles for service account to trigger Dataflow...${NC}" +ROLE_SUCCESS=true + +if ! gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$SA_EMAIL" \ + --role=roles/dataflow.developer \ + --condition=None; then + ROLE_SUCCESS=false +fi + +if ! gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$SA_EMAIL" \ + --role=roles/iam.serviceAccountUser \ + --condition=None; then + ROLE_SUCCESS=false +fi + +if ! gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$SA_EMAIL" \ + --role=roles/artifactregistry.writer \ + --condition=None; then + ROLE_SUCCESS=false +fi + +if [ "$ROLE_SUCCESS" = true ]; then + echo -e "${GREEN}Roles added successfully.${NC}" + SUCCESS_TASKS+=("${GREEN}${TICK} Roles added successfully") +else + echo -e "${RED}Failed to add one or more roles.${NC}" + FAILED_TASKS+=("${RED}${CROSS} Failed to add one or more roles.") +fi \ No newline at end of file diff --git a/kits/firestore-incremental-capture/legacy/install/run.sh b/kits/firestore-incremental-capture/legacy/install/run.sh new file mode 100755 index 000000000..b5654911d --- /dev/null +++ b/kits/firestore-incremental-capture/legacy/install/run.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Define color codes for better readability +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export YELLOW='\033[1;33m' +export NC='\033[0m' +export TICK="✓" +export CROSS="✗" + +# Initialize arrays to hold success and failure messages +export SUCCESS_TASKS=() +export FAILED_TASKS=() + +# Define variables at the top +export PROJECT_ID="" +export BUCKET_NAME="" +export DATABASE_ID="" +export DATABASE_LOCATION="nam5" +export LOCATION="us-central1" +export EXT_INSTANCE_ID="firestore-incremental-capture" +export JAR_PATH="restore-firestore.jar" + +# Detect default bucket automatically only if BUCKET_NAME is not set +detect_default_bucket() { + # Skip if BUCKET_NAME is already set + if [ -n "$BUCKET_NAME" ]; then + echo "Using user-specified bucket: $BUCKET_NAME" + return + fi + + echo "Detecting default storage bucket..." + + # Try to list buckets and check for default buckets + local buckets=$(gcloud storage buckets list --project=$PROJECT_ID --format="value(name)") + + # Check for both possible default bucket names + if echo "$buckets" | grep -q "$PROJECT_ID.appspot.com"; then + echo "Detected default bucket: $PROJECT_ID.appspot.com" + BUCKET_NAME="$PROJECT_ID.appspot.com" + elif echo "$buckets" | grep -q "$PROJECT_ID.firebasestorage.app"; then + echo "Detected default bucket: $PROJECT_ID.firebasestorage.app" + BUCKET_NAME="$PROJECT_ID.firebasestorage.app" + else + echo -e "${YELLOW}Warning: Could not detect default bucket, using fallback strategy${NC}" + # Use a fallback approach - newest default bucket format + BUCKET_NAME="$PROJECT_ID.firebasestorage.app" + fi + + echo "Using bucket: $BUCKET_NAME" +} + +# Call the detect function after PROJECT_ID is set +detect_default_bucket + +# Source all component scripts +source ./functions/download_restore_firestore.sh +source ./functions/enable_pitr.sh +source ./functions/setup_firestore.sh +source ./functions/setup_artifact_registry.sh +source ./functions/setup_service_account.sh +source ./functions/build_dataflow_template.sh + +# Print summary +echo -e "\n${GREEN}Setup process completed.${NC}" + +if [ ${#SUCCESS_TASKS[@]} -gt 0 ]; then + echo -e "\n${GREEN}Successful operations:${NC}" + for TASK in "${SUCCESS_TASKS[@]}"; do + echo -e "$TASK" + done +fi + +if [ ${#FAILED_TASKS[@]} -gt 0 ]; then + echo -e "\n${RED}Failed operations:${NC}" + for TASK in "${FAILED_TASKS[@]}"; do + echo -e "$TASK" + done + echo -e "\n${RED}Warning: Some operations failed. Please review the errors above.${NC}" + exit 1 +else + echo -e "\n${GREEN}All operations completed successfully!${NC}" +fi \ No newline at end of file From c744d3f2be636fbc9efb3106541e20bf7ddfadc4 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Tue, 11 Aug 2026 14:17:42 +0100 Subject: [PATCH 03/10] feat(kits): implement firestore-incremental-capture Replaces the skeleton with the five functions the extension actually deployed: the capture pair (Firestore trigger to task queue to BigQuery), the restoration pair (HTTP trigger to task queue to Dataflow), and a lifecycle task that provisions the changelog dataset and table. The three functions the extension exported but never wired into extension.yaml are dropped, along with their only callers - the Cloud Build template staging path could not work from the functions runtime, which has no gcloud. scripts/setup.sh takes over those prerequisites and builds the pipeline jar from the vendored source rather than downloading a prebuilt one. Fixes carried over from the extension: - The HTTP restoration guard compared a seconds-epoch timestamp against Date.now() in milliseconds, so it never rejected a future timestamp. Both entry points now share one validator. - DocumentReference values were tagged 'documentReference', which FirestoreReconstructor does not recognise, so references were dropped on restore. They are now tagged 'reference' to match the pipeline. - The default bucket fell back to .appspot.com, wrong for projects created after September 2024. The restoration endpoint remains unauthenticated, as in the extension. The exposure is documented in the README. firebase-functions is imported through narrow subpaths: the top-level and v2 barrels pull in the RTDB provider, which fails to load because @firebase/database-compat needs @firebase/app and npm does not install it. --- kits/firestore-incremental-capture/.gitignore | 2 + kits/firestore-incremental-capture/README.md | 171 +- .../firebase.json | 17 + .../package-lock.json | 5213 +++++++++++++++++ .../package.json | 21 +- .../scripts/setup.sh | 242 + .../src/bigquery.ts | 91 + .../src/capture-config.ts | 141 + .../src/changelog.ts | 51 + .../src/config.ts | 170 + .../src/dataflow.ts | 91 + .../src/handlers.ts | 218 + .../src/index.ts | 203 +- kits/firestore-incremental-capture/src/lib.ts | 65 + .../firestore-incremental-capture/src/logs.ts | 59 + .../src/serializer.ts | 126 + .../src/tasks.ts | 53 + .../tests/capture-config.test.ts | 107 + .../tests/handlers.test.ts | 302 + .../tests/serializer.test.ts | 96 + .../tsconfig.json | 11 +- .../vitest.config.ts | 25 + 22 files changed, 7406 insertions(+), 69 deletions(-) create mode 100644 kits/firestore-incremental-capture/.gitignore create mode 100644 kits/firestore-incremental-capture/firebase.json create mode 100644 kits/firestore-incremental-capture/package-lock.json create mode 100755 kits/firestore-incremental-capture/scripts/setup.sh create mode 100644 kits/firestore-incremental-capture/src/bigquery.ts create mode 100644 kits/firestore-incremental-capture/src/capture-config.ts create mode 100644 kits/firestore-incremental-capture/src/changelog.ts create mode 100644 kits/firestore-incremental-capture/src/config.ts create mode 100644 kits/firestore-incremental-capture/src/dataflow.ts create mode 100644 kits/firestore-incremental-capture/src/handlers.ts create mode 100644 kits/firestore-incremental-capture/src/lib.ts create mode 100644 kits/firestore-incremental-capture/src/logs.ts create mode 100644 kits/firestore-incremental-capture/src/serializer.ts create mode 100644 kits/firestore-incremental-capture/src/tasks.ts create mode 100644 kits/firestore-incremental-capture/tests/capture-config.test.ts create mode 100644 kits/firestore-incremental-capture/tests/handlers.test.ts create mode 100644 kits/firestore-incremental-capture/tests/serializer.test.ts create mode 100644 kits/firestore-incremental-capture/vitest.config.ts diff --git a/kits/firestore-incremental-capture/.gitignore b/kits/firestore-incremental-capture/.gitignore new file mode 100644 index 000000000..73c8594a9 --- /dev/null +++ b/kits/firestore-incremental-capture/.gitignore @@ -0,0 +1,2 @@ +lib +*.tsbuildinfo diff --git a/kits/firestore-incremental-capture/README.md b/kits/firestore-incremental-capture/README.md index 360cd6aa4..38b1288c9 100644 --- a/kits/firestore-incremental-capture/README.md +++ b/kits/firestore-incremental-capture/README.md @@ -1,65 +1,124 @@ # @firebase/firestore-incremental-capture -Incremental point-in-time capture of Firestore changes - -> **Status: skeleton — not yet implemented.** -> Migrated from the `firestore-incremental-capture` Firebase Extension to an npm-shared Firebase -> Function (v2). Track the reference implementation in -> [`packages/firestore-bigquery-export`](../firestore-bigquery-export) and the -> design in [`docs/rfc.md`](../../docs/rfc.md). - -Set `"private": false` in `package.json` when ready to publish. - -## Deploy - -The package's `firebase.json` declares a `kit` stanza (Firebase CLI 15.25.1 or -later, behind the `kits` experiment): - -```json -{ - "functions": [ - { - "source": ".", - "kit": "firestore-incremental-capture", - "instances": { - "default": "." - } - } - ] -} +Incremental point-in-time capture of Firestore changes, as a deployable Firebase Function. + +Migrated from the `firestore-incremental-capture` Firebase Extension. Every write to a watched +collection is serialized into a BigQuery changelog. A restoration then rebuilds a separate Firestore +database as it stood at a chosen second: a Dataflow pipeline reads a PITR snapshot of the source +database and replays the changelog on top of it. + +## How it works + +**Capture.** `syncData` fires on every document write, serializes the before/after data, and queues +it. `syncChangelogTask` inserts the queued row into BigQuery. The insert is a separate hop so a +BigQuery outage retries on the task queue's schedule instead of holding the Firestore trigger open. + +**Restore.** `onHttpRunRestoration` validates a timestamp and queues the work. `runRestorationTask` +launches the Dataflow flex template in `pipeline/`, which writes the PITR baseline into the backup +database and then replays every changelog row up to the timestamp. + +**Provisioning.** `initIncrementalCapture` runs after first deploy and after each redeploy, creating +the BigQuery dataset and changelog table. Everything restoration needs beyond that is provisioned by +`scripts/setup.sh` - see below. + +## Security + +`onHttpRunRestoration` is **unauthenticated**, matching the extension it was migrated from. Anyone +who can reach the URL can start a Dataflow job that batch-writes over the backup database. Before +deploying to production, restrict it: set Cloud Run ingress, apply an IAM invoker policy, or drop the +endpoint and have your own authorized code enqueue `runRestorationTask` directly. + +## Setup + +Restoration needs a PITR-enabled source database, an existing backup database, and a staged Dataflow +flex template. None can be provisioned from the functions runtime, which has neither gcloud nor +Maven. Run the setup script once, before deploying: + +```bash +PROJECT_ID=my-project BACKUP_INSTANCE_ID=my-backup ./scripts/setup.sh +``` + +It enables the required APIs, turns on PITR, creates the backup database, creates an Artifact +Registry repository, grants the Dataflow roles, builds the pipeline jar from `pipeline/`, and stages +the flex template. Every step is idempotent. See the header of `scripts/setup.sh` for the optional +variables. + +PITR only covers writes made after it is enabled, so restoration can only target a point in time +after setup ran. + +## Configuration + +Set these in `.env` or `.env.`. + +| Param | Default | Description | +| ---------------------- | ------------------------------- | -------------------------------------------------------------- | +| `LOCATION` | `us-central1` | Region for the functions. | +| `DATABASE` | `(default)` | Firestore database to capture. | +| `SYNC_COLLECTION_PATH` | `posts` | Collection to capture. `{document=**}` captures everything. | +| `SYNC_DATASET` | `backup_dataset` | BigQuery dataset for the changelog. | +| `SYNC_TABLE` | `backup_table` | BigQuery changelog table. | +| `BACKUP_INSTANCE_ID` | _required_ | Firestore database to restore into. Must not be `DATABASE`. | +| `DATASET_LOCATION` | `us` | BigQuery dataset location. | +| `DATAFLOW_REGION` | `LOCATION` | Region for Dataflow jobs. | +| `BUCKET_NAME` | `.firebasestorage.app` | Bucket holding the flex template. | +| `INSTANCE_ID` | `firestore-incremental-capture` | Namespaces the queues, template, jobs and status documents. | +| `LOG_LEVEL` | `info` | `debug`, `info`, `warn`, `error` or `silent`. | + +Projects created before September 2024 use `.appspot.com` as their default bucket and must +set `BUCKET_NAME` explicitly. + +## Usage + +Re-export the functions from your own functions codebase entry: + +```ts +export { + initIncrementalCapture, + onHttpRunRestoration, + runRestorationTask, + syncChangelogTask, + syncData, +} from "@firebase/firestore-incremental-capture"; +``` + +Trigger a restoration with a whole number of seconds since the Unix epoch: + +```bash +curl -X POST https://-.cloudfunctions.net/onHttpRunRestoration \ + -H 'Content-Type: application/json' \ + -d '{"timestamp": 1700000000}' ``` -`instances` maps each instance id to the directory (relative to -`firebase.json`) holding that instance's `.env`. The CLI prefixes every -function and task queue name with `kit--`. +To own trigger registration yourself, import the handlers from the side-effect-free surface: -```sh -firebase experiments:enable kits -firebase deploy --only functions +```ts +import { handleDocumentWrite, resolveCaptureConfig } from "@firebase/firestore-incremental-capture/lib"; ``` -Deploy a single instance with `firebase deploy --only functions:`. - -## Multiple instances - -To run several capture instances, add one entry per instance to the `instances` -map, each pointing at its own config directory with its own `.env`: - -```json -{ - "functions": [ - { - "source": ".", - "kit": "firestore-incremental-capture", - "instances": { - "users": "instances/users", - "orders": "instances/orders" - } - } - ] -} +## Restoration gaps + +The restoration pipeline in `pipeline/` is vendored from the original extension unchanged, and it +does not round trip everything the capture side records. `FirestoreReconstructor.buildFirestoreMap` +switches on each value's type tag and **silently drops any field whose tag it does not handle**: + +- **`binary` and `null` fields are dropped.** The pipeline has no case for either, so a restored + document loses them. +- **Arrays do not survive.** `buildFirestoreList` rebuilds every element as a map, so an array of + primitives restores as a list of empty maps. +- **Changelog replay writes to a malformed path.** `IncrementalCaptureLog.convertToFirestoreValue` + applies `createDocumentName` to a path that has already been through it, producing a doubled + `projects/…/databases/…/documents/` prefix. + +The PITR baseline half of a restoration is unaffected; these apply to the changelog replay on top of +it. Fixing them means changing the Java, which is out of scope for this migration. + +## Development + +```bash +npm install +npm run build +npm test ``` -Instance ids must be unique across all kit stanzas in the project, and every -instance's function names are namespaced by its `kit--` prefix, so -the instances cannot collide. +`pipeline/` is the Java/Beam restoration pipeline, built by `scripts/setup.sh`. To work on it +directly, see `pipeline/README.md`. diff --git a/kits/firestore-incremental-capture/firebase.json b/kits/firestore-incremental-capture/firebase.json new file mode 100644 index 000000000..254a2a570 --- /dev/null +++ b/kits/firestore-incremental-capture/firebase.json @@ -0,0 +1,17 @@ +{ + "functions": [ + { + "source": ".", + "codebase": "firestore-incremental-capture", + "ignore": [ + "node_modules", + ".git", + "src", + "legacy", + "pipeline", + "*.local" + ], + "predeploy": ["npm --prefix \"$RESOURCE_DIR\" run build"] + } + ] +} diff --git a/kits/firestore-incremental-capture/package-lock.json b/kits/firestore-incremental-capture/package-lock.json new file mode 100644 index 000000000..27b8e04aa --- /dev/null +++ b/kits/firestore-incremental-capture/package-lock.json @@ -0,0 +1,5213 @@ +{ + "name": "@firebase/firestore-incremental-capture", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@firebase/firestore-incremental-capture", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/bigquery": "^7.6.0", + "@google-cloud/dataflow": "^3.2.0", + "firebase-admin": "^14.1.0", + "firebase-functions": "7.3.0" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^6.0.0", + "vitest": "^3.2.4" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@fastify/busboy": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-3.2.0.tgz", + "integrity": "sha512-m9FVDXU3GT2ITSe0UaMA5rU3QkfC/UXtCU8y0gSN/GugTqtVldOBWIB5V6V3sbmenVZUIpU6f+mPEO2+m5iTaA==", + "license": "MIT" + }, + "node_modules/@firebase/app-check-interop-types": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@firebase/app-check-interop-types/-/app-check-interop-types-0.3.4.tgz", + "integrity": "sha512-zz3i6e13B8BfWiLy8MABtTh8aGIACgKbf9UVnyHcWs+yQzJXgQcl8A46b0zfaiJHdQ+niF0ouAfcpuf+3LMPQg==", + "license": "Apache-2.0" + }, + "node_modules/@firebase/app-types": { + "version": "0.9.5", + "resolved": "https://registry.npmjs.org/@firebase/app-types/-/app-types-0.9.5.tgz", + "integrity": "sha512-YevqTjvo7Iujsa9Dwowmd6dSoElhzmD63ZSrq6bzjvQ6POjYgNjOFHLmNIgJs48eNO093NCERibuFnxbfOvU7A==", + "license": "Apache-2.0", + "dependencies": { + "@firebase/logger": "0.5.1" + } + }, + "node_modules/@firebase/auth-interop-types": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/@firebase/auth-interop-types/-/auth-interop-types-0.2.5.tgz", + "integrity": "sha512-1Li/YuBDBAXcKv7BzY4U28gontUmAaw53sYiqbaVOMCFb2lFKK/c3CGMUWqtwe7+TXrl3poWnTCL5umYBg85Eg==", + "license": "Apache-2.0" + }, + "node_modules/@firebase/component": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/@firebase/component/-/component-0.7.4.tgz", + "integrity": "sha512-tLpOaaCol9ugUIYp2R3CbWPPA8Ajg/papX/XHEy8U52b/QXH3BbX8tTJX9aShDCjp+9sMAxMLD94i7lresdugQ==", + "license": "Apache-2.0", + "dependencies": { + "@firebase/util": "1.15.2", + "tslib": "^2.1.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@firebase/database": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@firebase/database/-/database-1.1.4.tgz", + "integrity": "sha512-D+j4+8uhGtNd1tVD+X+c8JrC4ppStGJKyujSQt2NPwdN26QcCk0BeIxue+UqspHkHiFHyQOimwlzjLewGq6S+A==", + "license": "Apache-2.0", + "dependencies": { + "@firebase/app-check-interop-types": "0.3.4", + "@firebase/auth-interop-types": "0.2.5", + "@firebase/component": "0.7.4", + "@firebase/logger": "0.5.1", + "@firebase/util": "1.15.2", + "faye-websocket": "0.11.4", + "tslib": "^2.1.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@firebase/database-compat": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@firebase/database-compat/-/database-compat-2.1.5.tgz", + "integrity": "sha512-m2KZDNXrg8DBzXWQNbbrjOhsJnM+ctsSFaDYKrqj1gEetQ8BSAwRuMUdeWLM9a6qPBgOvOA+o09j1BSEzdFqOg==", + "license": "Apache-2.0", + "dependencies": { + "@firebase/component": "0.7.4", + "@firebase/database": "1.1.4", + "@firebase/database-types": "1.0.21", + "@firebase/logger": "0.5.1", + "@firebase/util": "1.15.2", + "tslib": "^2.1.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@firebase/app": "0.x", + "@firebase/app-compat": "0.x" + }, + "peerDependenciesMeta": { + "@firebase/app": { + "optional": true + }, + "@firebase/app-compat": { + "optional": true + } + } + }, + "node_modules/@firebase/database-types": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/@firebase/database-types/-/database-types-1.0.21.tgz", + "integrity": "sha512-SX1jUqhttKgg/m9dYRTvqU9QvucBooziWfA986r4cpsbi4zlsvewe424j3Vpduwd6DG1MSAMfBVT2VqA61FnkA==", + "license": "Apache-2.0", + "dependencies": { + "@firebase/app-types": "0.9.5", + "@firebase/util": "1.15.2" + } + }, + "node_modules/@firebase/logger": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/@firebase/logger/-/logger-0.5.1.tgz", + "integrity": "sha512-vZKLsqE1ABOy8OjQiE7cUTFn4gvaqlk88yp8N94Pk/sDpq61YqZGqmVFZTvOyflTwuYFcWirBdYGoJgbDaXKYQ==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.1.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@firebase/util": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/@firebase/util/-/util-1.15.2.tgz", + "integrity": "sha512-974pWIZVLDMc5GW5YAsj8y0XxULxIy/sPUy7tsxmWbF93KRIyh9xpuHlh0zDL+shUcf5nHDjFOg9YLiQ763eiA==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.1.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@google-cloud/bigquery": { + "version": "7.9.4", + "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-7.9.4.tgz", + "integrity": "sha512-C7jeI+9lnCDYK3cRDujcBsPgiwshWKn/f0BiaJmClplfyosCLfWE83iGQ0eKH113UZzjR9c9q7aZQg0nU388sw==", + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/common": "^5.0.0", + "@google-cloud/paginator": "^5.0.2", + "@google-cloud/precise-date": "^4.0.0", + "@google-cloud/promisify": "4.0.0", + "arrify": "^2.0.1", + "big.js": "^6.0.0", + "duplexify": "^4.0.0", + "extend": "^3.0.2", + "is": "^3.3.0", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/common": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-5.0.2.tgz", + "integrity": "sha512-V7bmBKYQyu0eVG2BFejuUjlBt+zrya6vtsKdY+JxMM/dNntPF41vZ9+LhOshEUH01zOHEqBSvI7Dad7ZS6aUeA==", + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "arrify": "^2.0.1", + "duplexify": "^4.1.1", + "extend": "^3.0.2", + "google-auth-library": "^9.0.0", + "html-entities": "^2.5.2", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/dataflow": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@google-cloud/dataflow/-/dataflow-3.3.0.tgz", + "integrity": "sha512-b6ZqHP9bcmenoyKfTWfZJIv+yZYyaEY+60LOoXYOfM3wasC9jMEsILxHxAinXQZOuIM8MK2zdYRCFJUq/l2wtw==", + "license": "Apache-2.0", + "dependencies": { + "google-gax": "^4.0.3" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/firestore": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-8.7.0.tgz", + "integrity": "sha512-EvMpZQUXkTRdweSvOu6VL6EEQwHjHAgWz2UYZR+Mj6Ao52S+TWieHbSn15jiNnEw8F8RhbZj7IGXZ1PFB1eA+A==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "fast-deep-equal": "^3.1.3", + "functional-red-black-tree": "^1.0.1", + "google-gax": "^5.0.1", + "protobufjs": "^7.5.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@google-cloud/firestore/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "optional": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@google-cloud/firestore/node_modules/gaxios": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.3.0.tgz", + "integrity": "sha512-RB5vLV+vvQeoFPCX4QMK6/hjVkbIamPp1QSUD0CiZcnj12qbpiL+pLbYtgD+oZkWl0tl9z+o2Utp+MpM3QRhBA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/gcp-metadata": { + "version": "8.1.4", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.4.tgz", + "integrity": "sha512-iJ9KMsiu+xKtNRX0PmGLSaIU3bUBAyzWTyqKemKPzNPsmmsBCQYmlNg+brEbES7IHSXtdVwzBPzx1vz3FAaipw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "gaxios": "7.1.3", + "google-logging-utils": "1.1.3", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/gcp-metadata/node_modules/gaxios": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.3.tgz", + "integrity": "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2", + "rimraf": "^5.0.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/google-auth-library": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.5.0.tgz", + "integrity": "sha512-7ABviyMOlX5hIVD60YOfHw4/CxOfBhyduaYB+wbFWCWoni4N7SLcV46hrVRktuBbZjFC9ONyqamZITN7q3n32w==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.0.0", + "gcp-metadata": "^8.0.0", + "google-logging-utils": "^1.0.0", + "gtoken": "^8.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/google-gax": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-5.0.8.tgz", + "integrity": "sha512-M4vpZcXQIC1gqIVGQ7eaU3jXQA6zecStyTXu514TYfThlgSurYJOxHZo9fzU6hAgwPWvuEynAVHWyaIk80VeEA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@grpc/grpc-js": "^1.12.6", + "@grpc/proto-loader": "^0.8.0", + "duplexify": "^4.1.3", + "google-auth-library": "10.5.0", + "google-logging-utils": "1.1.3", + "node-fetch": "^3.3.2", + "object-hash": "^3.0.0", + "proto3-json-serializer": "3.0.4", + "protobufjs": "^7.5.4", + "retry-request": "^8.0.2", + "rimraf": "^5.0.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/firestore/node_modules/gtoken": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-8.0.0.tgz", + "integrity": "sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==", + "license": "MIT", + "optional": true, + "dependencies": { + "gaxios": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "optional": true, + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@google-cloud/firestore/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT", + "optional": true + }, + "node_modules/@google-cloud/firestore/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "optional": true, + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/@google-cloud/firestore/node_modules/proto3-json-serializer": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-3.0.4.tgz", + "integrity": "sha512-E1sbAYg3aEbXrq0n1ojJkRHQJGE1kaE/O6GLA94y8rnJBfgvOPTOd1b9hOceQK1FFZI9qMh1vBERCyO2ifubcw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "protobufjs": "^7.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/retry-request": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-8.0.4.tgz", + "integrity": "sha512-pI6/7eabUYkZxamkOq0g0uMxKLLGnjzhefY+vL8bVXag5rto4OU2YBTPytWLuHH7aEKD6fn7kJycQfid0Mwnkw==", + "license": "MIT", + "optional": true, + "dependencies": { + "extend": "^3.0.2", + "teeny-request": "^10.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore/node_modules/teeny-request": { + "version": "10.1.4", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.4.tgz", + "integrity": "sha512-R1Cg4Vu0UULeDfHL/kjABLaTW++9yD/B6n2g48y5dJ04hsEaxcfmAqbNDzNsbqAYJyIpZafjklLG9YxRu9uzOg==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2", + "stream-events": "^1.0.5" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/paginator": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", + "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", + "license": "Apache-2.0", + "dependencies": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/precise-date": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-4.0.0.tgz", + "integrity": "sha512-1TUx3KdaU3cN7nfCdNf+UVqA/PSX29Cjcox3fZZBtINlRrXVTmUkQnCKv2MbBUbCopbK4olAT1IHl76uZyCiVA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/projectify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", + "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/promisify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", + "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/storage": { + "version": "7.21.0", + "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.21.0.tgz", + "integrity": "sha512-l+IFTkd+6Y5LoAuXyYCKNAKtw/Ci+rAMqgdTB1jv4iZiLhw0rtq+0qjIRbBizXkNzEFmXiXUW0H7sZQQvk1ffA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@google-cloud/paginator": "^5.0.0", + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "<4.1.0", + "abort-controller": "^3.0.0", + "async-retry": "^1.3.3", + "duplexify": "^4.1.3", + "fast-xml-parser": "^5.3.4", + "gaxios": "^6.0.2", + "google-auth-library": "^9.6.3", + "html-entities": "^2.5.2", + "mime": "^3.0.0", + "p-limit": "^3.0.1", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.4.tgz", + "integrity": "sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/grpc-js/node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.7.15", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", + "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.2.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "optional": true, + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT", + "optional": true + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "optional": true, + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "optional": true, + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, + "node_modules/@napi-rs/lzma-linux-x64-gnu": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz", + "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.20 || ^24.12 || >=25" + } + }, + "node_modules/@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT", + "optional": true + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.4.tgz", + "integrity": "sha512-RrPokAb7dmbxFoeO3TloqHyOjgye8RkBhSqmp4aJMIex4c9r46ZstPnleDQOq1t46VOVjwIuwNogIqbodV1Vvg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.62.4.tgz", + "integrity": "sha512-JKuJc+pnpks2pjy7L/N3v/cAkZxYlnmuZoD840ldbMI5KDbC4iO9NKwPKYdjYFCMAIIlBzYSFHxIJVYzRo2/8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.62.4.tgz", + "integrity": "sha512-krw5uS2STmvJ02x0uTXHbqQNuz+9eZ1iw+qXk9dmW2gvV4jV7O2hEoOnuhFrpOPiel1mBFtqbxYZZtC46hXLOw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.62.4.tgz", + "integrity": "sha512-wsTxtgApb4PrOsNJIm0FZ1h3WvCC+k9uxLJ4ad75hgoS4NiRes2SoJFlDAyMwiUY8IssDqGcHbXuN0sx1tfF1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.62.4.tgz", + "integrity": "sha512-GUOnQlyZe3yAXhWOtOMsn5Qkrv5E5mZXa0thbARWi5Ei2szlVXJFQhddZ4HbAzh8q92w5twp+CQvs/eFanz9YQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.62.4.tgz", + "integrity": "sha512-/Y7f3QuxjzPKsjA/rfEDa3+0vXqyjmJ50Ln8dPpCmWkKTrUoWHG1cWhTqaAMLob2m2nESWuC7yGrREz019Ztqg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.62.4.tgz", + "integrity": "sha512-81wiiX3v7aqy+T+bT61TJ78yJjRquqFFTTbAPt08imfQQzkPIW8t6aJbkTagtCCrXMNc9D66+geqlK7ydLPNqA==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.62.4.tgz", + "integrity": "sha512-9kmDIvNZqdoHOBZgNtpTBeLWYO/LVipM3H/j62P8848/l/VPEQL6N3uxU9pvP1oZAsXyC2MEnFP3ovRjo7WYNQ==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.62.4.tgz", + "integrity": "sha512-CcnXHWnXg69g+DX5VWL3FHts3qMRN2uVEHX+BZvGLdd07/gXkn3ePjYtO1LDJvxkGKVHMclKBRa1QUTH+6toYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.62.4.tgz", + "integrity": "sha512-iFOibiHnTRuhrWLlRsOQFdZJJIa7S8OwkneJr4ocALP16u5yk6lWLINFwhHaEqBFMsKDUZofLkGos7+CPzGB3g==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.62.4.tgz", + "integrity": "sha512-XnWYMI7euHlb5a871xPja+Gm7DRCFU+FGRrtS2sMq9N8FvqtpagUy6gD4YOemC5MRk9xbh8+jYMEJbigFQwsgA==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.62.4.tgz", + "integrity": "sha512-qGDAlO0U8xedCcsdRm9oaoQY8DAx/QT7uIxJWhCdx0ceIWX783UC9QSYkdpzAe29wNiVfp24+bZdQmn49o45SQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.62.4.tgz", + "integrity": "sha512-ru4H6ezD7ysA5EiEK6qkkaEb4modH8CTej6kUy/gQi20u3kB3G7Zn8snXXkeJSCOFKG/rbPPtM/+9Wgas1961w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.62.4.tgz", + "integrity": "sha512-2W4MO5WQVJnbJaZdvDb9rhBDuFU1nKIepPFpJUBsTh2k1YY2g+ODViaWuyOAjQ5cOP7NvrvLzt3wvHOoiAvc7w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.62.4.tgz", + "integrity": "sha512-+fxjfuoAmVMCYV5QyjoIpu0cp5DOiOTeqYFk1AVaxGr+/ravWLX89XfQmptsoWcaVy/TGf2hexzbUOrCQIL1CQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.62.4.tgz", + "integrity": "sha512-jTn8JfHGL4djjFxPuM06LmNUJDsst2jeVlsd9OmIH6zc5sC9K6rIuO4YajXatLUpBmBKl6b35ro1QZocLi+tcA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.62.4.tgz", + "integrity": "sha512-oCJCJL4pXsoDcP2QZ+JVlPTIRc6266zsIaeJJsWImmF7HO0W8nb6HuSgZlMWxJwaPf8ehbSw8yo0EUw925hKsA==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.62.4.tgz", + "integrity": "sha512-W69hukhZ3KKNRCaMIEzKvcFye42hh0FE1+YoYaf5+Ikacuftoco6yO/xouz0hc5d5W/s3yBro5jRiuEE/Q5vUw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.62.4.tgz", + "integrity": "sha512-qiXbGG2jkjXhzXpsFZSR2Xpb8DN/UaxYsbb/STbuR/6fpaDgRmmaq1B/LmtF2wQFOFOSsK2jdE0RZ3a0zHn4QA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.62.4.tgz", + "integrity": "sha512-nWeM//hxv8mIo6jD7Hu4o48DVmV9pbV6gsKaWU+4NFyqHoPKwrkRiZGLKUhOBk8qNmDmpwFtPKg80Bo/Tn4xiQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.62.4.tgz", + "integrity": "sha512-s62SQ/vgsRSvMwDkOEfTqfgASF0f26ZNaQuTA6Aok5lrikf89yI2W0gFHvZb2Jpgc6N8JnOKZgCK2iciO3CsxQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.62.4.tgz", + "integrity": "sha512-J6wGf8TVGbXJq+HH+ttTvrcfNKPbuZecV6KT1B8I18BC5IURUh5kl4Yl5OEP5eFIUoI5BWxCsyYMhFsDx8kekw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.62.4.tgz", + "integrity": "sha512-zmfrQd/0wu6oJs8Vq8KwY/YtsKSsLtKe/HwAP4Wqy8LhWjeT55fHRAkOhYQ12wI3ayS4Tt12d5CDRD7N96SAYQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.62.4.tgz", + "integrity": "sha512-qPzHqdj9rfUD+w79dtE07zi/kFwKyCJqplp5K5ygeLTp7jLpAoc16OAH39HSmRC9UpozaecsleI8uAdEj6v2yw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.62.4.tgz", + "integrity": "sha512-zD6NdeWEByGE9QF9vCrlJ5YQB4oq9q91kPZS37Jwj5hOkvR1lTBSpsKhKDw4IJtbQ35LsTS1HD9DZYGKIshU1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@tootallnate/once": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.1.tgz", + "integrity": "sha512-HqmEUIGRJ5fSXchkVgR5F7qn48bDBzv0kWj/Kfu5e6uci4UlEeng4331LnBkWffb++Ei3FOVLxo8JJWMFBDMeQ==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/@types/body-parser": { + "version": "1.19.6", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", + "integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==", + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/caseless": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", + "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==", + "license": "MIT" + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/cors": { + "version": "2.8.19", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.19.tgz", + "integrity": "sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/express": { + "version": "4.17.25", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", + "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "^1" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "4.19.9", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.9.tgz", + "integrity": "sha512-QP2ESEe/ImWY0HDwNAnK9PvEffUyhLTnWkk7KXzHfyeWAnlrDe1fN77bXl6ia8KT3wPlmA7t9/VPRpnf4Ex9sg==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/http-errors": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", + "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", + "license": "MIT" + }, + "node_modules/@types/jsonwebtoken": { + "version": "9.0.10", + "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.10.tgz", + "integrity": "sha512-asx5hIG9Qmf/1oStypjanR7iKTv0gXQ1Ov/jfrX6kS/EO0OFni8orbmGCn0672NHR3kXHwpAwR+B368ZGN/2rA==", + "license": "MIT", + "dependencies": { + "@types/ms": "*", + "@types/node": "*" + } + }, + "node_modules/@types/long": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", + "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==", + "license": "MIT" + }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "license": "MIT" + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.1.tgz", + "integrity": "sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/qs": { + "version": "6.15.1", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.1.tgz", + "integrity": "sha512-GZHUBZR9hckSUhrxmp1nG6NwdpM9fCunJwyThLW1X3AyHgd9IlHb6VANpQQqDr2o/qQp6McZ3y/IA2rVzKzSbw==", + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "license": "MIT" + }, + "node_modules/@types/request": { + "version": "2.48.13", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.13.tgz", + "integrity": "sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg==", + "license": "MIT", + "dependencies": { + "@types/caseless": "*", + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.5" + } + }, + "node_modules/@types/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", + "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.10", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz", + "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==", + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "<1" + } + }, + "node_modules/@types/serve-static/node_modules/@types/send": { + "version": "0.17.6", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz", + "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==", + "license": "MIT", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", + "license": "MIT" + }, + "node_modules/@vitest/expect": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.7.tgz", + "integrity": "sha512-E8eBXaKibuvH2pSZErOjdVb5vF4PbKYcrnluBTYxEk1l/VhhwZg1kZQsdtjq+CsF5CFydf2Rdkz7jDHKSisi3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/spy": "3.2.7", + "@vitest/utils": "3.2.7", + "chai": "^5.2.0", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.7.tgz", + "integrity": "sha512-Trr0hYO9CM3Wj6ksWHRhK9IZpIY6wTMO5u/MqXurMxT57sWBaOPEtP3Oq60ihZuh5JsiagKfz95OcxdEP6dBrA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "3.2.7", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.17" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.7.tgz", + "integrity": "sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.7.tgz", + "integrity": "sha512-sB9y4ovltoQP+WaUPwmSxO9WIg9Ig694Di5PalVPsYHklAdE027mehpWF2SQSVq+k6sFgaivbTjTJwZLSHbedA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "3.2.7", + "pathe": "^2.0.3", + "strip-literal": "^3.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.7.tgz", + "integrity": "sha512-7C+MwShwtBSI5Buwoyg3s/iY1eHL9PKAf+O1wVh/TdnjXUtkoL/9YQtre90i4MtNXM6edP1wJ2zOBpfCyhIS7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.7", + "magic-string": "^0.30.17", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.7.tgz", + "integrity": "sha512-Q2eQGI6d2L/hBtZ0qNuKcAGid68XK6cv1xsoaIma6PaJhHPoqcEJhYpXZ/5myCMqkNgtP6UKuBhbc0nHKnrkuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyspy": "^4.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.7.tgz", + "integrity": "sha512-x6BDOd7dyo3PFLY3I9/HJ25X/6OurhGXk2/B9gOZNPF7XDVjeBK4k01lQE5uvDpbuheErh91qYuE1E2OEjK3Rw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.7", + "loupe": "^3.1.4", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "optional": true + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/async-retry": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", + "license": "MIT", + "optional": true, + "dependencies": { + "retry": "0.13.1" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT", + "optional": true + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/big.js": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-6.2.2.tgz", + "integrity": "sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==", + "license": "MIT", + "engines": { + "node": "*" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/bigjs" + } + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/body-parser": { + "version": "1.20.6", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.6.tgz", + "integrity": "sha512-p5tAzS57i5MV9fZFDj9LeIiTZEufbSe2eDozP+ElheSUq1m74CRq1jI4mYNDdVs9vQztXFLuk/Gd6BWTdwRJ5g==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.15.1", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/brace-expansion": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", + "license": "MIT", + "optional": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/check-error": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", + "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "license": "MIT" + }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "optional": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/deep-eql": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", + "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/duplexify": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", + "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.2" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT", + "optional": true + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true, + "license": "MIT" + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/express": { + "version": "4.22.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", + "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.5", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.15.1", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-xml-builder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.0.tgz", + "integrity": "sha512-F74cZEdCvuw9P41GAC3rod4X04jjWGM1JPEv/GWSqFTWLsdyMSBMBMlm9Hk3GLBgLBbdBNY8yee0pQh2RBVESQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "optional": true, + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.10.1.tgz", + "integrity": "sha512-IEMIf7298kXuZSRFoGfMYrl7is8LpavODgbNz1cwIudv7KwVFnuU+UsMporfq6PD6aXSlawZlARiA3UywCTfMw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "optional": true, + "dependencies": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.1", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/faye-websocket": { + "version": "0.11.4", + "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.4.tgz", + "integrity": "sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==", + "license": "Apache-2.0", + "dependencies": { + "websocket-driver": ">=0.5.1" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/firebase-admin": { + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/firebase-admin/-/firebase-admin-14.2.0.tgz", + "integrity": "sha512-zfs5PdccEgjX479bbMmz95Rqc7ttQ4LV3qkGFlPiqOaOu/suBZc3fG52Qzn2hQjiSHkBM4NP2AZV5r2NvAt0TQ==", + "license": "Apache-2.0", + "dependencies": { + "@fastify/busboy": "^3.0.0", + "@firebase/database-compat": "^2.1.4", + "@firebase/database-types": "^1.0.20", + "fast-deep-equal": "^3.1.1", + "google-auth-library": "^10.6.2", + "jsonwebtoken": "^9.0.0", + "jwks-rsa": "^4.0.1" + }, + "engines": { + "node": ">=22" + }, + "optionalDependencies": { + "@google-cloud/firestore": "^8.6.0", + "@google-cloud/storage": "^7.19.0" + } + }, + "node_modules/firebase-admin/node_modules/gaxios": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.3.0.tgz", + "integrity": "sha512-RB5vLV+vvQeoFPCX4QMK6/hjVkbIamPp1QSUD0CiZcnj12qbpiL+pLbYtgD+oZkWl0tl9z+o2Utp+MpM3QRhBA==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/firebase-admin/node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/firebase-admin/node_modules/google-auth-library": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.9.1.tgz", + "integrity": "sha512-i1ydyHrqcIxXkWh/uBmVkzCvIuq5yiK2ATndIe5XxKholrG/MTYP9xGYka4sQhrbIAgGjL2B6NOE7rFaiF3fXw==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/firebase-admin/node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/firebase-admin/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/firebase-functions": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/firebase-functions/-/firebase-functions-7.3.0.tgz", + "integrity": "sha512-S3JhjESOWMq13iDodwgUPWNQ3gLAu7oTLDwF7hTtisyjVanEeQzYezgW+JTfd8I0kCJQzjGPC/wHQ19IL7CgaA==", + "license": "MIT", + "dependencies": { + "@types/cors": "^2.8.5", + "@types/express": "^4.17.21", + "cors": "^2.8.5", + "express": "^4.21.0", + "protobufjs": "^7.2.2" + }, + "bin": { + "firebase-functions": "lib/bin/firebase-functions.js" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "@apollo/server": "^5.2.0", + "@as-integrations/express4": "^1.1.2", + "firebase-admin": "^11.10.0 || ^12.0.0 || ^13.0.0 || ^14.0.0", + "graphql": "^16.12.0" + }, + "peerDependenciesMeta": { + "@apollo/server": { + "optional": true + }, + "@as-integrations/express4": { + "optional": true + }, + "graphql": { + "optional": true + } + } + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "optional": true, + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/form-data": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.6.tgz", + "integrity": "sha512-Ogz/E85h9tlfJzpI6TuFpGcHZFhLrb9Gw8wq9v40CxSCPnv7ahKr6Xgtkn0KYCDQJ8DNn5VoMO8EXr9V5PadyA==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.4", + "mime-types": "^2.1.35", + "safe-buffer": "^5.2.1" + }, + "engines": { + "node": ">= 0.12" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g==", + "license": "MIT", + "optional": true + }, + "node_modules/gaxios": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", + "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^6.1.1", + "google-logging-utils": "^0.0.2", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "optional": true, + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/google-auth-library": { + "version": "9.15.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", + "integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/google-gax": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-4.6.1.tgz", + "integrity": "sha512-V6eky/xz2mcKfAd1Ioxyd6nmA61gao3n01C+YeuIwu3vzM9EDR6wcVzMSIbLMDXWeoi9SHYctXuKYC5uJUT3eQ==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.10.9", + "@grpc/proto-loader": "^0.7.13", + "@types/long": "^4.0.0", + "abort-controller": "^3.0.0", + "duplexify": "^4.0.0", + "google-auth-library": "^9.3.0", + "node-fetch": "^2.7.0", + "object-hash": "^3.0.0", + "proto3-json-serializer": "^2.0.2", + "protobufjs": "^7.3.2", + "retry-request": "^7.0.0", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/google-logging-utils": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", + "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "license": "MIT", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/html-entities": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.6.0.tgz", + "integrity": "sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/mdevils" + }, + { + "type": "patreon", + "url": "https://patreon.com/mdevils" + } + ], + "license": "MIT" + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/http-parser-js": { + "version": "0.5.10", + "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.10.tgz", + "integrity": "sha512-Pysuw9XpUq5dVc/2SMHpuTY01RFl8fttgcyunjL7eEMhGM3cI4eOmiCycJDVCo/7O7ClfQD3SaI6ftDzqOXYMA==", + "license": "MIT" + }, + "node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "license": "MIT", + "dependencies": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/http-proxy-agent/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/http-proxy-agent/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/http-proxy-agent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/https-proxy-agent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/is/-/is-3.3.2.tgz", + "integrity": "sha512-a2xr4E3s1PjDS8ORcGgXpWx6V+liNs+O3JRD2mb9aeugD7rtkkZ0zgLdYgw0tWsKhsdiezGYptSiMlVazCBTuQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-unsafe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.0.tgz", + "integrity": "sha512-2LdV822R+wmI86unXA93WCFpL6g+av8ynWk0nrHyJqGop5VoocYsSLFgN8jrfalT6iGeLNM4KXuVSsULP53kEA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "optional": true + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC", + "optional": true + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "license": "BlueOak-1.0.0", + "optional": true, + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/jose": { + "version": "6.2.8", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.8.tgz", + "integrity": "sha512-Bsdjwm3Qsd/P0jR+BHDe3LytDfY7WBq2HmCCLIwuVRHMuEC9ae7/R474GIUdF1NgCyZjzVo/A9DOiOBtXq8ZoQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/js-tokens": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", + "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/jsonwebtoken": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz", + "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==", + "license": "MIT", + "dependencies": { + "jws": "^4.0.1", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=12", + "npm": ">=6" + } + }, + "node_modules/jsonwebtoken/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jwks-rsa": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-4.1.0.tgz", + "integrity": "sha512-sbkByqyATKYJP5F4RXj03N5TUNC0QLTjCAZvwTzC4BwJZ8e0/cWxN8YROnyUth2g1/ONWi4eSFHeu6oYalrc3Q==", + "license": "MIT", + "dependencies": { + "@types/jsonwebtoken": "^9.0.4", + "debug": "^4.3.4", + "jose": "^6.1.3", + "limiter": "^1.1.5", + "lru-cache": "^11.0.0", + "lru-memoizer": "^3.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >= 23.0.0" + } + }, + "node_modules/jwks-rsa/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/jwks-rsa/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/limiter": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz", + "integrity": "sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA==" + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, + "node_modules/lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==", + "license": "MIT" + }, + "node_modules/lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", + "license": "MIT" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", + "license": "MIT" + }, + "node_modules/lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", + "license": "MIT" + }, + "node_modules/lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==", + "license": "MIT" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", + "license": "MIT" + }, + "node_modules/lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", + "license": "MIT" + }, + "node_modules/lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", + "license": "MIT" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/loupe": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", + "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "11.5.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", + "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/lru-memoizer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/lru-memoizer/-/lru-memoizer-3.0.0.tgz", + "integrity": "sha512-m83w/cYXLdUIboKSPxzPAGfYnk+vqeDYXuoSrQRw1q+yVEd8IXhvMufN8Q5TIPe7e2jyX4SRNrDJI2Skw1yznQ==", + "license": "MIT", + "dependencies": { + "lodash.clonedeep": "^4.5.0", + "lru-cache": "^11.0.1" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "license": "MIT", + "optional": true, + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "license": "ISC", + "optional": true, + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "optional": true, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.17", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.17.tgz", + "integrity": "sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0", + "optional": true + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "license": "BlueOak-1.0.0", + "optional": true, + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "license": "ISC", + "optional": true + }, + "node_modules/path-to-regexp": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", + "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", + "license": "MIT" + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/pathval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", + "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.16" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.25", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz", + "integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.16", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/proto3-json-serializer": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.2.tgz", + "integrity": "sha512-SAzp/O4Yh02jGdRc+uIrGoe87dkN/XtwxfZ4ZyafJHymd79ozp5VG5nyZ7ygqPM5+cpLDjjGnYFUkngonyDPOQ==", + "license": "Apache-2.0", + "dependencies": { + "protobufjs": "^7.2.5" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/protobufjs": { + "version": "7.6.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", + "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/retry-request": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", + "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", + "license": "MIT", + "dependencies": { + "@types/request": "^2.48.8", + "extend": "^3.0.2", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/rimraf": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.10.tgz", + "integrity": "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "glob": "^10.3.7" + }, + "bin": { + "rimraf": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rollup": { + "version": "4.62.4", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.62.4.tgz", + "integrity": "sha512-RXOqwaPsBGjMNMa4sQjDjHieHEZDFoj/Rdr46l2MU5DfEs16wHJPC2RPTPHWhNl+M3aI472LLqFkFKut4SblOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@napi-rs/lzma-linux-x64-gnu": "1.5.1", + "@rollup/rollup-android-arm-eabi": "4.62.4", + "@rollup/rollup-android-arm64": "4.62.4", + "@rollup/rollup-darwin-arm64": "4.62.4", + "@rollup/rollup-darwin-x64": "4.62.4", + "@rollup/rollup-freebsd-arm64": "4.62.4", + "@rollup/rollup-freebsd-x64": "4.62.4", + "@rollup/rollup-linux-arm-gnueabihf": "4.62.4", + "@rollup/rollup-linux-arm-musleabihf": "4.62.4", + "@rollup/rollup-linux-arm64-gnu": "4.62.4", + "@rollup/rollup-linux-arm64-musl": "4.62.4", + "@rollup/rollup-linux-loong64-gnu": "4.62.4", + "@rollup/rollup-linux-loong64-musl": "4.62.4", + "@rollup/rollup-linux-ppc64-gnu": "4.62.4", + "@rollup/rollup-linux-ppc64-musl": "4.62.4", + "@rollup/rollup-linux-riscv64-gnu": "4.62.4", + "@rollup/rollup-linux-riscv64-musl": "4.62.4", + "@rollup/rollup-linux-s390x-gnu": "4.62.4", + "@rollup/rollup-linux-x64-gnu": "4.62.4", + "@rollup/rollup-linux-x64-musl": "4.62.4", + "@rollup/rollup-openbsd-x64": "4.62.4", + "@rollup/rollup-openharmony-arm64": "4.62.4", + "@rollup/rollup-win32-arm64-msvc": "4.62.4", + "@rollup/rollup-win32-ia32-msvc": "4.62.4", + "@rollup/rollup-win32-x64-gnu": "4.62.4", + "@rollup/rollup-win32-x64-msvc": "4.62.4", + "fsevents": "~2.3.2" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.4.1", + "range-parser": "~1.2.1", + "statuses": "~2.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "~0.19.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "optional": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "optional": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true, + "license": "MIT" + }, + "node_modules/stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "license": "MIT", + "dependencies": { + "stubs": "^3.0.0" + } + }, + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", + "license": "MIT" + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "optional": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "optional": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-literal": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-3.1.0.tgz", + "integrity": "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-tokens": "^9.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/strnum": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.1.tgz", + "integrity": "sha512-M9eUSMT2dCB2cTNPG7UYj6KuK7RJR2SN2+yCV/fTW3xzTCS6EaGZ5pSMgDIjB7r8zSfTGk+dvvn9rTjpVS9Mwg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "optional": true, + "dependencies": { + "anynum": "^1.0.1" + } + }, + "node_modules/stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==", + "license": "MIT" + }, + "node_modules/teeny-request": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", + "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", + "license": "Apache-2.0", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.9", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/teeny-request/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/teeny-request/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/teeny-request/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/teeny-request/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinypool": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", + "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/tinyrainbow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", + "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tinyspy": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-4.0.4.tgz", + "integrity": "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vite": { + "version": "7.3.6", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.6.tgz", + "integrity": "sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.27.0 || ^0.28.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite-node": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-3.2.4.tgz", + "integrity": "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.4.1", + "es-module-lexer": "^1.7.0", + "pathe": "^2.0.3", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "bin": { + "vite-node": "vite-node.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/vite-node/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/vite-node/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/vitest": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.7.tgz", + "integrity": "sha512-KrxIJ62Fd89gfysR4WotlgZABiz2dqFPgqGzX7s+CwsqLFomRH7777ZcrOD6+WVAh7khPQP41A+BKbpcJFrdEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/expect": "3.2.7", + "@vitest/mocker": "3.2.7", + "@vitest/pretty-format": "^3.2.7", + "@vitest/runner": "3.2.7", + "@vitest/snapshot": "3.2.7", + "@vitest/spy": "3.2.7", + "@vitest/utils": "3.2.7", + "chai": "^5.2.0", + "debug": "^4.4.1", + "expect-type": "^1.2.1", + "magic-string": "^0.30.17", + "pathe": "^2.0.3", + "picomatch": "^4.0.2", + "std-env": "^3.9.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.14", + "tinypool": "^1.1.1", + "tinyrainbow": "^2.0.0", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", + "vite-node": "3.2.4", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/debug": "^4.1.12", + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "@vitest/browser": "3.2.7", + "@vitest/ui": "3.2.7", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/debug": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/vitest/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/vitest/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/websocket-driver": { + "version": "0.7.5", + "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.5.tgz", + "integrity": "sha512-ZL2+3c7kMBdIRCMz6l8jQMHyGVxj+UL+xVk74Ombiciboca8rHa15L86B19E5oh1pL9Ii/uj54gtsIrZGMo6zA==", + "license": "Apache-2.0", + "dependencies": { + "http-parser-js": ">=0.5.1", + "safe-buffer": ">=5.1.0", + "websocket-extensions": ">=0.1.1" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/websocket-extensions": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz", + "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "optional": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "optional": true, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/kits/firestore-incremental-capture/package.json b/kits/firestore-incremental-capture/package.json index ac529337f..100af95a1 100644 --- a/kits/firestore-incremental-capture/package.json +++ b/kits/firestore-incremental-capture/package.json @@ -1,8 +1,7 @@ { "name": "@firebase/firestore-incremental-capture", "version": "0.1.0", - "private": true, - "description": "Incremental point-in-time capture of Firestore changes", + "description": "Incremental point-in-time capture of Firestore changes as a deployable Firebase Function", "license": "Apache-2.0", "main": "lib/index.js", "types": "lib/index.d.ts", @@ -20,7 +19,21 @@ "node": ">=22" }, "scripts": { - "build": "tsc -b", - "clean": "tsc -b --clean" + "build": "tsc -p tsconfig.json", + "clean": "tsc -b --clean", + "test": "vitest run", + "deploy": "pnpm build && firebase deploy --only functions", + "serve": "firebase emulators:start --only functions" + }, + "dependencies": { + "@google-cloud/bigquery": "^7.6.0", + "@google-cloud/dataflow": "^3.2.0", + "firebase-admin": "^14.1.0", + "firebase-functions": "7.3.0" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^6.0.0", + "vitest": "^3.2.4" } } diff --git a/kits/firestore-incremental-capture/scripts/setup.sh b/kits/firestore-incremental-capture/scripts/setup.sh new file mode 100755 index 000000000..8f6716477 --- /dev/null +++ b/kits/firestore-incremental-capture/scripts/setup.sh @@ -0,0 +1,242 @@ +#!/usr/bin/env bash +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Provisions the restoration prerequisites that the deployed functions cannot +# provision themselves: they need gcloud and a Maven build, neither of which +# exists in the Cloud Functions runtime. +# +# Every step is idempotent, so re-running after a partial failure is safe. +# +# Usage: +# PROJECT_ID=my-project BACKUP_INSTANCE_ID=my-backup ./scripts/setup.sh +# +# Required: +# PROJECT_ID Project holding the databases, changelog and jobs. +# BACKUP_INSTANCE_ID Firestore database to restore into. Created if absent. +# Must not be the captured database. +# Optional: +# SOURCE_DATABASE Captured database. Default "(default)". +# DATABASE_LOCATION Location for a newly created backup database. +# Default "nam5". +# LOCATION Region for the functions and Artifact Registry. +# Default "us-central1". +# BUCKET_NAME Bucket holding the flex template. Defaults to the +# project's default bucket. +# INSTANCE_ID Namespace for the deployed resources. Must match the +# kit's INSTANCE_ID param. Default +# "firestore-incremental-capture". +# SERVICE_ACCOUNT Runtime service account of the deployed functions. +# Defaults to the App Engine default service account. + +set -euo pipefail + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly PIPELINE_DIR="${SCRIPT_DIR}/../pipeline" + +readonly PROJECT_ID="${PROJECT_ID:-}" +readonly BACKUP_INSTANCE_ID="${BACKUP_INSTANCE_ID:-}" +readonly SOURCE_DATABASE="${SOURCE_DATABASE:-(default)}" +readonly DATABASE_LOCATION="${DATABASE_LOCATION:-nam5}" +readonly LOCATION="${LOCATION:-us-central1}" +readonly INSTANCE_ID="${INSTANCE_ID:-firestore-incremental-capture}" +readonly JAR_NAME="restore-firestore.jar" + +readonly GREEN='\033[0;32m' +readonly YELLOW='\033[1;33m' +readonly RED='\033[0;31m' +readonly NC='\033[0m' + +step() { echo -e "\n${YELLOW}==> $*${NC}"; } +ok() { echo -e "${GREEN} $*${NC}"; } +die() { + echo -e "${RED}Error: $*${NC}" >&2 + exit 1 +} + +require_config() { + [[ -n "${PROJECT_ID}" ]] || die "PROJECT_ID is required." + [[ -n "${BACKUP_INSTANCE_ID}" ]] || die "BACKUP_INSTANCE_ID is required." + + # A restoration batch-writes over the backup database. Pointing it at the + # captured database would destroy the data being restored. + [[ "${BACKUP_INSTANCE_ID}" != "${SOURCE_DATABASE}" ]] || + die "BACKUP_INSTANCE_ID must differ from SOURCE_DATABASE (${SOURCE_DATABASE})." + + command -v gcloud >/dev/null || die "gcloud is required but was not found." + command -v mvn >/dev/null || die "Maven is required but was not found." +} + +# Resolves the bucket holding the flex template. Projects created after +# September 2024 default to .firebasestorage.app; older ones to .appspot.com. +resolve_bucket() { + if [[ -n "${BUCKET_NAME:-}" ]]; then + echo "${BUCKET_NAME}" + return + fi + + local buckets + buckets="$(gcloud storage buckets list --project="${PROJECT_ID}" --format='value(name)')" + + if grep -qx "${PROJECT_ID}.firebasestorage.app" <<<"${buckets}"; then + echo "${PROJECT_ID}.firebasestorage.app" + elif grep -qx "${PROJECT_ID}.appspot.com" <<<"${buckets}"; then + echo "${PROJECT_ID}.appspot.com" + else + die "Could not find a default bucket for ${PROJECT_ID}. Set BUCKET_NAME." + fi +} + +resolve_service_account() { + if [[ -n "${SERVICE_ACCOUNT:-}" ]]; then + echo "${SERVICE_ACCOUNT}" + else + echo "${PROJECT_ID}@appspot.gserviceaccount.com" + fi +} + +enable_apis() { + step "Enabling required APIs" + gcloud services enable \ + bigquery.googleapis.com \ + cloudbuild.googleapis.com \ + dataflow.googleapis.com \ + firestore.googleapis.com \ + artifactregistry.googleapis.com \ + --project="${PROJECT_ID}" + ok "APIs enabled." +} + +# Restoration reads a PITR snapshot of the captured database, so PITR must be on +# before the point in time you later want to restore to. +enable_pitr() { + step "Enabling point-in-time recovery on ${SOURCE_DATABASE}" + gcloud firestore databases update \ + --database="${SOURCE_DATABASE}" \ + --enable-pitr \ + --project="${PROJECT_ID}" + ok "PITR enabled." +} + +create_backup_database() { + step "Ensuring backup database ${BACKUP_INSTANCE_ID} exists" + + if gcloud firestore databases describe \ + --database="${BACKUP_INSTANCE_ID}" \ + --project="${PROJECT_ID}" >/dev/null 2>&1; then + ok "Database already exists." + return + fi + + gcloud firestore databases create \ + --database="${BACKUP_INSTANCE_ID}" \ + --location="${DATABASE_LOCATION}" \ + --type=firestore-native \ + --project="${PROJECT_ID}" + ok "Database created." +} + +create_artifact_registry() { + step "Ensuring Artifact Registry repository ${INSTANCE_ID} exists" + + if gcloud artifacts repositories describe "${INSTANCE_ID}" \ + --location="${LOCATION}" \ + --project="${PROJECT_ID}" >/dev/null 2>&1; then + ok "Repository already exists." + return + fi + + gcloud artifacts repositories create "${INSTANCE_ID}" \ + --repository-format=docker \ + --location="${LOCATION}" \ + --project="${PROJECT_ID}" + ok "Repository created." +} + +grant_roles() { + local service_account="$1" + + step "Granting Dataflow roles to ${service_account}" + + # Launching a flex template needs dataflow.developer, and needs to act as the + # worker service account. + local role + for role in roles/dataflow.developer roles/iam.serviceAccountUser; do + gcloud projects add-iam-policy-binding "${PROJECT_ID}" \ + --member="serviceAccount:${service_account}" \ + --role="${role}" \ + --condition=None \ + --quiet >/dev/null + done + + gcloud artifacts repositories add-iam-policy-binding "${INSTANCE_ID}" \ + --location="${LOCATION}" \ + --project="${PROJECT_ID}" \ + --member="serviceAccount:${service_account}" \ + --role=roles/artifactregistry.writer \ + --condition=None \ + --quiet >/dev/null + + ok "Roles granted." +} + +# Built from the vendored source rather than downloaded: the prebuilt jar the +# extension fetched from GitHub is not a durable artifact. +build_jar() { + step "Building the restoration pipeline" + mvn -q -f "${PIPELINE_DIR}/pom.xml" clean package -DskipTests + local jar="${PIPELINE_DIR}/target/${JAR_NAME}" + [[ -f "${jar}" ]] || die "Expected ${jar} after the Maven build." + ok "Built ${jar}." + echo "${jar}" +} + +# The template path must match ResolvedCaptureConfig.flexTemplatePath, which is +# what the deployed function launches. +stage_flex_template() { + local jar="$1" bucket="$2" + local template_path="gs://${bucket}/${INSTANCE_ID}-dataflow-restore" + + step "Staging the Dataflow flex template at ${template_path}" + gcloud dataflow flex-template build "${template_path}" \ + --image-gcr-path "${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${INSTANCE_ID}/dataflow/restore:latest" \ + --sdk-language JAVA \ + --flex-template-base-image JAVA11 \ + --jar "${jar}" \ + --env FLEX_TEMPLATE_JAVA_MAIN_CLASS="com.pipeline.RestorationPipeline" \ + --project "${PROJECT_ID}" + ok "Template staged." +} + +main() { + require_config + + local bucket service_account jar + bucket="$(resolve_bucket)" + service_account="$(resolve_service_account)" + + enable_apis + enable_pitr + create_backup_database + create_artifact_registry + grant_roles "${service_account}" + jar="$(build_jar | tail -n 1)" + stage_flex_template "${jar}" "${bucket}" + + echo -e "\n${GREEN}Setup complete.${NC}" + echo "Deploy the kit with BACKUP_INSTANCE_ID=${BACKUP_INSTANCE_ID}, INSTANCE_ID=${INSTANCE_ID}, LOCATION=${LOCATION}, BUCKET_NAME=${bucket}." +} + +main "$@" diff --git a/kits/firestore-incremental-capture/src/bigquery.ts b/kits/firestore-incremental-capture/src/bigquery.ts new file mode 100644 index 000000000..dd84d193f --- /dev/null +++ b/kits/firestore-incremental-capture/src/bigquery.ts @@ -0,0 +1,91 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BigQuery } from "@google-cloud/bigquery"; +import type { ResolvedCaptureConfig } from "./capture-config"; +import { CHANGELOG_SCHEMA, type ChangelogRow } from "./changelog"; +import * as logs from "./logs"; + +/** + * BigQuery access for the changelog table. + */ +export class ChangelogTable { + private readonly bq: BigQuery; + + /** + * @param config - The resolved capture configuration. + * @param bq - BigQuery client, injectable for tests. + */ + constructor( + private readonly config: ResolvedCaptureConfig, + bq: BigQuery = new BigQuery({ projectId: config.projectId }) + ) { + this.bq = bq; + } + + /** + * Creates the changelog dataset and table if they do not already exist. + * + * Safe to call repeatedly: it is the provisioning path for both first deploy + * and redeploy, and existing resources are left untouched. + */ + async initialize(): Promise { + const { datasetId, tableId, datasetLocation } = this.config; + const dataset = this.bq.dataset(datasetId, { location: datasetLocation }); + const [datasetExists] = await dataset.exists(); + + if (datasetExists) { + logs.info(`BigQuery dataset already exists: ${datasetId}`); + } else { + logs.debug(`Creating BigQuery dataset: ${datasetId}`); + await this.bq.createDataset(datasetId, { location: datasetLocation }); + logs.info(`Created BigQuery dataset: ${datasetId}`); + } + + const table = dataset.table(tableId); + const [tableExists] = await table.exists(); + + if (tableExists) { + logs.info(`BigQuery table already exists: ${tableId}`); + return; + } + + logs.debug(`Creating BigQuery table: ${tableId}`); + await dataset.createTable(tableId, { + schema: [...CHANGELOG_SCHEMA], + location: datasetLocation, + }); + logs.info(`Created BigQuery table: ${tableId}`); + } + + /** + * Inserts changelog rows. + * + * @param rows - The rows to insert. + * @throws The underlying insert error. Callers run on a task queue and rely + * on the rejection to trigger a retry. + */ + async insert(rows: ChangelogRow[]): Promise { + const { datasetId, tableId } = this.config; + + try { + await this.bq.dataset(datasetId).table(tableId).insert(rows); + } catch (err) { + logs.error(`Failed to insert ${rows.length} changelog row(s)`, err); + throw err; + } + } +} diff --git a/kits/firestore-incremental-capture/src/capture-config.ts b/kits/firestore-incremental-capture/src/capture-config.ts new file mode 100644 index 000000000..cdbccf53d --- /dev/null +++ b/kits/firestore-incremental-capture/src/capture-config.ts @@ -0,0 +1,141 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Log verbosity accepted by the kit. */ +export type LogLevel = "debug" | "info" | "warn" | "error" | "silent"; + +/** + * Configuration as supplied by a caller. Only the fields without a sensible + * default are required; {@link resolveCaptureConfig} fills in the rest. + */ +export interface CaptureConfig { + /** GCP project holding the Firestore databases, BigQuery dataset and jobs. */ + projectId: string; + /** + * Collection to capture, relative to the database root. `{document=**}` + * captures every collection. + */ + syncCollectionPath: string; + /** Firestore database captured from. Defaults to `(default)`. */ + databaseId?: string; + /** + * Firestore database restored into. Must already exist and must not be the + * captured database - a restoration batch-writes over its contents. + */ + backupInstanceId: string; + /** BigQuery dataset holding the changelog table. */ + datasetId: string; + /** BigQuery changelog table. */ + tableId: string; + /** BigQuery dataset location. Defaults to `us`. */ + datasetLocation?: string; + /** Region the functions are deployed to. Defaults to `us-central1`. */ + location?: string; + /** Region Dataflow jobs run in. Defaults to {@link CaptureConfig.location}. */ + dataflowRegion?: string; + /** + * Cloud Storage bucket holding the Dataflow flex template. Defaults to + * `.firebasestorage.app`, the default bucket for projects created + * after September 2024. Projects older than that use `.appspot.com` + * and must set this explicitly. + */ + bucketName?: string; + /** + * Namespaces the deployed resources: the task queues, the flex template + * object, the Dataflow job names and the Firestore status documents. Deploy + * the kit twice under one project by giving each deployment its own value. + * Defaults to `firestore-incremental-capture`. + */ + instanceId?: string; + /** Defaults to `info`. */ + logLevel?: LogLevel; +} + +/** {@link CaptureConfig} with every default applied and paths derived. */ +export interface ResolvedCaptureConfig + extends Required> { + dataflowRegion: string; + bucketName: string; + /** Fully-qualified name of the database restored into. */ + backupInstanceName: string; + /** + * Cloud Storage path of the Dataflow flex template spec. Built out-of-band by + * the setup script, which must write to this exact path. + */ + flexTemplatePath: string; + /** Firestore document tracking the state of each restoration run. */ + restoreCollection: string; +} + +const DEFAULT_DATABASE_ID = "(default)"; +const DEFAULT_INSTANCE_ID = "firestore-incremental-capture"; +const DEFAULT_LOCATION = "us-central1"; +const DEFAULT_DATASET_LOCATION = "us"; + +/** + * Applies defaults and derives the resource paths the handlers need. + * + * @param config - Caller-supplied configuration. + * @returns The fully resolved configuration. + * @throws If the backup database is the same as the captured database, which + * would make a restoration overwrite the source it is restoring from. + */ +export function resolveCaptureConfig( + config: CaptureConfig +): ResolvedCaptureConfig { + const databaseId = config.databaseId || DEFAULT_DATABASE_ID; + const location = config.location || DEFAULT_LOCATION; + const instanceId = config.instanceId || DEFAULT_INSTANCE_ID; + const bucketName = + config.bucketName || `${config.projectId}.firebasestorage.app`; + + if (config.backupInstanceId === databaseId) { + throw new Error( + `Invalid configuration for firestore-incremental-capture: BACKUP_INSTANCE_ID ` + + `("${config.backupInstanceId}") must differ from the captured database ` + + `("${databaseId}"). A restoration batch-writes over the backup database.` + ); + } + + return { + projectId: config.projectId, + syncCollectionPath: config.syncCollectionPath, + databaseId, + backupInstanceId: config.backupInstanceId, + datasetId: config.datasetId, + tableId: config.tableId, + datasetLocation: config.datasetLocation || DEFAULT_DATASET_LOCATION, + location, + dataflowRegion: config.dataflowRegion || location, + bucketName, + instanceId, + logLevel: config.logLevel || "info", + backupInstanceName: `projects/${config.projectId}/databases/${config.backupInstanceId}`, + flexTemplatePath: `gs://${bucketName}/${instanceId}-dataflow-restore`, + restoreCollection: `_${instanceId}/runs/restorations`, + }; +} + +/** + * Collection id the Dataflow pipeline reads from. The pipeline takes `*` to + * mean every collection, where the Firestore trigger spells that `{document=**}`. + * + * @param syncCollectionPath - The configured collection path. + * @returns The collection id in the pipeline's spelling. + */ +export function toPipelineCollectionId(syncCollectionPath: string): string { + return syncCollectionPath === "{document=**}" ? "*" : syncCollectionPath; +} diff --git a/kits/firestore-incremental-capture/src/changelog.ts b/kits/firestore-incremental-capture/src/changelog.ts new file mode 100644 index 000000000..eb5e07558 --- /dev/null +++ b/kits/firestore-incremental-capture/src/changelog.ts @@ -0,0 +1,51 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** How a captured document changed. */ +export type ChangeType = "CREATE" | "UPDATE" | "DELETE"; + +/** + * One row of the BigQuery changelog. + * + * `beforeData` and `afterData` are JSON strings rather than objects because the + * columns are BigQuery `JSON`, and because the Dataflow pipeline parses them + * with Gson. See {@link serializeDocument} for the value encoding. + */ +export interface ChangelogRow { + documentId: string; + documentPath: string; + beforeData: string; + afterData: string; + changeType: ChangeType; + /** RFC 3339 event time, matching the BigQuery `TIMESTAMP` column. */ + timestamp: string; +} + +/** + * Schema of the changelog table. + * + * The Dataflow pipeline queries these columns by name + * (`IncrementalCaptureLog`), so the column names are part of the contract + * between the kit and `pipeline/`. + */ +export const CHANGELOG_SCHEMA = [ + { name: "documentId", type: "STRING", mode: "REQUIRED" }, + { name: "documentPath", type: "STRING", mode: "REQUIRED" }, + { name: "beforeData", type: "JSON" }, + { name: "afterData", type: "JSON" }, + { name: "changeType", type: "STRING", mode: "REQUIRED" }, + { name: "timestamp", type: "TIMESTAMP", mode: "REQUIRED" }, +] as const; diff --git a/kits/firestore-incremental-capture/src/config.ts b/kits/firestore-incremental-capture/src/config.ts new file mode 100644 index 000000000..e069a9a59 --- /dev/null +++ b/kits/firestore-incremental-capture/src/config.ts @@ -0,0 +1,170 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Expression } from "firebase-functions/params"; +import { defineString, projectID, select } from "firebase-functions/params"; +import type { CaptureConfig, LogLevel } from "./capture-config"; + +const DATASET_LOCATION_OPTIONS = [ + "us", + "eu", + "us-central1", + "us-east1", + "us-east4", + "us-west1", + "us-west2", + "us-west3", + "us-west4", + "northamerica-northeast1", + "southamerica-east1", + "europe-central2", + "europe-north1", + "europe-west1", + "europe-west2", + "europe-west3", + "europe-west4", + "europe-west6", + "asia-east1", + "asia-east2", + "asia-northeast1", + "asia-northeast2", + "asia-northeast3", + "asia-south1", + "asia-southeast1", + "asia-southeast2", + "australia-southeast1", +] as const; + +/** + * Regions the functions and Dataflow jobs can run in. Restricted to the regions + * the extension offered, which are the ones Dataflow flex templates support. + */ +const LOCATION_OPTIONS = [ + "us-central1", + "us-east1", + "us-east4", + "us-west2", + "us-west3", + "us-west4", + "europe-central2", + "europe-west1", + "europe-west2", + "europe-west3", + "europe-west6", + "asia-east1", + "asia-east2", + "asia-northeast1", + "asia-northeast2", + "asia-northeast3", + "asia-south1", + "asia-southeast1", + "asia-southeast2", + "northamerica-northeast1", + "southamerica-east1", + "australia-southeast1", +] as const; + +const LOG_LEVEL_OPTIONS = ["debug", "info", "warn", "error", "silent"] as const; + +/** Deploy-time expressions the entry point needs before params can be read. */ +export interface ConfigExpressions { + syncCollectionPath: Expression; + database: Expression; + location: Expression; +} + +/** + * Deploy-time parameters. Set these via a `.env` / `.env.` file or the + * interactive prompts shown by `firebase deploy`. + * + * @see https://firebase.google.com/docs/functions/config-env + */ +const params = { + location: defineString("LOCATION", { + default: "us-central1", + input: select([...LOCATION_OPTIONS]), + }), + database: defineString("DATABASE", { default: "(default)" }), + syncCollectionPath: defineString("SYNC_COLLECTION_PATH", { + default: "posts", + }), + syncDataset: defineString("SYNC_DATASET", { default: "backup_dataset" }), + syncTable: defineString("SYNC_TABLE", { default: "backup_table" }), + backupInstanceId: defineString("BACKUP_INSTANCE_ID"), + datasetLocation: defineString("DATASET_LOCATION", { + default: "us", + input: select([...DATASET_LOCATION_OPTIONS]), + }), + dataflowRegion: defineString("DATAFLOW_REGION", { default: "" }), + bucketName: defineString("BUCKET_NAME", { default: "" }), + instanceId: defineString("INSTANCE_ID", { + default: "firestore-incremental-capture", + }), + logLevel: defineString("LOG_LEVEL", { + default: "info", + input: select([...LOG_LEVEL_OPTIONS]), + }), +}; + +export const CONFIG_EXPRESSIONS: ConfigExpressions = { + syncCollectionPath: params.syncCollectionPath, + database: params.database, + location: params.location, +}; + +/** Coerce an empty-string param value to `undefined`. */ +function optional(value: string): string | undefined { + return value.length > 0 ? value : undefined; +} + +function normalizeLogLevel(level: string): LogLevel { + switch (level.toLowerCase()) { + case "debug": + case "info": + case "warn": + case "error": + case "silent": + return level.toLowerCase() as LogLevel; + default: + return "info"; + } +} + +/** + * Resolves all deploy-time params into a {@link CaptureConfig}. + * + * Param values are read when this is called, not at import, so the Firebase + * deploy-time discovery pass can analyze the entry point without resolving + * params early. + * + * @returns The capture configuration assembled from environment params. + */ +export function configFromEnv(): CaptureConfig { + return { + projectId: projectID.value(), + syncCollectionPath: params.syncCollectionPath.value(), + databaseId: optional(params.database.value()), + backupInstanceId: params.backupInstanceId.value(), + datasetId: params.syncDataset.value(), + tableId: params.syncTable.value(), + datasetLocation: optional(params.datasetLocation.value()), + location: optional(params.location.value()), + dataflowRegion: optional(params.dataflowRegion.value()), + bucketName: optional(params.bucketName.value()), + instanceId: optional(params.instanceId.value()), + logLevel: normalizeLogLevel(params.logLevel.value()), + }; +} diff --git a/kits/firestore-incremental-capture/src/dataflow.ts b/kits/firestore-incremental-capture/src/dataflow.ts new file mode 100644 index 000000000..86fff7172 --- /dev/null +++ b/kits/firestore-incremental-capture/src/dataflow.ts @@ -0,0 +1,91 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { FlexTemplatesServiceClient } from "@google-cloud/dataflow"; +import { getFirestore } from "firebase-admin/firestore"; +import { + type ResolvedCaptureConfig, + toPipelineCollectionId, +} from "./capture-config"; +import type { RestorationJob, RestorationRequest } from "./handlers"; +import * as logs from "./logs"; + +/** + * Launches the Dataflow restoration pipeline. + */ +export class RestorationLauncher { + /** + * @param config - The resolved capture configuration. + * @param client - Dataflow flex templates client, injectable for tests. + * @param now - Clock, injectable so run ids are deterministic in tests. + */ + constructor( + private readonly config: ResolvedCaptureConfig, + private readonly client: FlexTemplatesServiceClient = new FlexTemplatesServiceClient(), + private readonly now: () => number = Date.now + ) {} + + /** + * Launches a restoration job and records the run in Firestore. + * + * The flex template must already be staged at + * {@link ResolvedCaptureConfig.flexTemplatePath}; see the kit's setup script. + * Launching against a missing template fails here rather than at deploy. + * + * @param request - The validated restoration request. + * @returns The launched job. + */ + async launch(request: RestorationRequest): Promise { + const { config } = this; + const runId = `${config.instanceId}-dataflow-run-${this.now()}`; + + logs.info(`Launching restoration job ${runId}`, { + labels: { run_id: runId }, + }); + + const [response] = await this.client.launchFlexTemplate({ + projectId: config.projectId, + location: config.dataflowRegion, + launchParameter: { + jobName: runId, + parameters: { + timestamp: request.timestamp.toString(), + firestoreCollectionId: toPipelineCollectionId( + config.syncCollectionPath + ), + firestoreDb: config.backupInstanceId, + bigQueryDataset: config.datasetId, + bigQueryTable: config.tableId, + }, + containerSpecGcsPath: config.flexTemplatePath, + }, + }); + + const jobName = response.job?.name ?? undefined; + + await getFirestore(config.databaseId) + .collection(config.restoreCollection) + .doc(runId) + .set({ + runId, + jobName: jobName ?? null, + timestamp: request.timestamp, + status: "launched", + }); + + return { runId, jobName }; + } +} diff --git a/kits/firestore-incremental-capture/src/handlers.ts b/kits/firestore-incremental-capture/src/handlers.ts new file mode 100644 index 000000000..f911cbcdd --- /dev/null +++ b/kits/firestore-incremental-capture/src/handlers.ts @@ -0,0 +1,218 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { + Change, + DocumentSnapshot, + FirestoreEvent, +} from "firebase-functions/firestore"; +import type { ResolvedCaptureConfig } from "./capture-config"; +import type { ChangelogRow, ChangeType } from "./changelog"; +import * as logs from "./logs"; +import { serializeDocument } from "./serializer"; + +/** The Firestore document-write event passed to {@link handleDocumentWrite}. */ +export type DocumentWriteEvent = FirestoreEvent< + Change | undefined, + Record +>; + +/** A request to restore the backup database to a point in time. */ +export interface RestorationRequest { + /** Point to restore to, in whole seconds since the Unix epoch. */ + timestamp: number; +} + +/** A launched Dataflow restoration job. */ +export interface RestorationJob { + /** Identifies the run in logs and in the Firestore status document. */ + runId: string; + /** Dataflow's name for the job, absent if it did not report one. */ + jobName?: string; +} + +/** + * Everything the handlers need to do their work, injected by the caller so the + * handlers stay free of global state and remain testable without emulators. + */ +export interface HandlerContext { + config: ResolvedCaptureConfig; + /** Enqueues a changelog row for asynchronous insertion into BigQuery. */ + enqueueChangelogRow(row: ChangelogRow): Promise; + /** Inserts changelog rows into the BigQuery changelog table. */ + insertChangelogRows(rows: ChangelogRow[]): Promise; + /** Enqueues a restoration, to be run outside the request's lifetime. */ + enqueueRestoration(request: RestorationRequest): Promise; + /** Launches the Dataflow restoration job. */ + launchRestorationJob(request: RestorationRequest): Promise; +} + +/** + * Classifies a document write. + * + * @param change - The before/after snapshots from the trigger. + * @returns The change type. + */ +export function getChangeType(change: Change): ChangeType { + if (!change.before?.exists) return "CREATE"; + if (!change.after?.exists) return "DELETE"; + return "UPDATE"; +} + +/** + * Checks that a value is a point in time the pipeline can restore to: a whole + * number of seconds since the Unix epoch, not in the future. + * + * Restoration reads a Firestore PITR snapshot, so a timestamp older than the + * PITR window is clamped by the pipeline rather than rejected here - the window + * is a property of the database, not of the request. + * + * @param timestamp - The candidate timestamp. + * @returns Whether the timestamp can be restored to. + */ +export function isValidRestorationTimestamp( + timestamp: unknown +): timestamp is number { + if (typeof timestamp !== "number" || !Number.isInteger(timestamp)) { + return false; + } + + if (timestamp <= 0) return false; + + return timestamp <= Math.floor(Date.now() / 1000); +} + +/** + * Captures a Firestore document write onto the changelog queue. + * + * The write is serialized here and inserted into BigQuery by + * {@link handleChangelogTask}, so that a BigQuery outage retries on the task + * queue's schedule rather than holding the Firestore trigger open. + * + * @param event - The Firestore document-write event. + * @param ctx - The handler context. + */ +export async function handleDocumentWrite( + event: DocumentWriteEvent, + ctx: HandlerContext +): Promise { + const change = event.data; + if (!change) return; + + const changeType = getChangeType(change); + const documentId = change.after?.id ?? change.before.id; + const documentPath = change.after?.ref?.path ?? change.before.ref.path; + + logs.debug("Capturing Firestore write", { + documentPath, + changeType, + }); + + const row: ChangelogRow = { + documentId, + documentPath, + beforeData: JSON.stringify(serializeDocument(change.before?.data())), + afterData: JSON.stringify(serializeDocument(change.after?.data())), + changeType, + timestamp: event.time, + }; + + await ctx.enqueueChangelogRow(row); +} + +/** + * Inserts one queued changelog row into BigQuery. + * + * @param row - The row enqueued by {@link handleDocumentWrite}. + * @param ctx - The handler context. + * @throws If the insert fails, so the task queue retries it. A dropped row is a + * permanent hole in the changelog, and therefore in any restoration that + * replays across it. + */ +export async function handleChangelogTask( + row: ChangelogRow, + ctx: HandlerContext +): Promise { + await ctx.insertChangelogRows([row]); + + logs.debug("Wrote changelog row", { documentPath: row.documentPath }); +} + +/** Outcome of {@link handleRestorationRequest}, for the caller to send. */ +export interface RestorationResponse { + status: number; + body: string; +} + +/** + * Validates an inbound restoration request and enqueues the work. + * + * @param body - The parsed request body. + * @param ctx - The handler context. + * @returns The status and body to respond with. + */ +export async function handleRestorationRequest( + body: unknown, + ctx: HandlerContext +): Promise { + const timestamp = (body as { timestamp?: unknown } | null)?.timestamp; + + if (!isValidRestorationTimestamp(timestamp)) { + logs.error( + "Rejected restoration request: 'timestamp' must be a whole number of " + + "seconds since the Unix epoch, and cannot be in the future.", + { timestamp } + ); + return { + status: 400, + body: "'timestamp' must be a past Unix timestamp in seconds.", + }; + } + + await ctx.enqueueRestoration({ timestamp }); + + logs.info("Enqueued restoration", { timestamp }); + + return { status: 200, body: "Restoration task enqueued" }; +} + +/** + * Runs a queued restoration by launching the Dataflow job. + * + * @param request - The restoration request enqueued by + * {@link handleRestorationRequest}. + * @param ctx - The handler context. + * @returns The launched job, or `undefined` if the request was not restorable. + */ +export async function handleRestorationTask( + request: RestorationRequest, + ctx: HandlerContext +): Promise { + if (!isValidRestorationTimestamp(request?.timestamp)) { + logs.error("Discarding restoration task with an invalid timestamp", { + timestamp: request?.timestamp, + }); + return undefined; + } + + logs.info("Restoring to point in time", { timestamp: request.timestamp }); + + const job = await ctx.launchRestorationJob(request); + + logs.info("Launched restoration job", job); + + return job; +} diff --git a/kits/firestore-incremental-capture/src/index.ts b/kits/firestore-incremental-capture/src/index.ts index 4851c8758..0919d76ac 100644 --- a/kits/firestore-incremental-capture/src/index.ts +++ b/kits/firestore-incremental-capture/src/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2019 Google LLC + * Copyright 2026 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,200 @@ */ /** - * firestore-incremental-capture — npm-shared Firebase Function migrated from the Firebase Extension of - * the same name. + * Main entry point. Exports the wired functions with deploy-time param + * expressions, then resolves concrete config lazily at runtime. Re-export these + * from your own functions codebase entry; configuration comes from a `.env` + * (or `.env.`), which the Firebase CLI loads at deploy. * - * Skeleton package: not yet implemented. Target shape follows the - * firestore-bigquery-export reference package — a `define...` factory (tier 3) - * over injectable handlers (tier 2), with a side-effect-free `./lib` surface and - * this env-driven entry registering functions for the clone-and-deploy example. + * Because this module initializes runtime dependencies lazily, deploy discovery + * can analyze it without resolving params too early. + * + * For side-effect-free imports (the handlers, config and wire-format types), + * import from `./lib` instead. + * + * Restoration additionally depends on out-of-band setup - a PITR-enabled source + * database, an existing backup database, and a staged Dataflow flex template. + * See `scripts/setup.sh` and the README. + */ + +import { getApps, initializeApp } from "firebase-admin/app"; +import { onDocumentWritten } from "firebase-functions/firestore"; +import { onRequest } from "firebase-functions/https"; +import { expr } from "firebase-functions/params"; +import { onTaskDispatched } from "firebase-functions/tasks"; +// Imported from the narrow subpath, not the `firebase-functions/v2` barrel: the +// barrel pulls in the RTDB provider, whose firebase-admin dependency fails to +// load without @firebase/app installed. +import type { Role } from "firebase-functions/v2/options"; +import { requiresRole } from "firebase-functions/v2/options"; +import { + afterFirstDeploy, + afterRedeploy, +} from "firebase-functions/v2/lifecycle"; +import { ChangelogTable } from "./bigquery"; +import { resolveCaptureConfig } from "./capture-config"; +import type { ChangelogRow } from "./changelog"; +import { CONFIG_EXPRESSIONS, configFromEnv } from "./config"; +import { RestorationLauncher } from "./dataflow"; +import { + handleChangelogTask, + handleDocumentWrite, + handleRestorationRequest, + handleRestorationTask, + type HandlerContext, + type RestorationRequest, +} from "./handlers"; +import * as logs from "./logs"; +import { + CHANGELOG_TASK_FUNCTION, + enqueue, + RESTORATION_TASK_FUNCTION, +} from "./tasks"; + +// Re-export the side-effect-free library surface. +export * from "./lib"; + +const INIT_FUNCTION = "initIncrementalCapture"; +const LIFECYCLE_RETRY_CONFIG = { + maxAttempts: 15, + minBackoffSeconds: 60, +} as const; +const REQUIRED_ROLES: ReadonlyArray = [ + "roles/bigquery.dataEditor", + "roles/bigquery.user", + "roles/datastore.user", + "roles/dataflow.developer", +]; + +for (const role of REQUIRED_ROLES) { + requiresRole(role); +} + +afterFirstDeploy({ task: { function: INIT_FUNCTION } }); +afterRedeploy({ task: { function: INIT_FUNCTION } }); + +let ctx: HandlerContext | undefined; + +function getHandlerContext(): HandlerContext { + if (ctx) { + return ctx; + } + + const config = resolveCaptureConfig(configFromEnv()); + + logs.setLogLevel(config.logLevel); + + if (getApps().length === 0) { + initializeApp(); + } + + const changelog = new ChangelogTable(config); + const launcher = new RestorationLauncher(config); + + ctx = { + config, + enqueueChangelogRow: (row) => + enqueue(config, CHANGELOG_TASK_FUNCTION, row as unknown as object), + insertChangelogRows: (rows) => changelog.insert(rows), + enqueueRestoration: (request) => + enqueue(config, RESTORATION_TASK_FUNCTION, request), + launchRestorationJob: (request) => launcher.launch(request), + }; + + return ctx; +} + +const functionOptions = { + region: CONFIG_EXPRESSIONS.location, +}; + +/** + * Firestore trigger: serializes each document write on the watched collection + * and queues it for insertion into the BigQuery changelog. Failed executions + * are retried by the Firebase Functions runtime, because a dropped write is a + * permanent hole in the changelog. + */ +export const syncData = onDocumentWritten( + { + ...functionOptions, + document: expr`${CONFIG_EXPRESSIONS.syncCollectionPath}/{documentId}`, + database: CONFIG_EXPRESSIONS.database, + retry: true, + }, + (event) => handleDocumentWrite(event, getHandlerContext()) +); + +/** + * Inserts a queued changelog row into BigQuery. Separated from the trigger so a + * BigQuery outage retries on the queue's schedule. + */ +export const syncChangelogTask = onTaskDispatched( + { + ...functionOptions, + retryConfig: { maxAttempts: 15, minBackoffSeconds: 10 }, + }, + (request) => handleChangelogTask(request.data, getHandlerContext()) +); + +/** + * Starts a restoration of the backup database to a point in time. + * + * SECURITY: this endpoint is intentionally unauthenticated, matching the + * firestore-incremental-capture extension it was migrated from. Anyone who can + * reach the URL can trigger a Dataflow job that batch-writes over the backup + * database. Restrict it before deploying to production - with Cloud Run ingress + * settings, an IAM invoker policy, or by fronting it with your own authorized + * endpoint that enqueues `runRestorationTask` directly. + */ +export const onHttpRunRestoration = onRequest( + functionOptions, + async (request, response) => { + const result = await handleRestorationRequest( + request.body, + getHandlerContext() + ); + response.status(result.status).send(result.body); + } +); + +/** + * Runs a queued restoration by launching the Dataflow pipeline. The job itself + * runs asynchronously in Dataflow; this only starts it. + */ +export const runRestorationTask = onTaskDispatched( + { + ...functionOptions, + memory: "1GiB", + }, + async (request) => { + await handleRestorationTask(request.data, getHandlerContext()); + } +); + +/** + * Provisioning lifecycle task. Creates the BigQuery dataset and changelog table + * if they are missing, running in the function's own identity so it has the + * runtime service account the creation needs. Enqueued after first deploy and + * after each redeploy; idempotent, and retried by Cloud Tasks on a transient + * BigQuery error so a blip does not leave the changelog unprovisioned. + * + * It does not provision the restoration prerequisites - PITR, the backup + * database and the Dataflow flex template all need gcloud, which is not + * available in the functions runtime. Run `scripts/setup.sh` for those. */ +export const initIncrementalCapture = onTaskDispatched( + { + ...functionOptions, + retryConfig: LIFECYCLE_RETRY_CONFIG, + }, + async () => { + const { config } = getHandlerContext(); -export {}; + try { + await new ChangelogTable(config).initialize(); + } catch (err) { + logs.error("Failed to initialize BigQuery changelog resources", err); + throw err; + } + } +); diff --git a/kits/firestore-incremental-capture/src/lib.ts b/kits/firestore-incremental-capture/src/lib.ts new file mode 100644 index 000000000..71197026b --- /dev/null +++ b/kits/firestore-incremental-capture/src/lib.ts @@ -0,0 +1,65 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Side-effect-free library surface: + * + * - The handlers, for consumers who want to own trigger registration + * themselves. Each takes an injected {@link HandlerContext}. + * - Config types and helpers for building that context. + * - The changelog schema and the document serializer, for consumers reading the + * changelog or reimplementing the restoration side. + * + * Importing this module has no side effects (it reads no environment and opens + * no clients), so it is safe to import anywhere. The main entry point + * (`./index`) is the one that reads env params and exports wired functions. + */ + +// Config +export { + type CaptureConfig, + type LogLevel, + type ResolvedCaptureConfig, + resolveCaptureConfig, + toPipelineCollectionId, +} from "./capture-config"; +// Changelog wire format +export { + CHANGELOG_SCHEMA, + type ChangelogRow, + type ChangeType, +} from "./changelog"; +// Handlers +export { + type DocumentWriteEvent, + getChangeType, + handleChangelogTask, + handleDocumentWrite, + handleRestorationRequest, + handleRestorationTask, + type HandlerContext, + isValidRestorationTimestamp, + type RestorationJob, + type RestorationRequest, + type RestorationResponse, +} from "./handlers"; +// Serialization +export { + type SerializedDocument, + type SerializedType, + type SerializedValue, + serializeDocument, +} from "./serializer"; diff --git a/kits/firestore-incremental-capture/src/logs.ts b/kits/firestore-incremental-capture/src/logs.ts new file mode 100644 index 000000000..04093a4b3 --- /dev/null +++ b/kits/firestore-incremental-capture/src/logs.ts @@ -0,0 +1,59 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Imported from the narrow subpath, not the `firebase-functions` barrel, which +// pulls in the RTDB provider. See the note in `./index`. +import * as logger from "firebase-functions/logger"; +import type { LogLevel } from "./capture-config"; + +const SEVERITY: Record = { + debug: 10, + info: 20, + warn: 30, + error: 40, + silent: 100, +}; + +let threshold = SEVERITY.info; + +/** + * Sets the minimum severity that will be emitted. + * + * @param level - The configured log level. + */ +export function setLogLevel(level: LogLevel): void { + threshold = SEVERITY[level] ?? SEVERITY.info; +} + +/** Logs at debug severity. */ +export function debug(message: string, data?: unknown): void { + if (threshold <= SEVERITY.debug) logger.debug(message, data); +} + +/** Logs at info severity. */ +export function info(message: string, data?: unknown): void { + if (threshold <= SEVERITY.info) logger.info(message, data); +} + +/** Logs at warn severity. */ +export function warn(message: string, data?: unknown): void { + if (threshold <= SEVERITY.warn) logger.warn(message, data); +} + +/** Logs at error severity. */ +export function error(message: string, err?: unknown): void { + if (threshold <= SEVERITY.error) logger.error(message, err); +} diff --git a/kits/firestore-incremental-capture/src/serializer.ts b/kits/firestore-incremental-capture/src/serializer.ts new file mode 100644 index 000000000..129c682b5 --- /dev/null +++ b/kits/firestore-incremental-capture/src/serializer.ts @@ -0,0 +1,126 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + DocumentReference, + GeoPoint, + Timestamp, +} from "firebase-admin/firestore"; + +/** + * Tag identifying how a serialized value should be reconstructed. Firestore's + * own types are named explicitly; everything else carries its `typeof` tag. + */ +export type SerializedType = + | "array" + | "binary" + | "geopoint" + | "map" + | "null" + | "reference" + | "timestamp" + | "bigint" + | "boolean" + | "number" + | "string"; + +/** A single tagged value in a serialized document. */ +export interface SerializedValue { + type: SerializedType; + value: unknown; +} + +/** A serialized Firestore document: field name to tagged value. */ +export type SerializedDocument = Record; + +/** + * Serializes Firestore document data into a self-describing tree. + * + * Every value carries the tag needed to rebuild it, because the changelog round + * trips through BigQuery JSON columns, which cannot represent a Timestamp, + * GeoPoint, DocumentReference or Buffer. + * + * The tags are a wire format shared with the Dataflow restoration pipeline: + * `FirestoreReconstructor.buildFirestoreMap` upper-cases each tag and switches + * on it, dropping any field whose tag it does not recognise. Renaming a tag on + * one side silently discards data on restore. `reference` is spelled to match + * the pipeline's `REFERENCE` case, and carries the relative document path + * because the pipeline prefixes `projects/…/databases/…/documents/` itself. + * + * The pipeline has no case for `binary` or `null`, so those fields are dropped + * on restore. See the kit README for the full list of restoration gaps. + * + * @param data - Firestore document data, or `undefined` for a document that + * does not exist on this side of the change. + * @returns The serialized document; empty for `undefined`/`null` input. + */ +export function serializeDocument(data: unknown): SerializedDocument { + if (data === null || data === undefined || typeof data !== "object") { + return {}; + } + + const serialized: SerializedDocument = {}; + + for (const [key, value] of Object.entries(data as Record)) { + serialized[key] = serializeValue(value); + } + + return serialized; +} + +/** + * Serializes a single value, recursing through maps and arrays. + * + * @param value - The value to serialize. + * @returns The tagged value. + */ +function serializeValue(value: unknown): SerializedValue { + if (value === null || value === undefined) { + return { type: "null", value: null }; + } + + if (Buffer.isBuffer(value)) { + return { type: "binary", value: value.toString("base64") }; + } + + if (value instanceof Timestamp) { + return { type: "timestamp", value: value.toDate().toISOString() }; + } + + if (value instanceof GeoPoint) { + return { + type: "geopoint", + value: { + latitude: { type: "number", value: value.latitude }, + longitude: { type: "number", value: value.longitude }, + }, + }; + } + + if (value instanceof DocumentReference) { + return { type: "reference", value: value.path }; + } + + if (Array.isArray(value)) { + return { type: "array", value: value.map(serializeValue) }; + } + + if (typeof value === "object") { + return { type: "map", value: serializeDocument(value) }; + } + + return { type: typeof value as SerializedType, value }; +} diff --git a/kits/firestore-incremental-capture/src/tasks.ts b/kits/firestore-incremental-capture/src/tasks.ts new file mode 100644 index 000000000..806e1c1d2 --- /dev/null +++ b/kits/firestore-incremental-capture/src/tasks.ts @@ -0,0 +1,53 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getFunctions } from "firebase-admin/functions"; +import type { ResolvedCaptureConfig } from "./capture-config"; + +/** Names of the deployed task-queue functions. */ +export const CHANGELOG_TASK_FUNCTION = "syncChangelogTask"; +export const RESTORATION_TASK_FUNCTION = "runRestorationTask"; + +/** + * Builds the fully-qualified task queue name for a deployed function. + * + * @param config - The resolved capture configuration. + * @param functionName - The deployed function's name. + * @returns The queue resource name. + */ +export function queueName( + config: ResolvedCaptureConfig, + functionName: string +): string { + return `locations/${config.location}/functions/${functionName}`; +} + +/** + * Enqueues a payload onto a deployed function's task queue. + * + * @param config - The resolved capture configuration. + * @param functionName - The deployed function's name. + * @param payload - The task payload. + */ +export async function enqueue( + config: ResolvedCaptureConfig, + functionName: string, + payload: object +): Promise { + await getFunctions() + .taskQueue(queueName(config, functionName)) + .enqueue(payload); +} diff --git a/kits/firestore-incremental-capture/tests/capture-config.test.ts b/kits/firestore-incremental-capture/tests/capture-config.test.ts new file mode 100644 index 000000000..98345ef36 --- /dev/null +++ b/kits/firestore-incremental-capture/tests/capture-config.test.ts @@ -0,0 +1,107 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, expect, test } from "vitest"; +import { + type CaptureConfig, + resolveCaptureConfig, + toPipelineCollectionId, +} from "../src/capture-config"; + +function config(overrides: Partial = {}): CaptureConfig { + return { + projectId: "test-project", + syncCollectionPath: "users", + backupInstanceId: "backup-db", + datasetId: "backup_dataset", + tableId: "backup_table", + ...overrides, + }; +} + +describe("resolveCaptureConfig", () => { + test("applies defaults", () => { + const resolved = resolveCaptureConfig(config()); + + expect(resolved.databaseId).toBe("(default)"); + expect(resolved.location).toBe("us-central1"); + expect(resolved.datasetLocation).toBe("us"); + expect(resolved.instanceId).toBe("firestore-incremental-capture"); + expect(resolved.logLevel).toBe("info"); + }); + + test("defaults the Dataflow region to the functions location", () => { + expect( + resolveCaptureConfig(config({ location: "europe-west1" })).dataflowRegion + ).toBe("europe-west1"); + }); + + test("keeps an explicit Dataflow region", () => { + const resolved = resolveCaptureConfig( + config({ location: "europe-west1", dataflowRegion: "us-central1" }) + ); + + expect(resolved.dataflowRegion).toBe("us-central1"); + }); + + test("defaults the bucket to the post-2024 default bucket name", () => { + expect(resolveCaptureConfig(config()).bucketName).toBe( + "test-project.firebasestorage.app" + ); + }); + + test("derives the backup instance name and flex template path", () => { + const resolved = resolveCaptureConfig(config()); + + expect(resolved.backupInstanceName).toBe( + "projects/test-project/databases/backup-db" + ); + expect(resolved.flexTemplatePath).toBe( + "gs://test-project.firebasestorage.app/firestore-incremental-capture-dataflow-restore" + ); + }); + + test("namespaces derived paths by instance id", () => { + const resolved = resolveCaptureConfig(config({ instanceId: "second" })); + + expect(resolved.flexTemplatePath).toBe( + "gs://test-project.firebasestorage.app/second-dataflow-restore" + ); + expect(resolved.restoreCollection).toBe("_second/runs/restorations"); + }); + + test("rejects a backup database that is the captured database", () => { + expect(() => + resolveCaptureConfig(config({ backupInstanceId: "(default)" })) + ).toThrow(/must differ from the captured database/); + + expect(() => + resolveCaptureConfig( + config({ databaseId: "primary", backupInstanceId: "primary" }) + ) + ).toThrow(/must differ from the captured database/); + }); +}); + +describe("toPipelineCollectionId", () => { + test("maps the capture-everything wildcard to the pipeline's spelling", () => { + expect(toPipelineCollectionId("{document=**}")).toBe("*"); + }); + + test("passes a concrete collection through", () => { + expect(toPipelineCollectionId("users")).toBe("users"); + }); +}); diff --git a/kits/firestore-incremental-capture/tests/handlers.test.ts b/kits/firestore-incremental-capture/tests/handlers.test.ts new file mode 100644 index 000000000..d8fe4dcfd --- /dev/null +++ b/kits/firestore-incremental-capture/tests/handlers.test.ts @@ -0,0 +1,302 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { resolveCaptureConfig } from "../src/capture-config"; +import { + type DocumentWriteEvent, + getChangeType, + handleChangelogTask, + handleDocumentWrite, + handleRestorationRequest, + handleRestorationTask, + type HandlerContext, + isValidRestorationTimestamp, +} from "../src/handlers"; + +// Stubbed with a factory rather than automocked: automocking loads the real +// module, which pulls firebase-functions and firebase-admin into the test. +vi.mock("../src/logs", () => ({ + setLogLevel: vi.fn(), + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +})); + +/** Fake Firestore snapshot with only the fields the handlers read. */ +function snap(exists: boolean, id: string, data: unknown = {}) { + return { exists, id, data: () => data, ref: { path: `users/${id}` } }; +} + +function writeEvent( + before: ReturnType, + after: ReturnType +): DocumentWriteEvent { + return { + data: { before, after }, + id: "evt-1", + time: "2026-01-01T00:00:00Z", + document: "users/doc1", + params: { documentId: "doc1" }, + } as unknown as DocumentWriteEvent; +} + +function makeCtx(): HandlerContext { + return { + config: resolveCaptureConfig({ + projectId: "test-project", + syncCollectionPath: "users", + backupInstanceId: "backup-db", + datasetId: "ds", + tableId: "tbl", + }), + enqueueChangelogRow: vi.fn().mockResolvedValue(undefined), + insertChangelogRows: vi.fn().mockResolvedValue(undefined), + enqueueRestoration: vi.fn().mockResolvedValue(undefined), + launchRestorationJob: vi + .fn() + .mockResolvedValue({ runId: "run-1", jobName: "job-1" }), + }; +} + +describe("getChangeType", () => { + test("classifies a create", () => { + expect( + getChangeType({ + before: snap(false, "d"), + after: snap(true, "d"), + } as never) + ).toBe("CREATE"); + }); + + test("classifies an update", () => { + expect( + getChangeType({ + before: snap(true, "d"), + after: snap(true, "d"), + } as never) + ).toBe("UPDATE"); + }); + + test("classifies a delete", () => { + expect( + getChangeType({ + before: snap(true, "d"), + after: snap(false, "d"), + } as never) + ).toBe("DELETE"); + }); +}); + +describe("handleDocumentWrite", () => { + test("enqueues a serialized changelog row", async () => { + const ctx = makeCtx(); + + await handleDocumentWrite( + writeEvent(snap(true, "doc1", { n: 1 }), snap(true, "doc1", { n: 2 })), + ctx + ); + + expect(ctx.enqueueChangelogRow).toHaveBeenCalledTimes(1); + expect(ctx.enqueueChangelogRow).toHaveBeenCalledWith({ + documentId: "doc1", + documentPath: "users/doc1", + beforeData: JSON.stringify({ n: { type: "number", value: 1 } }), + afterData: JSON.stringify({ n: { type: "number", value: 2 } }), + changeType: "UPDATE", + timestamp: "2026-01-01T00:00:00Z", + }); + }); + + test("records a delete with empty after data", async () => { + const ctx = makeCtx(); + + await handleDocumentWrite( + writeEvent(snap(true, "doc1", { n: 1 }), snap(false, "doc1", undefined)), + ctx + ); + + const row = vi.mocked(ctx.enqueueChangelogRow).mock.calls[0][0]; + expect(row.changeType).toBe("DELETE"); + expect(row.afterData).toBe("{}"); + }); + + test("ignores an event with no change payload", async () => { + const ctx = makeCtx(); + + await handleDocumentWrite({ data: undefined } as DocumentWriteEvent, ctx); + + expect(ctx.enqueueChangelogRow).not.toHaveBeenCalled(); + }); +}); + +describe("handleChangelogTask", () => { + test("inserts the queued row", async () => { + const ctx = makeCtx(); + const row = { + documentId: "doc1", + documentPath: "users/doc1", + beforeData: "{}", + afterData: "{}", + changeType: "CREATE" as const, + timestamp: "2026-01-01T00:00:00Z", + }; + + await handleChangelogTask(row, ctx); + + expect(ctx.insertChangelogRows).toHaveBeenCalledWith([row]); + }); + + test("propagates an insert failure so the queue retries", async () => { + const ctx = makeCtx(); + vi.mocked(ctx.insertChangelogRows).mockRejectedValue(new Error("boom")); + + await expect( + handleChangelogTask( + { + documentId: "doc1", + documentPath: "users/doc1", + beforeData: "{}", + afterData: "{}", + changeType: "CREATE", + timestamp: "2026-01-01T00:00:00Z", + }, + ctx + ) + ).rejects.toThrow("boom"); + }); +}); + +describe("isValidRestorationTimestamp", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-01-01T00:00:00Z")); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + test("accepts a past whole-second timestamp", () => { + expect(isValidRestorationTimestamp(1700000000)).toBe(true); + }); + + test("accepts the current second", () => { + expect(isValidRestorationTimestamp(Math.floor(Date.now() / 1000))).toBe( + true + ); + }); + + test("rejects a future timestamp", () => { + expect(isValidRestorationTimestamp(Math.floor(Date.now() / 1000) + 1)).toBe( + false + ); + }); + + test("rejects a millisecond timestamp, which reads as the far future", () => { + expect(isValidRestorationTimestamp(Date.now())).toBe(false); + }); + + test("rejects non-integers, zero and negatives", () => { + expect(isValidRestorationTimestamp(1.5)).toBe(false); + expect(isValidRestorationTimestamp(0)).toBe(false); + expect(isValidRestorationTimestamp(-1)).toBe(false); + }); + + test("rejects non-numbers", () => { + expect(isValidRestorationTimestamp("1700000000")).toBe(false); + expect(isValidRestorationTimestamp(undefined)).toBe(false); + expect(isValidRestorationTimestamp(null)).toBe(false); + }); +}); + +describe("handleRestorationRequest", () => { + test("enqueues a valid request", async () => { + const ctx = makeCtx(); + + const result = await handleRestorationRequest( + { timestamp: 1700000000 }, + ctx + ); + + expect(result).toEqual({ status: 200, body: "Restoration task enqueued" }); + expect(ctx.enqueueRestoration).toHaveBeenCalledWith({ + timestamp: 1700000000, + }); + }); + + test("rejects a missing timestamp with 400", async () => { + const ctx = makeCtx(); + + const result = await handleRestorationRequest({}, ctx); + + expect(result.status).toBe(400); + expect(ctx.enqueueRestoration).not.toHaveBeenCalled(); + }); + + test("rejects a future timestamp with 400", async () => { + const ctx = makeCtx(); + + const result = await handleRestorationRequest( + { timestamp: Math.floor(Date.now() / 1000) + 3600 }, + ctx + ); + + expect(result.status).toBe(400); + expect(ctx.enqueueRestoration).not.toHaveBeenCalled(); + }); + + test("rejects a millisecond timestamp with 400", async () => { + const ctx = makeCtx(); + + const result = await handleRestorationRequest( + { timestamp: Date.now() }, + ctx + ); + + expect(result.status).toBe(400); + expect(ctx.enqueueRestoration).not.toHaveBeenCalled(); + }); + + test("tolerates a null body", async () => { + const ctx = makeCtx(); + + expect((await handleRestorationRequest(null, ctx)).status).toBe(400); + }); +}); + +describe("handleRestorationTask", () => { + test("launches the job for a valid request", async () => { + const ctx = makeCtx(); + + const job = await handleRestorationTask({ timestamp: 1700000000 }, ctx); + + expect(job).toEqual({ runId: "run-1", jobName: "job-1" }); + expect(ctx.launchRestorationJob).toHaveBeenCalledWith({ + timestamp: 1700000000, + }); + }); + + test("discards a task whose timestamp is invalid", async () => { + const ctx = makeCtx(); + + const job = await handleRestorationTask({ timestamp: -1 }, ctx); + + expect(job).toBeUndefined(); + expect(ctx.launchRestorationJob).not.toHaveBeenCalled(); + }); +}); diff --git a/kits/firestore-incremental-capture/tests/serializer.test.ts b/kits/firestore-incremental-capture/tests/serializer.test.ts new file mode 100644 index 000000000..69aadc476 --- /dev/null +++ b/kits/firestore-incremental-capture/tests/serializer.test.ts @@ -0,0 +1,96 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GeoPoint, Timestamp } from "firebase-admin/firestore"; +import { describe, expect, test } from "vitest"; +import { serializeDocument } from "../src/serializer"; + +describe("serializeDocument", () => { + test("returns an empty document for absent data", () => { + expect(serializeDocument(undefined)).toEqual({}); + expect(serializeDocument(null)).toEqual({}); + }); + + test("tags primitives with their typeof", () => { + expect(serializeDocument({ a: "x", b: 1, c: true })).toEqual({ + a: { type: "string", value: "x" }, + b: { type: "number", value: 1 }, + c: { type: "boolean", value: true }, + }); + }); + + test("tags null fields rather than omitting them", () => { + expect(serializeDocument({ a: null })).toEqual({ + a: { type: "null", value: null }, + }); + }); + + test("converts a Timestamp to an ISO string", () => { + const date = new Date("2026-01-02T03:04:05.000Z"); + + expect(serializeDocument({ at: Timestamp.fromDate(date) })).toEqual({ + at: { type: "timestamp", value: "2026-01-02T03:04:05.000Z" }, + }); + }); + + test("nests a GeoPoint as tagged coordinates", () => { + expect(serializeDocument({ where: new GeoPoint(1.5, -2.5) })).toEqual({ + where: { + type: "geopoint", + value: { + latitude: { type: "number", value: 1.5 }, + longitude: { type: "number", value: -2.5 }, + }, + }, + }); + }); + + test("base64-encodes a Buffer", () => { + expect(serializeDocument({ blob: Buffer.from("hi") })).toEqual({ + blob: { type: "binary", value: "aGk=" }, + }); + }); + + test("recurses into maps", () => { + expect(serializeDocument({ outer: { inner: 1 } })).toEqual({ + outer: { + type: "map", + value: { inner: { type: "number", value: 1 } }, + }, + }); + }); + + test("recurses into arrays", () => { + expect(serializeDocument({ list: [1, "two"] })).toEqual({ + list: { + type: "array", + value: [ + { type: "number", value: 1 }, + { type: "string", value: "two" }, + ], + }, + }); + }); + + test("survives a JSON round trip", () => { + const serialized = serializeDocument({ + at: Timestamp.fromDate(new Date("2026-01-02T03:04:05.000Z")), + nested: { list: [1, 2] }, + }); + + expect(JSON.parse(JSON.stringify(serialized))).toEqual(serialized); + }); +}); diff --git a/kits/firestore-incremental-capture/tsconfig.json b/kits/firestore-incremental-capture/tsconfig.json index 5ebac374b..6c0af7feb 100644 --- a/kits/firestore-incremental-capture/tsconfig.json +++ b/kits/firestore-incremental-capture/tsconfig.json @@ -1,8 +1,17 @@ { - "extends": "../../tsconfig.base.json", + "extends": "../../tsconfig.json", "compilerOptions": { "outDir": "lib", "rootDir": "src", + "target": "ES2022", + "module": "Node16", + "moduleResolution": "Node16", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "strict": false, + "esModuleInterop": true, + "skipLibCheck": true, "types": ["node"] }, "include": ["src"], diff --git a/kits/firestore-incremental-capture/vitest.config.ts b/kits/firestore-incremental-capture/vitest.config.ts new file mode 100644 index 000000000..52314463a --- /dev/null +++ b/kits/firestore-incremental-capture/vitest.config.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + // Scoped to the kit's own tests: `pipeline/` is Java, and `legacy/` is the + // reference copy of the extension this kit was migrated from. + include: ["tests/**/*.test.ts"], + }, +}); From 179ffe5093b592e03a4f81f0069f92d739ae64fc Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Tue, 11 Aug 2026 14:53:21 +0100 Subject: [PATCH 04/10] fix(kits): correct incremental capture wire format and IAM Findings from an audit of the migration. The array encoding was a regression introduced by the rewrite, not an inherited gap. FirestoreReconstructor.buildFirestoreList passes each element straight to buildFirestoreMap, which reads field names at the top level, so a map element must be a bare field map. The extension emitted exactly that; the hand-rolled recursion wrapped elements in a {type:"map"} envelope, which restores as an empty map. Arrays of maps were the one array shape that worked. tests/wire-format.test.ts pins the format against the extension's own serializer tests, which are the authoritative record and are about to be deleted with legacy/. It fails on the pre-fix encoding. The IAM setup targeted an identity the functions do not run as. requiresRole makes the CLI provision a managed runtime service account, so setup.sh could not grant to it - the account does not exist until first deploy. iam.serviceAccountUser, needed to act as the Dataflow worker when launching a flex template, was granted only by the script and so reached nothing; every restoration would have failed with PERMISSION_DENIED. It moves to requiresRole, and the script now grants the separate roles the Dataflow worker itself needs. Also: - Drop the DATABASE param. RestorationPipeline reads its PITR baseline from the default database, so a non-default source was captured to the changelog but silently absent from the restored baseline. - Require BUCKET_NAME rather than guessing .firebasestorage.app. The entry point reads the project's real default bucket, so it agrees with the bucket setup.sh stages to on pre-2024 projects. - Derive the restoration run id from the target timestamp. A retry after a failed post-launch status write no longer starts a second Dataflow job writing over the backup database concurrently. - Reject an empty BACKUP_INSTANCE_ID. - Construct the Dataflow client on first use, off the capture path. - Enable the compute and storage APIs; workers are Compute Engine VMs. - README: document the single-collection and default-database limits, the required IAM, and two further restoration gaps (id-not-path collision in the replay query, and arrays of maps surviving where primitives do not). Add .env.example. --- .../.env.example | 32 +++ kits/firestore-incremental-capture/README.md | 70 ++++-- .../scripts/setup.sh | 76 ++++--- .../src/capture-config.ts | 64 ++++-- .../src/config.ts | 11 +- .../src/dataflow.ts | 12 +- .../src/index.ts | 34 ++- .../src/serializer.ts | 37 +++- .../tests/capture-config.test.ts | 21 +- .../tests/dataflow.test.ts | 199 +++++++++++++++++ .../tests/handlers.test.ts | 1 + .../tests/wire-format.test.ts | 206 ++++++++++++++++++ 12 files changed, 671 insertions(+), 92 deletions(-) create mode 100644 kits/firestore-incremental-capture/.env.example create mode 100644 kits/firestore-incremental-capture/tests/dataflow.test.ts create mode 100644 kits/firestore-incremental-capture/tests/wire-format.test.ts diff --git a/kits/firestore-incremental-capture/.env.example b/kits/firestore-incremental-capture/.env.example new file mode 100644 index 000000000..db37b4b6c --- /dev/null +++ b/kits/firestore-incremental-capture/.env.example @@ -0,0 +1,32 @@ +# Copy to .env or .env.. Run scripts/setup.sh before deploying; it +# prints the values to use here. + +# Region for the functions. Must be one of the Dataflow flex template regions. +LOCATION=us-central1 + +# Collection to capture. A multi-segment wildcard is not supported: a Firestore +# trigger only accepts one as its final path segment. +SYNC_COLLECTION_PATH=posts + +# BigQuery changelog destination. +SYNC_DATASET=backup_dataset +SYNC_TABLE=backup_table +DATASET_LOCATION=us + +# Firestore database restorations are written into. Required, and must not be +# "(default)" - a restoration batch-writes over it. Created by setup.sh. +BACKUP_INSTANCE_ID= + +# Bucket the flex template was staged to. Defaults to the project's default +# bucket, which is what setup.sh stages to unless you override it there. +# BUCKET_NAME= + +# Region for Dataflow jobs. Defaults to LOCATION. +# DATAFLOW_REGION= + +# Namespaces the task queues, template object, job names and status documents. +# Must match the INSTANCE_ID passed to setup.sh. +# INSTANCE_ID=firestore-incremental-capture + +# debug | info | warn | error | silent +# LOG_LEVEL=info diff --git a/kits/firestore-incremental-capture/README.md b/kits/firestore-incremental-capture/README.md index 38b1288c9..387178546 100644 --- a/kits/firestore-incremental-capture/README.md +++ b/kits/firestore-incremental-capture/README.md @@ -50,22 +50,51 @@ after setup ran. Set these in `.env` or `.env.`. -| Param | Default | Description | -| ---------------------- | ------------------------------- | -------------------------------------------------------------- | -| `LOCATION` | `us-central1` | Region for the functions. | -| `DATABASE` | `(default)` | Firestore database to capture. | -| `SYNC_COLLECTION_PATH` | `posts` | Collection to capture. `{document=**}` captures everything. | -| `SYNC_DATASET` | `backup_dataset` | BigQuery dataset for the changelog. | -| `SYNC_TABLE` | `backup_table` | BigQuery changelog table. | -| `BACKUP_INSTANCE_ID` | _required_ | Firestore database to restore into. Must not be `DATABASE`. | -| `DATASET_LOCATION` | `us` | BigQuery dataset location. | -| `DATAFLOW_REGION` | `LOCATION` | Region for Dataflow jobs. | -| `BUCKET_NAME` | `.firebasestorage.app` | Bucket holding the flex template. | -| `INSTANCE_ID` | `firestore-incremental-capture` | Namespaces the queues, template, jobs and status documents. | -| `LOG_LEVEL` | `info` | `debug`, `info`, `warn`, `error` or `silent`. | - -Projects created before September 2024 use `.appspot.com` as their default bucket and must -set `BUCKET_NAME` explicitly. +| Param | Default | Description | +| ---------------------- | ------------------------------- | ----------------------------------------------------------- | +| `LOCATION` | `us-central1` | Region for the functions. | +| `SYNC_COLLECTION_PATH` | `posts` | Collection to capture. | +| `SYNC_DATASET` | `backup_dataset` | BigQuery dataset for the changelog. | +| `SYNC_TABLE` | `backup_table` | BigQuery changelog table. | +| `BACKUP_INSTANCE_ID` | _required_ | Firestore database to restore into. Must not be `(default)`. | +| `DATASET_LOCATION` | `us` | BigQuery dataset location. | +| `DATAFLOW_REGION` | `LOCATION` | Region for Dataflow jobs. | +| `BUCKET_NAME` | the project's default bucket | Bucket the flex template was staged to. | +| `INSTANCE_ID` | `firestore-incremental-capture` | Namespaces the queues, template, jobs and status documents. | +| `LOG_LEVEL` | `info` | `debug`, `info`, `warn`, `error` or `silent`. | + +**Only the `(default)` database can be captured.** The restoration pipeline reads its PITR baseline +from `FirestoreOptions.getDefaultInstance()` (`RestorationPipeline.java`), so a non-default source +database would be captured to the changelog but absent from the restored baseline. There is +deliberately no param for it. + +**Only a single collection can be captured.** A Firestore trigger takes a multi-segment wildcard only +as its final path segment, so `SYNC_COLLECTION_PATH={document=**}` produces the undeployable pattern +`{document=**}/{documentId}`. Whole-database capture is not available. + +`BUCKET_NAME` is read from the project's default bucket when unset, rather than guessed from the +project id - the default bucket is `.firebasestorage.app` for projects created after +September 2024 and `.appspot.com` for older ones, and a wrong guess means restoration +launches against a template that was never staged there. + +## Required IAM + +The package declares the roles below with `requiresRole(...)`. Firebase CLI 15.23.0 or later creates a +managed runtime service account for the codebase, grants it these roles, and attaches it to every +function. Declarative security cannot be combined with a custom runtime service account. + +| Role | Why | +| ----------------------------- | --------------------------------------------------------------- | +| `roles/bigquery.dataEditor` | create the changelog dataset/table; insert rows | +| `roles/bigquery.user` | run BigQuery jobs | +| `roles/datastore.user` | write the restoration run-status document | +| `roles/dataflow.developer` | launch the restoration job | +| `roles/iam.serviceAccountUser`| act as the Dataflow worker service account when launching | +| `roles/storage.objectViewer` | read the staged flex template spec | + +`scripts/setup.sh` cannot grant these: the managed account does not exist until the first deploy. What +the script does grant is the separate set of roles the **Dataflow worker** service account needs +(`dataflow.worker`, `datastore.user`, BigQuery read, staging bucket access). ## Usage @@ -103,11 +132,16 @@ switches on each value's type tag and **silently drops any field whose tag it do - **`binary` and `null` fields are dropped.** The pipeline has no case for either, so a restored document loses them. -- **Arrays do not survive.** `buildFirestoreList` rebuilds every element as a map, so an array of - primitives restores as a list of empty maps. +- **Arrays of primitives do not survive.** `buildFirestoreList` rebuilds every element by passing it + to `buildFirestoreMap`, which reads field names at the top level, so `[1, 2]` restores as a list of + empty maps. Arrays of maps do round trip - see the note in `src/serializer.ts` on why array + elements are encoded differently from map fields. - **Changelog replay writes to a malformed path.** `IncrementalCaptureLog.convertToFirestoreValue` applies `createDocumentName` to a path that has already been through it, producing a doubled `projects/…/databases/…/documents/` prefix. +- **Documents sharing an id across collections collide.** The replay query ranks with + `ROW_NUMBER() OVER(PARTITION BY documentId …)`, partitioning by document id rather than path, so + only one of `users/x` and `orders/x` is replayed. The PITR baseline half of a restoration is unaffected; these apply to the changelog replay on top of it. Fixing them means changing the Java, which is out of scope for this migration. diff --git a/kits/firestore-incremental-capture/scripts/setup.sh b/kits/firestore-incremental-capture/scripts/setup.sh index 8f6716477..159952ab3 100755 --- a/kits/firestore-incremental-capture/scripts/setup.sh +++ b/kits/firestore-incremental-capture/scripts/setup.sh @@ -28,7 +28,6 @@ # BACKUP_INSTANCE_ID Firestore database to restore into. Created if absent. # Must not be the captured database. # Optional: -# SOURCE_DATABASE Captured database. Default "(default)". # DATABASE_LOCATION Location for a newly created backup database. # Default "nam5". # LOCATION Region for the functions and Artifact Registry. @@ -38,8 +37,14 @@ # INSTANCE_ID Namespace for the deployed resources. Must match the # kit's INSTANCE_ID param. Default # "firestore-incremental-capture". -# SERVICE_ACCOUNT Runtime service account of the deployed functions. -# Defaults to the App Engine default service account. +# WORKER_SERVICE_ACCOUNT Service account the Dataflow workers run as. +# Defaults to the Compute Engine default service account. +# +# This grants roles to the Dataflow *worker* service account only. The deployed +# functions run as a managed runtime service account that the Firebase CLI +# creates on first deploy and grants the roles declared with requiresRole() in +# src/index.ts - it does not exist yet while this script runs, and cannot be +# granted here. set -euo pipefail @@ -48,7 +53,9 @@ readonly PIPELINE_DIR="${SCRIPT_DIR}/../pipeline" readonly PROJECT_ID="${PROJECT_ID:-}" readonly BACKUP_INSTANCE_ID="${BACKUP_INSTANCE_ID:-}" -readonly SOURCE_DATABASE="${SOURCE_DATABASE:-(default)}" +# The restoration pipeline reads its PITR baseline from the default database +# (RestorationPipeline.java), so that is the only database the kit can capture. +readonly SOURCE_DATABASE="(default)" readonly DATABASE_LOCATION="${DATABASE_LOCATION:-nam5}" readonly LOCATION="${LOCATION:-us-central1}" readonly INSTANCE_ID="${INSTANCE_ID:-firestore-incremental-capture}" @@ -99,12 +106,15 @@ resolve_bucket() { fi } -resolve_service_account() { - if [[ -n "${SERVICE_ACCOUNT:-}" ]]; then - echo "${SERVICE_ACCOUNT}" - else - echo "${PROJECT_ID}@appspot.gserviceaccount.com" +resolve_worker_service_account() { + if [[ -n "${WORKER_SERVICE_ACCOUNT:-}" ]]; then + echo "${WORKER_SERVICE_ACCOUNT}" + return fi + + local project_number + project_number="$(gcloud projects describe "${PROJECT_ID}" --format='value(projectNumber)')" + echo "${project_number}-compute@developer.gserviceaccount.com" } enable_apis() { @@ -115,6 +125,8 @@ enable_apis() { dataflow.googleapis.com \ firestore.googleapis.com \ artifactregistry.googleapis.com \ + storage.googleapis.com \ + compute.googleapis.com \ --project="${PROJECT_ID}" ok "APIs enabled." } @@ -165,15 +177,22 @@ create_artifact_registry() { ok "Repository created." } -grant_roles() { +# Grants what the Dataflow workers need to run a restoration: read the changelog, +# write the backup database, and use the staging bucket. On projects where the +# default compute service account still holds Editor these are already implied, +# but org policy commonly removes that. +grant_worker_roles() { local service_account="$1" - step "Granting Dataflow roles to ${service_account}" + step "Granting Dataflow worker roles to ${service_account}" - # Launching a flex template needs dataflow.developer, and needs to act as the - # worker service account. local role - for role in roles/dataflow.developer roles/iam.serviceAccountUser; do + for role in \ + roles/dataflow.worker \ + roles/datastore.user \ + roles/bigquery.dataViewer \ + roles/bigquery.jobUser \ + roles/storage.objectAdmin; do gcloud projects add-iam-policy-binding "${PROJECT_ID}" \ --member="serviceAccount:${service_account}" \ --role="${role}" \ @@ -181,15 +200,7 @@ grant_roles() { --quiet >/dev/null done - gcloud artifacts repositories add-iam-policy-binding "${INSTANCE_ID}" \ - --location="${LOCATION}" \ - --project="${PROJECT_ID}" \ - --member="serviceAccount:${service_account}" \ - --role=roles/artifactregistry.writer \ - --condition=None \ - --quiet >/dev/null - - ok "Roles granted." + ok "Worker roles granted." } # Built from the vendored source rather than downloaded: the prebuilt jar the @@ -223,20 +234,29 @@ stage_flex_template() { main() { require_config - local bucket service_account jar + # APIs first: resolving the bucket and the worker account both need them. + enable_apis + + local bucket worker_service_account jar bucket="$(resolve_bucket)" - service_account="$(resolve_service_account)" + worker_service_account="$(resolve_worker_service_account)" - enable_apis enable_pitr create_backup_database create_artifact_registry - grant_roles "${service_account}" + grant_worker_roles "${worker_service_account}" jar="$(build_jar | tail -n 1)" stage_flex_template "${jar}" "${bucket}" echo -e "\n${GREEN}Setup complete.${NC}" - echo "Deploy the kit with BACKUP_INSTANCE_ID=${BACKUP_INSTANCE_ID}, INSTANCE_ID=${INSTANCE_ID}, LOCATION=${LOCATION}, BUCKET_NAME=${bucket}." + echo + echo "Set these in .env before deploying:" + echo " BACKUP_INSTANCE_ID=${BACKUP_INSTANCE_ID}" + echo " INSTANCE_ID=${INSTANCE_ID}" + echo " LOCATION=${LOCATION}" + echo " BUCKET_NAME=${bucket}" + echo + echo "The functions' own roles are granted by the Firebase CLI on first deploy." } main "$@" diff --git a/kits/firestore-incremental-capture/src/capture-config.ts b/kits/firestore-incremental-capture/src/capture-config.ts index cdbccf53d..60fcb46b5 100644 --- a/kits/firestore-incremental-capture/src/capture-config.ts +++ b/kits/firestore-incremental-capture/src/capture-config.ts @@ -29,8 +29,6 @@ export interface CaptureConfig { * captures every collection. */ syncCollectionPath: string; - /** Firestore database captured from. Defaults to `(default)`. */ - databaseId?: string; /** * Firestore database restored into. Must already exist and must not be the * captured database - a restoration batch-writes over its contents. @@ -47,12 +45,14 @@ export interface CaptureConfig { /** Region Dataflow jobs run in. Defaults to {@link CaptureConfig.location}. */ dataflowRegion?: string; /** - * Cloud Storage bucket holding the Dataflow flex template. Defaults to - * `.firebasestorage.app`, the default bucket for projects created - * after September 2024. Projects older than that use `.appspot.com` - * and must set this explicitly. + * Cloud Storage bucket holding the Dataflow flex template. Required, and not + * guessed: the default bucket is `.firebasestorage.app` for + * projects created after September 2024 and `.appspot.com` for + * older ones, and guessing wrong means restoration launches against a + * template that is not there. The entry point fills this from the project's + * actual default bucket when the `BUCKET_NAME` param is unset. */ - bucketName?: string; + bucketName: string; /** * Namespaces the deployed resources: the task queues, the flex template * object, the Dataflow job names and the Firestore status documents. Deploy @@ -66,9 +66,15 @@ export interface CaptureConfig { /** {@link CaptureConfig} with every default applied and paths derived. */ export interface ResolvedCaptureConfig - extends Required> { + extends Required> { dataflowRegion: string; - bucketName: string; + /** + * Database the changes are captured from. Always `(default)`: the restoration + * pipeline reads its PITR baseline from `FirestoreOptions.getDefaultInstance()` + * (`RestorationPipeline.java`), so a non-default source would be captured to + * the changelog but silently absent from the restored baseline. + */ + databaseId: "(default)"; /** Fully-qualified name of the database restored into. */ backupInstanceName: string; /** @@ -80,7 +86,8 @@ export interface ResolvedCaptureConfig restoreCollection: string; } -const DEFAULT_DATABASE_ID = "(default)"; +/** The only database the restoration pipeline can read a PITR baseline from. */ +const SOURCE_DATABASE_ID = "(default)"; const DEFAULT_INSTANCE_ID = "firestore-incremental-capture"; const DEFAULT_LOCATION = "us-central1"; const DEFAULT_DATASET_LOCATION = "us"; @@ -90,30 +97,47 @@ const DEFAULT_DATASET_LOCATION = "us"; * * @param config - Caller-supplied configuration. * @returns The fully resolved configuration. - * @throws If the backup database is the same as the captured database, which - * would make a restoration overwrite the source it is restoring from. + * @throws If `backupInstanceId` is empty or is the captured database, either of + * which would make a restoration write over the source it restores from; or if + * `bucketName` is empty, which would leave the flex template path unresolvable. */ export function resolveCaptureConfig( config: CaptureConfig ): ResolvedCaptureConfig { - const databaseId = config.databaseId || DEFAULT_DATABASE_ID; const location = config.location || DEFAULT_LOCATION; const instanceId = config.instanceId || DEFAULT_INSTANCE_ID; - const bucketName = - config.bucketName || `${config.projectId}.firebasestorage.app`; - if (config.backupInstanceId === databaseId) { + const invalid = (detail: string): never => { throw new Error( - `Invalid configuration for firestore-incremental-capture: BACKUP_INSTANCE_ID ` + - `("${config.backupInstanceId}") must differ from the captured database ` + - `("${databaseId}"). A restoration batch-writes over the backup database.` + `Invalid configuration for firestore-incremental-capture: ${detail}` ); + }; + + if (!config.backupInstanceId) { + invalid("BACKUP_INSTANCE_ID is required."); } + if (config.backupInstanceId === SOURCE_DATABASE_ID) { + invalid( + `BACKUP_INSTANCE_ID ("${config.backupInstanceId}") must differ from the ` + + `captured database ("${SOURCE_DATABASE_ID}"). A restoration batch-writes ` + + `over the backup database.` + ); + } + + if (!config.bucketName) { + invalid( + "BUCKET_NAME is required. It must name the bucket the Dataflow flex " + + "template was staged to by scripts/setup.sh." + ); + } + + const bucketName = config.bucketName; + return { projectId: config.projectId, syncCollectionPath: config.syncCollectionPath, - databaseId, + databaseId: SOURCE_DATABASE_ID, backupInstanceId: config.backupInstanceId, datasetId: config.datasetId, tableId: config.tableId, diff --git a/kits/firestore-incremental-capture/src/config.ts b/kits/firestore-incremental-capture/src/config.ts index e069a9a59..d7eaa023f 100644 --- a/kits/firestore-incremental-capture/src/config.ts +++ b/kits/firestore-incremental-capture/src/config.ts @@ -82,7 +82,6 @@ const LOG_LEVEL_OPTIONS = ["debug", "info", "warn", "error", "silent"] as const; /** Deploy-time expressions the entry point needs before params can be read. */ export interface ConfigExpressions { syncCollectionPath: Expression; - database: Expression; location: Expression; } @@ -97,7 +96,6 @@ const params = { default: "us-central1", input: select([...LOCATION_OPTIONS]), }), - database: defineString("DATABASE", { default: "(default)" }), syncCollectionPath: defineString("SYNC_COLLECTION_PATH", { default: "posts", }), @@ -121,7 +119,6 @@ const params = { export const CONFIG_EXPRESSIONS: ConfigExpressions = { syncCollectionPath: params.syncCollectionPath, - database: params.database, location: params.location, }; @@ -150,20 +147,22 @@ function normalizeLogLevel(level: string): LogLevel { * deploy-time discovery pass can analyze the entry point without resolving * params early. * + * @param defaultBucketName - The project's default storage bucket, used when + * the `BUCKET_NAME` param is unset. The param is not defaulted to a guessed + * name because the default bucket's domain differs by project age. * @returns The capture configuration assembled from environment params. */ -export function configFromEnv(): CaptureConfig { +export function configFromEnv(defaultBucketName?: string): CaptureConfig { return { projectId: projectID.value(), syncCollectionPath: params.syncCollectionPath.value(), - databaseId: optional(params.database.value()), backupInstanceId: params.backupInstanceId.value(), datasetId: params.syncDataset.value(), tableId: params.syncTable.value(), datasetLocation: optional(params.datasetLocation.value()), location: optional(params.location.value()), dataflowRegion: optional(params.dataflowRegion.value()), - bucketName: optional(params.bucketName.value()), + bucketName: optional(params.bucketName.value()) || defaultBucketName || "", instanceId: optional(params.instanceId.value()), logLevel: normalizeLogLevel(params.logLevel.value()), }; diff --git a/kits/firestore-incremental-capture/src/dataflow.ts b/kits/firestore-incremental-capture/src/dataflow.ts index 86fff7172..c2ee0678f 100644 --- a/kits/firestore-incremental-capture/src/dataflow.ts +++ b/kits/firestore-incremental-capture/src/dataflow.ts @@ -30,12 +30,10 @@ export class RestorationLauncher { /** * @param config - The resolved capture configuration. * @param client - Dataflow flex templates client, injectable for tests. - * @param now - Clock, injectable so run ids are deterministic in tests. */ constructor( private readonly config: ResolvedCaptureConfig, - private readonly client: FlexTemplatesServiceClient = new FlexTemplatesServiceClient(), - private readonly now: () => number = Date.now + private readonly client: FlexTemplatesServiceClient = new FlexTemplatesServiceClient() ) {} /** @@ -45,12 +43,18 @@ export class RestorationLauncher { * {@link ResolvedCaptureConfig.flexTemplatePath}; see the kit's setup script. * Launching against a missing template fails here rather than at deploy. * + * The run id is derived from the target timestamp rather than the wall clock, + * so it is stable across task-queue retries. Dataflow rejects a duplicate + * active job name, which is what stops a retry after a partial failure - the + * launch succeeded but the status write did not - from starting a second job + * that writes over the backup database concurrently. + * * @param request - The validated restoration request. * @returns The launched job. */ async launch(request: RestorationRequest): Promise { const { config } = this; - const runId = `${config.instanceId}-dataflow-run-${this.now()}`; + const runId = `${config.instanceId}-restore-${request.timestamp}`; logs.info(`Launching restoration job ${runId}`, { labels: { run_id: runId }, diff --git a/kits/firestore-incremental-capture/src/index.ts b/kits/firestore-incremental-capture/src/index.ts index 0919d76ac..0dd2d8381 100644 --- a/kits/firestore-incremental-capture/src/index.ts +++ b/kits/firestore-incremental-capture/src/index.ts @@ -32,6 +32,7 @@ */ import { getApps, initializeApp } from "firebase-admin/app"; +import { getStorage } from "firebase-admin/storage"; import { onDocumentWritten } from "firebase-functions/firestore"; import { onRequest } from "firebase-functions/https"; import { expr } from "firebase-functions/params"; @@ -73,11 +74,20 @@ const LIFECYCLE_RETRY_CONFIG = { maxAttempts: 15, minBackoffSeconds: 60, } as const; +// Granted by the Firebase CLI to the managed runtime service account it creates +// for this codebase. The setup script cannot grant these: the account does not +// exist until the first deploy, and declarative security rules out supplying a +// runtime service account of your own. const REQUIRED_ROLES: ReadonlyArray = [ "roles/bigquery.dataEditor", "roles/bigquery.user", "roles/datastore.user", "roles/dataflow.developer", + // Launching a flex template acts as the Dataflow worker service account. + // Without this, every restoration fails with iam.serviceAccounts.actAs denied. + "roles/iam.serviceAccountUser", + // Reads the staged flex template spec from Cloud Storage. + "roles/storage.objectViewer", ]; for (const role of REQUIRED_ROLES) { @@ -94,16 +104,24 @@ function getHandlerContext(): HandlerContext { return ctx; } - const config = resolveCaptureConfig(configFromEnv()); - - logs.setLogLevel(config.logLevel); - if (getApps().length === 0) { initializeApp(); } + // Read from the initialized app rather than assembled from the project id: + // the default bucket is .firebasestorage.app for projects created + // after September 2024 and .appspot.com for older ones. + const config = resolveCaptureConfig( + configFromEnv(getStorage().bucket().name) + ); + + logs.setLogLevel(config.logLevel); + const changelog = new ChangelogTable(config); - const launcher = new RestorationLauncher(config); + + // Constructed on first use: it loads gRPC protos, and the capture path - which + // is every invocation except a restoration - never touches Dataflow. + let launcher: RestorationLauncher | undefined; ctx = { config, @@ -112,7 +130,10 @@ function getHandlerContext(): HandlerContext { insertChangelogRows: (rows) => changelog.insert(rows), enqueueRestoration: (request) => enqueue(config, RESTORATION_TASK_FUNCTION, request), - launchRestorationJob: (request) => launcher.launch(request), + launchRestorationJob: (request) => { + launcher ??= new RestorationLauncher(config); + return launcher.launch(request); + }, }; return ctx; @@ -132,7 +153,6 @@ export const syncData = onDocumentWritten( { ...functionOptions, document: expr`${CONFIG_EXPRESSIONS.syncCollectionPath}/{documentId}`, - database: CONFIG_EXPRESSIONS.database, retry: true, }, (event) => handleDocumentWrite(event, getHandlerContext()) diff --git a/kits/firestore-incremental-capture/src/serializer.ts b/kits/firestore-incremental-capture/src/serializer.ts index 129c682b5..678e3abeb 100644 --- a/kits/firestore-incremental-capture/src/serializer.ts +++ b/kits/firestore-incremental-capture/src/serializer.ts @@ -61,7 +61,7 @@ export type SerializedDocument = Record; * because the pipeline prefixes `projects/…/databases/…/documents/` itself. * * The pipeline has no case for `binary` or `null`, so those fields are dropped - * on restore. See the kit README for the full list of restoration gaps. + * on restore. See the restoration gaps section of the kit README. * * @param data - Firestore document data, or `undefined` for a document that * does not exist on this side of the change. @@ -115,7 +115,7 @@ function serializeValue(value: unknown): SerializedValue { } if (Array.isArray(value)) { - return { type: "array", value: value.map(serializeValue) }; + return { type: "array", value: value.map(serializeArrayElement) }; } if (typeof value === "object") { @@ -124,3 +124,36 @@ function serializeValue(value: unknown): SerializedValue { return { type: typeof value as SerializedType, value }; } + +/** + * Serializes one array element. + * + * Map elements are emitted as a bare field map, NOT wrapped in a + * `{ type: "map" }` envelope like a map field would be. This asymmetry is + * required by the restoration pipeline: `FirestoreReconstructor.buildFirestoreList` + * passes each element straight to `buildFirestoreMap`, which expects field + * names at the top level and skips anything it cannot read as a tagged field. + * Wrapping a map element restores it as an empty map. + * + * Primitive elements stay tagged, matching the original extension. The pipeline + * cannot reconstruct those either - see the restoration gaps in the README - + * but changing the encoding here would not fix it. + * + * @param element - One element of a Firestore array field. + * @returns The serialized element. + */ +function serializeArrayElement(element: unknown): unknown { + if ( + element !== null && + typeof element === "object" && + !Array.isArray(element) && + !Buffer.isBuffer(element) && + !(element instanceof Timestamp) && + !(element instanceof GeoPoint) && + !(element instanceof DocumentReference) + ) { + return serializeDocument(element); + } + + return serializeValue(element); +} diff --git a/kits/firestore-incremental-capture/tests/capture-config.test.ts b/kits/firestore-incremental-capture/tests/capture-config.test.ts index 98345ef36..052beb827 100644 --- a/kits/firestore-incremental-capture/tests/capture-config.test.ts +++ b/kits/firestore-incremental-capture/tests/capture-config.test.ts @@ -28,6 +28,7 @@ function config(overrides: Partial = {}): CaptureConfig { backupInstanceId: "backup-db", datasetId: "backup_dataset", tableId: "backup_table", + bucketName: "test-project.firebasestorage.app", ...overrides, }; } @@ -57,12 +58,18 @@ describe("resolveCaptureConfig", () => { expect(resolved.dataflowRegion).toBe("us-central1"); }); - test("defaults the bucket to the post-2024 default bucket name", () => { - expect(resolveCaptureConfig(config()).bucketName).toBe( - "test-project.firebasestorage.app" + test("requires an explicit bucket rather than guessing one", () => { + // Guessing is unsafe: the default bucket domain differs by project age, and + // a wrong guess means launching against a template that is not staged there. + expect(() => resolveCaptureConfig(config({ bucketName: "" }))).toThrow( + /BUCKET_NAME is required/ ); }); + test("pins the captured database to the only one the pipeline can restore", () => { + expect(resolveCaptureConfig(config()).databaseId).toBe("(default)"); + }); + test("derives the backup instance name and flex template path", () => { const resolved = resolveCaptureConfig(config()); @@ -87,12 +94,12 @@ describe("resolveCaptureConfig", () => { expect(() => resolveCaptureConfig(config({ backupInstanceId: "(default)" })) ).toThrow(/must differ from the captured database/); + }); + test("rejects an empty backup database", () => { expect(() => - resolveCaptureConfig( - config({ databaseId: "primary", backupInstanceId: "primary" }) - ) - ).toThrow(/must differ from the captured database/); + resolveCaptureConfig(config({ backupInstanceId: "" })) + ).toThrow(/BACKUP_INSTANCE_ID is required/); }); }); diff --git a/kits/firestore-incremental-capture/tests/dataflow.test.ts b/kits/firestore-incremental-capture/tests/dataflow.test.ts new file mode 100644 index 000000000..dc406c7a0 --- /dev/null +++ b/kits/firestore-incremental-capture/tests/dataflow.test.ts @@ -0,0 +1,199 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { + type CaptureConfig, + resolveCaptureConfig, +} from "../src/capture-config"; + +vi.mock("../src/logs", () => ({ + setLogLevel: vi.fn(), + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +})); + +const set = vi.fn().mockResolvedValue(undefined); +const doc = vi.fn(() => ({ set })); +const collection = vi.fn(() => ({ doc })); +const getFirestore = vi.fn(() => ({ collection })); + +vi.mock("firebase-admin/firestore", () => ({ + getFirestore: (...args: unknown[]) => getFirestore(...(args as [])), +})); + +// Stubbed so constructing the launcher does not load gRPC protos. +vi.mock("@google-cloud/dataflow", () => ({ + FlexTemplatesServiceClient: class {}, +})); + +const { RestorationLauncher } = await import("../src/dataflow"); + +function config(overrides: Partial = {}) { + return resolveCaptureConfig({ + projectId: "test-project", + syncCollectionPath: "users", + backupInstanceId: "backup-db", + datasetId: "ds", + tableId: "tbl", + bucketName: "test-project.firebasestorage.app", + ...overrides, + }); +} + +/** + * Fake flex templates client capturing the launch request. Pass a response of + * `{}` to simulate Dataflow not reporting a job name. + */ +function fakeClient(response: object = { job: { name: "job-1" } }) { + return { + launchFlexTemplate: vi.fn().mockResolvedValue([response]), + }; +} + +describe("RestorationLauncher", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + test("launches from the same template path the setup script stages to", async () => { + const client = fakeClient(); + const cfg = config(); + + await new RestorationLauncher(cfg, client as never).launch({ + timestamp: 1700000000, + }); + + const [request] = client.launchFlexTemplate.mock.calls[0]; + expect(request.launchParameter.containerSpecGcsPath).toBe( + "gs://test-project.firebasestorage.app/firestore-incremental-capture-dataflow-restore" + ); + expect(request.launchParameter.containerSpecGcsPath).toBe( + cfg.flexTemplatePath + ); + }); + + test("passes the five parameters the pipeline declares", async () => { + const client = fakeClient(); + + await new RestorationLauncher(config(), client as never).launch({ + timestamp: 1700000000, + }); + + const [request] = client.launchFlexTemplate.mock.calls[0]; + expect(request.projectId).toBe("test-project"); + expect(request.launchParameter.parameters).toEqual({ + timestamp: "1700000000", + firestoreCollectionId: "users", + firestoreDb: "backup-db", + bigQueryDataset: "ds", + bigQueryTable: "tbl", + }); + }); + + test("launches in the Dataflow region, not the functions region", async () => { + const client = fakeClient(); + + await new RestorationLauncher( + config({ location: "us-central1", dataflowRegion: "europe-west1" }), + client as never + ).launch({ timestamp: 1700000000 }); + + expect(client.launchFlexTemplate.mock.calls[0][0].location).toBe( + "europe-west1" + ); + }); + + test("maps the capture-everything wildcard for the pipeline", async () => { + const client = fakeClient(); + + await new RestorationLauncher( + config({ syncCollectionPath: "{document=**}" }), + client as never + ).launch({ timestamp: 1700000000 }); + + expect( + client.launchFlexTemplate.mock.calls[0][0].launchParameter.parameters + .firestoreCollectionId + ).toBe("*"); + }); + + test("derives a run id that is stable across retries of the same request", async () => { + // Dataflow rejects a duplicate active job name, which is what stops a retry + // after a partial failure from starting a second concurrent restoration. + const first = fakeClient(); + const second = fakeClient(); + + const a = await new RestorationLauncher(config(), first as never).launch({ + timestamp: 1700000000, + }); + const b = await new RestorationLauncher(config(), second as never).launch({ + timestamp: 1700000000, + }); + + expect(a.runId).toBe("firestore-incremental-capture-restore-1700000000"); + expect(b.runId).toBe(a.runId); + expect( + first.launchFlexTemplate.mock.calls[0][0].launchParameter.jobName + ).toBe(a.runId); + }); + + test("gives a different run id to a different target timestamp", async () => { + const client = fakeClient(); + const launcher = new RestorationLauncher(config(), client as never); + + const a = await launcher.launch({ timestamp: 1700000000 }); + const b = await launcher.launch({ timestamp: 1700000001 }); + + expect(a.runId).not.toBe(b.runId); + }); + + test("records the run against the captured database", async () => { + const client = fakeClient(); + const cfg = config(); + + await new RestorationLauncher(cfg, client as never).launch({ + timestamp: 1700000000, + }); + + expect(getFirestore).toHaveBeenCalledWith("(default)"); + expect(collection).toHaveBeenCalledWith(cfg.restoreCollection); + expect(doc).toHaveBeenCalledWith( + "firestore-incremental-capture-restore-1700000000" + ); + expect(set).toHaveBeenCalledWith({ + runId: "firestore-incremental-capture-restore-1700000000", + jobName: "job-1", + timestamp: 1700000000, + status: "launched", + }); + }); + + test("records a null job name when Dataflow reports none", async () => { + const client = fakeClient({}); + + const job = await new RestorationLauncher(config(), client as never).launch( + { + timestamp: 1700000000, + } + ); + + expect(job.jobName).toBeUndefined(); + expect(set.mock.calls[0][0].jobName).toBeNull(); + }); +}); diff --git a/kits/firestore-incremental-capture/tests/handlers.test.ts b/kits/firestore-incremental-capture/tests/handlers.test.ts index d8fe4dcfd..6a0fd7adc 100644 --- a/kits/firestore-incremental-capture/tests/handlers.test.ts +++ b/kits/firestore-incremental-capture/tests/handlers.test.ts @@ -63,6 +63,7 @@ function makeCtx(): HandlerContext { backupInstanceId: "backup-db", datasetId: "ds", tableId: "tbl", + bucketName: "test-project.firebasestorage.app", }), enqueueChangelogRow: vi.fn().mockResolvedValue(undefined), insertChangelogRows: vi.fn().mockResolvedValue(undefined), diff --git a/kits/firestore-incremental-capture/tests/wire-format.test.ts b/kits/firestore-incremental-capture/tests/wire-format.test.ts new file mode 100644 index 000000000..dc17fff9f --- /dev/null +++ b/kits/firestore-incremental-capture/tests/wire-format.test.ts @@ -0,0 +1,206 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Golden shapes for the changelog wire format. + * + * The restoration pipeline (`pipeline/`, Java) parses whatever this produces, so + * these assertions are the contract between the two languages, not merely a + * description of the current implementation. The expected values are + * transcribed from the original extension's serializer tests, which are the + * authoritative record of the format the pipeline was written against. + * + * One deliberate divergence: DocumentReference is tagged `reference`, not the + * extension's `documentReference`. `FirestoreReconstructor` upper-cases the tag + * and switches on `REFERENCE`, so the extension's spelling fell through to + * `default: continue` and dropped the field. + */ + +import { initializeApp } from "firebase-admin/app"; +import { + type DocumentReference, + getFirestore, + GeoPoint, + Timestamp, +} from "firebase-admin/firestore"; +import { beforeAll, describe, expect, test } from "vitest"; +import { serializeDocument } from "../src/serializer"; + +let ref: DocumentReference; + +beforeAll(() => { + // Constructing a reference is offline; nothing here contacts Firestore. + initializeApp({ projectId: "demo-test" }); + ref = getFirestore().doc("products/abc"); +}); + +describe("changelog wire format", () => { + test("tags a string, number and boolean", () => { + expect( + serializeDocument({ s: "Hello, Firestore!", n: 42, b: true }) + ).toEqual({ + s: { type: "string", value: "Hello, Firestore!" }, + n: { type: "number", value: 42 }, + b: { type: "boolean", value: true }, + }); + }); + + test("nests a GeoPoint as tagged coordinates", () => { + expect( + serializeDocument({ geoPointValue: new GeoPoint(52.379189, 4.899431) }) + ).toEqual({ + geoPointValue: { + type: "geopoint", + value: { + latitude: { type: "number", value: 52.379189 }, + longitude: { type: "number", value: 4.899431 }, + }, + }, + }); + }); + + test("tags a DocumentReference as 'reference' with its relative path", () => { + // The pipeline prefixes `projects/…/databases/…/documents/` itself, so the + // value must be the relative path, and the tag must match `case "REFERENCE"`. + expect(serializeDocument({ documentReferenceValue: ref })).toEqual({ + documentReferenceValue: { type: "reference", value: "products/abc" }, + }); + }); + + test("converts a Timestamp to an ISO string", () => { + const timestampValue = Timestamp.fromDate( + new Date("2026-01-02T03:04:05.000Z") + ); + + expect(serializeDocument({ timestampValue })).toEqual({ + timestampValue: { + type: "timestamp", + value: "2026-01-02T03:04:05.000Z", + }, + }); + }); + + test("wraps a map field in a 'map' envelope", () => { + expect(serializeDocument({ mapValue: { nested: "test" } })).toEqual({ + mapValue: { + type: "map", + value: { nested: { type: "string", value: "test" } }, + }, + }); + }); + + test("emits array elements that are maps as bare field maps", () => { + // No `{ type: "map" }` envelope: buildFirestoreList passes each element + // straight to buildFirestoreMap, which reads field names at the top level. + // Wrapping these restores them as empty maps. + expect( + serializeDocument({ + arrayValue: [ + { + stringValue: "test", + integerValue: 42, + floatValue: 42.42, + booleanValue: true, + nullValue: null, + }, + ], + }) + ).toEqual({ + arrayValue: { + type: "array", + value: [ + { + stringValue: { type: "string", value: "test" }, + integerValue: { type: "number", value: 42 }, + floatValue: { type: "number", value: 42.42 }, + booleanValue: { type: "boolean", value: true }, + nullValue: { type: "null", value: null }, + }, + ], + }, + }); + }); + + test("keeps the envelope on Firestore types nested inside an array element", () => { + const timestampValue = Timestamp.fromDate( + new Date("2026-01-02T03:04:05.000Z") + ); + + expect( + serializeDocument({ + arrayValue: [ + { + nestedString: "nestedTest", + nestedObject: { deepNestedValue: "deepValue" }, + geoPointValue: new GeoPoint(52.379189, 4.899431), + timestampValue, + }, + ], + }) + ).toEqual({ + arrayValue: { + type: "array", + value: [ + { + nestedString: { type: "string", value: "nestedTest" }, + nestedObject: { + type: "map", + value: { + deepNestedValue: { type: "string", value: "deepValue" }, + }, + }, + geoPointValue: { + type: "geopoint", + value: { + latitude: { type: "number", value: 52.379189 }, + longitude: { type: "number", value: 4.899431 }, + }, + }, + timestampValue: { + type: "timestamp", + value: "2026-01-02T03:04:05.000Z", + }, + }, + ], + }, + }); + }); + + test("tags primitive array elements individually", () => { + expect(serializeDocument({ arrayValue: ["string", 42, true] })).toEqual({ + arrayValue: { + type: "array", + value: [ + { type: "string", value: "string" }, + { type: "number", value: 42 }, + { type: "boolean", value: true }, + ], + }, + }); + }); + + test("base64-encodes a Buffer", () => { + expect(serializeDocument({ binaryValue: Buffer.from("hi") })).toEqual({ + binaryValue: { type: "binary", value: "aGk=" }, + }); + }); + + test("tags null rather than omitting the field", () => { + expect(serializeDocument({ nullValue: null })).toEqual({ + nullValue: { type: "null", value: null }, + }); + }); +}); From 507b08399b9397a9a0b5036849a343e8b4119ff9 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Tue, 11 Aug 2026 15:03:59 +0100 Subject: [PATCH 05/10] chore(kits): remove incremental capture legacy reference The kit is written and the wire format it shares with the Dataflow pipeline is pinned by tests/wire-format.test.ts, transcribed from the reference copy's own serializer tests. Recoverable from 43df05ea. Also drops the now-dead legacy entry from the firebase.json deploy ignore list. --- .../firebase.json | 9 +- .../legacy/CHANGELOG.md | 52 --- .../legacy/POSTINSTALL.md | 104 ----- .../legacy/PREINSTALL.md | 53 --- .../legacy/README.md | 116 ----- .../legacy/extension.yaml | 223 ---------- .../legacy/functions/.gitignore | 9 - .../__tests__/backupDatabase.test.ts | 75 ---- .../__tests__/firestoreSerializer.test.ts | 418 ------------------ .../functions/__tests__/functions.test.ts | 126 ------ .../legacy/functions/__tests__/helpers.ts | 54 --- .../__tests__/manualTesting/backup-test.js | 72 --- .../__tests__/manualTesting/createTestData.js | 74 ---- .../__tests__/manualTesting/exportfromBQ.js | 30 -- .../legacy/functions/__tests__/tsconfig.json | 4 - .../legacy/functions/__tests__/types.ts | 24 - .../legacy/functions/jest.config.js | 32 -- .../legacy/functions/package.json | 48 -- .../legacy/functions/src/config.ts | 71 --- .../src/constants/bq_backup_schema.ts | 24 - .../src/dataflow/build_flex_template.ts | 54 --- .../functions/src/dataflow/cloud_build.ts | 95 ---- .../src/dataflow/on_complete_handler.ts | 30 -- .../src/dataflow/trigger_dataflow_job.ts | 71 --- .../legacy/functions/src/index.ts | 75 ---- .../legacy/functions/src/logs.ts | 49 -- .../src/tasks/on_backup_restore_handler.ts | 143 ------ .../tasks/on_firestore_backup_init_handler.ts | 77 ---- .../tasks/on_http_run_restoration_handler.ts | 53 --- .../src/tasks/on_run_initial_setup_handler.ts | 44 -- .../src/tasks/on_run_restoration_handler.ts | 30 -- .../src/tasks/on_sync_data_handler.ts | 64 --- .../src/tasks/sync_data_task_handler.ts | 35 -- .../legacy/functions/src/utils/big_query.ts | 124 ------ .../legacy/functions/src/utils/database.ts | 45 -- .../src/utils/firestore_serializer.ts | 149 ------- .../functions/src/utils/import_export.ts | 77 ---- .../legacy/functions/src/utils/serialize.ts | 41 -- .../legacy/functions/tsconfig.dev.json | 3 - .../legacy/functions/tsconfig.json | 14 - .../functions/build_dataflow_template.sh | 14 - .../functions/download_restore_firestore.sh | 42 -- .../legacy/install/functions/enable_pitr.sh | 9 - .../functions/setup_artifact_registry.sh | 18 - .../install/functions/setup_firestore.sh | 18 - .../functions/setup_service_account.sh | 71 --- .../legacy/install/run.sh | 83 ---- .../vitest.config.ts | 3 +- 48 files changed, 2 insertions(+), 3147 deletions(-) delete mode 100644 kits/firestore-incremental-capture/legacy/CHANGELOG.md delete mode 100644 kits/firestore-incremental-capture/legacy/POSTINSTALL.md delete mode 100644 kits/firestore-incremental-capture/legacy/PREINSTALL.md delete mode 100644 kits/firestore-incremental-capture/legacy/README.md delete mode 100644 kits/firestore-incremental-capture/legacy/extension.yaml delete mode 100644 kits/firestore-incremental-capture/legacy/functions/.gitignore delete mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/backupDatabase.test.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/firestoreSerializer.test.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/functions.test.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/helpers.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/backup-test.js delete mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/createTestData.js delete mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/exportfromBQ.js delete mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/tsconfig.json delete mode 100644 kits/firestore-incremental-capture/legacy/functions/__tests__/types.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/jest.config.js delete mode 100644 kits/firestore-incremental-capture/legacy/functions/package.json delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/config.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/constants/bq_backup_schema.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/dataflow/build_flex_template.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/dataflow/cloud_build.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/dataflow/on_complete_handler.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/dataflow/trigger_dataflow_job.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/index.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/logs.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/tasks/on_backup_restore_handler.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/tasks/on_firestore_backup_init_handler.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/tasks/on_http_run_restoration_handler.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_initial_setup_handler.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_restoration_handler.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/tasks/on_sync_data_handler.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/tasks/sync_data_task_handler.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/utils/big_query.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/utils/database.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/utils/firestore_serializer.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/utils/import_export.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/src/utils/serialize.ts delete mode 100644 kits/firestore-incremental-capture/legacy/functions/tsconfig.dev.json delete mode 100644 kits/firestore-incremental-capture/legacy/functions/tsconfig.json delete mode 100644 kits/firestore-incremental-capture/legacy/install/functions/build_dataflow_template.sh delete mode 100644 kits/firestore-incremental-capture/legacy/install/functions/download_restore_firestore.sh delete mode 100644 kits/firestore-incremental-capture/legacy/install/functions/enable_pitr.sh delete mode 100644 kits/firestore-incremental-capture/legacy/install/functions/setup_artifact_registry.sh delete mode 100644 kits/firestore-incremental-capture/legacy/install/functions/setup_firestore.sh delete mode 100644 kits/firestore-incremental-capture/legacy/install/functions/setup_service_account.sh delete mode 100755 kits/firestore-incremental-capture/legacy/install/run.sh diff --git a/kits/firestore-incremental-capture/firebase.json b/kits/firestore-incremental-capture/firebase.json index 254a2a570..0abf0a395 100644 --- a/kits/firestore-incremental-capture/firebase.json +++ b/kits/firestore-incremental-capture/firebase.json @@ -3,14 +3,7 @@ { "source": ".", "codebase": "firestore-incremental-capture", - "ignore": [ - "node_modules", - ".git", - "src", - "legacy", - "pipeline", - "*.local" - ], + "ignore": ["node_modules", ".git", "src", "pipeline", "*.local"], "predeploy": ["npm --prefix \"$RESOURCE_DIR\" run build"] } ] diff --git a/kits/firestore-incremental-capture/legacy/CHANGELOG.md b/kits/firestore-incremental-capture/legacy/CHANGELOG.md deleted file mode 100644 index 502298cff..000000000 --- a/kits/firestore-incremental-capture/legacy/CHANGELOG.md +++ /dev/null @@ -1,52 +0,0 @@ -## Version 0.0.12 - -chore: complete runtime migration to Node.js 22 - -## Version 0.0.11 - -chore: bump runtime to Node.js 22 -chore: npm run audit - -## Version 0.0.10 - -chore: bump dependencies to fix vulnerabilities - -## Version 0.0.9 - -chore: bump dependencies - -## Version 0.0.8 - -chore: update and audit packages - -## Version 0.0.7 - -fixed: bump to nodejs20 runtime in functions and run npm audit fix - -fixed: support new default bucket suffix - -## Version 0.0.6 - -fixed - deployment, documentation and scripting updates - -## Version 0.0.5 - -docs: fix POSTINSTALL instruction scripts, improve backup instance id param and regexes - -## Version 0.0.4 - -docs: update PREINSTALL, display name, and icon. - -refactor: removed legacy code - -## Version 0.0.3 - -docs: Add author and contributors field, add license headers - -## Version 0.0.2 - -docs: Add to the PREINSTALL.md and generate README.md - -## Version 0.0.1 - -Initial release of the firestore-incremental-capture extension. diff --git a/kits/firestore-incremental-capture/legacy/POSTINSTALL.md b/kits/firestore-incremental-capture/legacy/POSTINSTALL.md deleted file mode 100644 index e178caf38..000000000 --- a/kits/firestore-incremental-capture/legacy/POSTINSTALL.md +++ /dev/null @@ -1,104 +0,0 @@ -## Enable PITR in the Google Cloud Console - -Follow the guidelines here [here](https://firebase.google.com/docs/firestore/use-pitr#gcloud) to enable PITR on your current database. - -## Creating a secondary Firestore database - -```bash - gcloud alpha firestore databases create --database=DATABASE_ID --location=LOCATION --type=firestore-native --project=${param:PROJECT_ID} -``` - -More information on this can be found [here](https://cloud.google.com/sdk/gcloud/reference/alpha/firestore/databases/create) - -## Building the Dataflow Flex Template - -Before this extension can run restoration jobs from BigQuery to Firestore, you must build the Dataflow Flex Template. This is a one-time process that you must perform before you can use the extension. - -We have detailed the steps below, or there is a single script you can run which will perform all the steps for you [here](https://github.com/GoogleCloudPlatform/firebase-extensions/blob/main/firestore-incremental-capture/install/run.sh). - -1. Find your extensions's service account email: - - ```bash - gcloud iam service-accounts list --format="value(EMAIL)" --filter="displayName='Firebase Extensions ${param:EXT_INSTANCE_ID} service account' AND DISABLED=False" --project="${param:PROJECT_ID}" - ``` - - You can also do this through the console, by navigating to https://console.cloud.google.com/iam-admin/serviceaccounts?authuser=0&project=${param:PROJECT_ID} - -2. [Configure the Artificat Registery](https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates?hl=en#configure): - -```bash - gcloud artifacts repositories create ${param:EXT_INSTANCE_ID} \ - --repository-format=docker \ - --location=${param:LOCATION} \ - --project=${param:PROJECT_ID} \ - --async -``` - -Configure Docker to authenticate requests for Artifact Registry: - -```bash -gcloud auth configure-docker ${param:LOCATION}-docker.pkg.dev -``` - -3. Add required policy binding for the repository: - -```bash - gcloud artifacts repositories add-iam-policy-binding ${param:EXT_INSTANCE_ID} \ - --location=${param:LOCATION} \ - --project=${param:PROJECT_ID} \ - --member=serviceAccount:SERVICE_ACCOUNT_EMAIL \ - --role=roles/artifactregistry.writer -``` - -4. Add the required role for the extension service account to trigger Dataflow: - - ```bash - gcloud projects add-iam-policy-binding ${param:PROJECT_ID} \ - --project ${param:PROJECT_ID} \ - --member=serviceAccount:SA_EMAIL \ - --role=roles/dataflow.developer - ``` - -5. Add the required role for the extension service account to trigger Dataflow: - - ```bash - gcloud projects add-iam-policy-binding ${param:PROJECT_ID} \ - --project ${param:PROJECT_ID} \ - --member=serviceAccount:SA_EMAIL \ - --role=roles/iam.serviceAccountUser - ``` - -6. Add the required role for the extension service account to trigger Dataflow: - - ```bash - gcloud projects add-iam-policy-binding ${param:PROJECT_ID} \ - --project ${param:PROJECT_ID} \ - --member=serviceAccount:SA_EMAIL \ - --role=roles/artifactregistry.writer - ``` - -7. Download the JAR file for the Dataflow Flex Template [here](https://github.com/GoogleCloudPlatform/firebase-extensions/tree/main/firestore-incremental-capture-pipeline/target/restore-firestore.jar). -8. Run the following command to build the Dataflow Flex Template. Note that Cloud Storage buckets provisioned after September 30th 2024 are suffixed by `.firebasestorage.app` rather than `.appspot.com` and you should change the following command accordingly: - -```bash - gcloud dataflow flex-template build gs://${param:PROJECT_ID}.appspot.com/${param:EXT_INSTANCE_ID}-dataflow-restore \ - --image-gcr-path ${param:LOCATION}-docker.pkg.dev/${param:PROJECT_ID}/${param:EXT_INSTANCE_ID}/dataflow/restore:latest \ - --sdk-language JAVA \ - --flex-template-base-image JAVA11 \ - --jar /path/to/restore-firestore.jar \ - --env FLEX_TEMPLATE_JAVA_MAIN_CLASS="com.pipeline.RestorationPipeline" \ - --project ${param:PROJECT_ID} -``` - -## Triggering a restoration job - -You can trigger a restoration job by calling the `restoreFirestore` function [here](https://${LOCATION}-${PROJECT_ID}.cloudfunctions.net/${EXT_INSTANCE_ID}). - -Here is an example that will run from one hour ago: - -```bash -curl -m 70 -X POST https://us-central1-${PROJECT_ID}.cloudfunctions.net/ext-firestore-incremental-capture-onHttpRunRestoration \ --H "Authorization: bearer $(gcloud auth print-identity-token)" \ --H "Content-Type: application/json" \ --d "{\"timestamp\":$(date -u -v-1H +%s)}" -``` diff --git a/kits/firestore-incremental-capture/legacy/PREINSTALL.md b/kits/firestore-incremental-capture/legacy/PREINSTALL.md deleted file mode 100644 index 914f90252..000000000 --- a/kits/firestore-incremental-capture/legacy/PREINSTALL.md +++ /dev/null @@ -1,53 +0,0 @@ -This extension provides an automated, incremental backup solution that extends native Firestore capabilities. Generally, you should consider [Firestore’s native Point in Time Recovery](https://firebase.google.com/docs/firestore/use-pitr) and [Scheduled Backups](https://cloud.google.com/firestore/docs/backups) solutions as a first option. However, if those features don’t meet your needs, this extension can be a more flexible alternative. - -This extension provides an automated, incremental backup solution that extends native Firestore capabilities. Generally, you should consider Firestore’s native Point in Time Recovery and Scheduled Backups solution as a first option. However, if those features don’t meet your needs, this extension can be a more flexible alternative. - -With this extension, you can capture and retain incremental changes in Firestore for up to 30 days or more, allowing for point-in-time recovery well beyond the default 7-day window. - -The extension captures changes on every Firestore write and stores the change incrementally in BigQuery. This data capture mechanism ensures a complete history is maintained, enabling recovery to any point within the configured backup period. - -You can choose to incrementally capture a single collection, a collection group using wildcards, or an entire Firestore database. - -The extension also provides a Dataflow connector that can incrementally restore data from BigQuery to Firestore. Installation is done through a simple script that needs to be executed by you, and instructions to do this are provided upon installation. After installation, triggering the restoration is as simple as calling a Cloud Function. - -This extension is subject to [BigQuery write throughput limitations and availability limitations](https://cloud.google.com/bigquery/quotas), as well as [Cloud Functions at-least-once delivery guarantee](https://cloud.google.com/functions/docs/concepts/execution-environment). Since data is mirrored into BigQuery through Cloud Events, it is recommended to restore to timestamp prior to the current time to prevent missing data. - -## Additional Setup - -Before this extension can run restoration jobs from BigQuery to Firestore, you’ll need to: - -- [Set up Cloud Firestore in your Firebase project](https://firebase.google.com/docs/firestore/quickstart). -- [Enable PiTR in your Firestore database instance](https://firebase.google.com/docs/firestore/use-pitr) -- Ensure that a separate Firestore instance exists. A valid database must exist for the restoration to backup to. Ensure that a separate Firestore instance exists If one does not exist, you can create one with the following script: - -```bash - gcloud alpha firestore databases create \ - --database=DATABASE_ID \ - --location=LOCATION \ - --type=firestore-native \ - --project=PROJECT_ID -``` - -(Note that this extension currently only works on database instances in `firestore-native` mode). - -Further instructions are provided upon installation. - -### Billing - -To install an extension, your project must be on the Blaze (pay as you go) plan. You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). -This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service's no-cost tier: - -- Dataflow -- BigQuery -- Artifact Registry -- Cloud EventArc -- Cloud Functions (See [FAQs](https://firebase.google.com/support/faq#extensions-pricing)) - -[Learn more about Firebase billing](https://firebase.google.com/pricing). - -### Additional Uninstall Steps - -> ⚠️ The extension does not delete various resources automatically on uninstall. - -After you have uninstalled this extension, you will be required to remove the dataflow pipeline which was set up. You can do this through the -Google Cloud Console [here](https://console.cloud.google.com/dataflow/pipelines). This extension will also create artifacts stored in the Artifact Registry, which you can also manage from the console [here](https://console.cloud.google.com/artifacts). diff --git a/kits/firestore-incremental-capture/legacy/README.md b/kits/firestore-incremental-capture/legacy/README.md deleted file mode 100644 index d43f3374f..000000000 --- a/kits/firestore-incremental-capture/legacy/README.md +++ /dev/null @@ -1,116 +0,0 @@ -# Firestore Incremental Backup Stream - -**Author**: Google Cloud (**[https://cloud.google.com/](https://cloud.google.com/)**) - -**Description**: Offers a cost-effective, flexible disaster recovery mechanism for Firestore. - - - -**Details**: This extension provides an automated, incremental backup solution that extends native Firestore capabilities. Generally, you should consider [Firestore’s native Point in Time Recovery](https://firebase.google.com/docs/firestore/use-pitr) and [Scheduled Backups](https://cloud.google.com/firestore/docs/backups) solutions as a first option. However, if those features don’t meet your needs, this extension can be a more flexible alternative. - -This extension provides an automated, incremental backup solution that extends native Firestore capabilities. Generally, you should consider Firestore’s native Point in Time Recovery and Scheduled Backups solution as a first option. However, if those features don’t meet your needs, this extension can be a more flexible alternative. - -With this extension, you can capture and retain incremental changes in Firestore for up to 30 days or more, allowing for point-in-time recovery well beyond the default 7-day window. - -The extension captures changes on every Firestore write and stores the change incrementally in BigQuery. This data capture mechanism ensures a complete history is maintained, enabling recovery to any point within the configured backup period. - -You can choose to incrementally capture a single collection, a collection group using wildcards, or an entire Firestore database. - -The extension also provides a Dataflow connector that can incrementally restore data from BigQuery to Firestore. Installation is done through a simple script that needs to be executed by you, and instructions to do this are provided upon installation. After installation, triggering the restoration is as simple as calling a Cloud Function. - -This extension is subject to [BigQuery write throughput limitations and availability limitations](https://cloud.google.com/bigquery/quotas), as well as [Cloud Functions at-least-once delivery guarantee](https://cloud.google.com/functions/docs/concepts/execution-environment). Since data is mirrored into BigQuery through Cloud Events, it is recommended to restore to timestamp prior to the current time to prevent missing data. - -## Additional Setup - -Before this extension can run restoration jobs from BigQuery to Firestore, you’ll need to: - -- [Set up Cloud Firestore in your Firebase project](https://firebase.google.com/docs/firestore/quickstart). -- [Enable PiTR in your Firestore database instance](https://firebase.google.com/docs/firestore/use-pitr) -- Ensure that a separate Firestore instance exists. A valid database must exist for the restoration to backup to. Ensure that a separate Firestore instance exists If one does not exist, you can create one with the following script: - -```bash - gcloud alpha firestore databases create \ - --database=DATABASE_ID \ - --location=LOCATION \ - --type=firestore-native \ - --project=PROJECT_ID -``` - -(Note that this extension currently only works on database instances in `firestore-native` mode). - -Further instructions are provided upon installation. - -### Billing - -To install an extension, your project must be on the Blaze (pay as you go) plan. You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). -This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service's no-cost tier: - -- Dataflow -- BigQuery -- Artifact Registry -- Cloud EventArc -- Cloud Functions (See [FAQs](https://firebase.google.com/support/faq#extensions-pricing)) - -[Learn more about Firebase billing](https://firebase.google.com/pricing). - -### Additional Uninstall Steps - -> ⚠️ The extension does not delete various resources automatically on uninstall. - -After you have uninstalled this extension, you will be required to remove the dataflow pipeline which was set up. You can do this through the -Google Cloud Console [here](https://console.cloud.google.com/dataflow/pipelines). This extension will also create artifacts stored in the Artifact Registry, which you can also manage from the console [here](https://console.cloud.google.com/artifacts). - - - - -**Configuration Parameters:** - -* Cloud Functions location: Where do you want to deploy the functions created for this extension? You usually want a location close to your database. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). - -* Collection path: What is the path to the collection that contains the strings that you want to capture all changes of? Use `{document=**}` to capture all collections. - - -* Bigquery dataset Id: The id of the Bigquery dataset to sync data to. - - -* Bigquery table Id: The id of the Bigquery table to sync data to. - - -* Backup instance Id: The name of the Firestore instance to backup the database to. - - - - -**Cloud Functions:** - -* **runInitialSetup:** Creates the backup BigQuery database if it does not exist - -* **syncData:** Enqueues a task to sync data to BigQuery - -* **syncDataTask:** Distributed cloud task for syncing data to BigQuery - -* **onHttpRunRestoration:** Starts a new restoration task - -* **onBackupRestore:** Exports data from storage to a pre-defined Firestore instance. - - - -**APIs Used**: - -* eventarc.googleapis.com (Reason: Powers all events and triggers) - -* bigquery.googleapis.com (Reason: Running queries) - -* dataflow.googleapis.com (Reason: Running dataflow jobs) - - - -**Access Required**: - - - -This extension will operate with the following project IAM roles: - -* datastore.user (Reason: Allows the extension to write updates to the database.) - -* bigquery.dataEditor (Reason: Allows the creation of BQ jobs to import Firestore backups.) diff --git a/kits/firestore-incremental-capture/legacy/extension.yaml b/kits/firestore-incremental-capture/legacy/extension.yaml deleted file mode 100644 index 885b96667..000000000 --- a/kits/firestore-incremental-capture/legacy/extension.yaml +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: firestore-incremental-capture -version: 0.0.12 -specVersion: v1beta - -icon: icon.png - -displayName: Firestore Incremental Backup Stream -description: - Offers a cost-effective, flexible disaster recovery mechanism for Firestore. - -license: Apache-2.0 - -author: - authorName: Google Cloud - url: https://cloud.google.com/ - -contributors: - - authorName: Invertase - email: oss@invertase.io - url: https://github.com/invertase - -sourceUrl: https://github.com/GoogleCloudPlatform/firebase-extensions/tree/main/ -releaseNotesUrl: https://github.com/GoogleCloudPlatform/firebase-extensions/tree/main/ - -apis: - - apiName: eventarc.googleapis.com - reason: Powers all events and triggers - - - apiName: bigquery.googleapis.com - reason: Running queries - - - apiName: dataflow.googleapis.com - reason: Running dataflow jobs - -roles: - - role: datastore.user - reason: Allows the extension to write updates to the database. - - - role: bigquery.dataEditor - reason: Allows the creation of BQ jobs to import Firestore backups. - - # - role: dataflow.developer - # reason: Allows this extension to create and run dataflow jobs. - - # - role: artifactregistry.writer - # reason: Allows this extension to write to the artifact registry. - -billingRequired: true - -resources: - - name: runInitialSetup - type: firebaseextensions.v1beta.function - description: >- - Creates the backup BigQuery database if it does not exist - properties: - availableMemoryMb: 512 - location: ${LOCATION} - runtime: nodejs22 - timeout: 540s - taskQueueTrigger: {} - - - name: syncData - type: firebaseextensions.v1beta.function - description: Enqueues a task to sync data to BigQuery - properties: - runtime: nodejs22 - location: ${LOCATION} - eventTrigger: - eventType: providers/cloud.firestore/eventTypes/document.write - resource: projects/${param:PROJECT_ID}/databases/(default)/documents/${param:SYNC_COLLECTION_PATH}/{docId} - - - name: syncDataTask - type: firebaseextensions.v1beta.function - description: >- - Distributed cloud task for syncing data to BigQuery - properties: - availableMemoryMb: 512 - location: ${LOCATION} - runtime: nodejs22 - timeout: 540s - taskQueueTrigger: {} - - - name: onHttpRunRestoration - type: firebaseextensions.v1beta.function - description: >- - Starts a new restoration task - properties: - location: ${LOCATION} - runtime: nodejs22 - httpsTrigger: {} - - # TODO change to a Firestore trigger - - name: onBackupRestore - type: firebaseextensions.v1beta.function - description: >- - Exports data from storage to a pre-defined Firestore instance. - properties: - location: ${LOCATION} - runtime: nodejs22 - availableMemoryMb: 1024 - taskQueueTrigger: {} - -params: - - param: LOCATION - label: Cloud Functions location - description: >- - Where do you want to deploy the functions created for this extension? You - usually want a location close to your database. For help selecting a - location, refer to the [location selection - guide](https://firebase.google.com/docs/functions/locations). - type: select - options: - - label: Iowa (us-central1) - value: us-central1 - - label: South Carolina (us-east1) - value: us-east1 - - label: Northern Virginia (us-east4) - value: us-east4 - - label: Los Angeles (us-west2) - value: us-west2 - - label: Salt Lake City (us-west3) - value: us-west3 - - label: Las Vegas (us-west4) - value: us-west4 - - label: Warsaw (europe-central2) - value: europe-central2 - - label: Belgium (europe-west1) - value: europe-west1 - - label: London (europe-west2) - value: europe-west2 - - label: Frankfurt (europe-west3) - value: europe-west3 - - label: Zurich (europe-west6) - value: europe-west6 - - label: Taiwan (asia-east1) - value: asia-east1 - - label: Hong Kong (asia-east2) - value: asia-east2 - - label: Tokyo (asia-northeast1) - value: asia-northeast1 - - label: Osaka (asia-northeast2) - value: asia-northeast2 - - label: Seoul (asia-northeast3) - value: asia-northeast3 - - label: Mumbai (asia-south1) - value: asia-south1 - - label: Singapore (asia-southeast1) - value: asia-southeast1 - - label: Jakarta (asia-southeast2) - value: asia-southeast2 - - label: Montreal (northamerica-northeast1) - value: northamerica-northeast1 - - label: Sao Paulo (southamerica-east1) - value: southamerica-east1 - - label: Sydney (australia-southeast1) - value: australia-southeast1 - default: us-central1 - required: true - immutable: true - - - param: SYNC_COLLECTION_PATH - label: Collection path - description: > - What is the path to the collection that contains the strings that you want - to capture all changes of? Use `{document=**}` to capture all collections. - example: users - validationRegex: "^[^/]+(/[^/]+/[^/]+)*$" - validationErrorMessage: Must be a valid Cloud Firestore Collection - required: true - - - param: SYNC_DATASET - label: Bigquery dataset Id - description: > - The id of the Bigquery dataset to sync data to. - example: backup_dataset - default: backup_dataset - validationRegex: "^[a-zA-Z0-9_]+$" - validationErrorMessage: > - BigQuery dataset IDs must be alphanumeric (plus underscores) and must be - no more than 1024 characters. - required: true - - - param: SYNC_TABLE - label: Bigquery table Id - description: > - The id of the Bigquery table to sync data to. - example: backup_table - default: backup_table - required: true - - - param: BACKUP_INSTANCE_ID - label: Backup instance Id - description: > - The name of the Firestore instance to backup the database to. - example: my-backup - validationRegex: "^[a-zA-Z][a-zA-Z0-9-]{2,61}[a-zA-Z0-9]$" - validationErrorMessage: Enter a valid instance id - required: true - -lifecycleEvents: - onInstall: - function: runInitialSetup - processingMessage: Creates the backup BigQuery database if it does not exist - onUpdate: - function: runInitialSetup - processingMessage: Creates the backup BigQuery database if it does not exist - onConfigure: - function: runInitialSetup - processingMessage: Creates the backup BigQuery database if it does not exist diff --git a/kits/firestore-incremental-capture/legacy/functions/.gitignore b/kits/firestore-incremental-capture/legacy/functions/.gitignore deleted file mode 100644 index 65b4c06ec..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -# Compiled JavaScript files -lib/**/*.js -lib/**/*.js.map - -# TypeScript v1 declaration files -typings/ - -# Node.js dependency directory -node_modules/ diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/backupDatabase.test.ts b/kits/firestore-incremental-capture/legacy/functions/__tests__/backupDatabase.test.ts deleted file mode 100644 index 0ec017c7a..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/__tests__/backupDatabase.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { runInitialSetup } from "../src/index"; - -const mockQueue = jest.fn(); - -const getFunctionsMock = () => ({ - taskQueue: (functionName: string, instanceId: string) => ({ - enqueue: (data: any) => { - mockQueue(data); - return Promise.resolve(); - }, - }), -}); - -const mockSetProcessingState = jest.fn(); - -const getExtensionsMock = () => ({ - runtime: () => ({ - setProcessingState: (state: string, message: string) => - mockSetProcessingState(state, message), - }), -}); - -jest.mock("firebase-admin/functions", () => ({ - ...jest.requireActual("firebase-admin/functions"), - getFunctions: () => getFunctionsMock(), -})); - -jest.mock("firebase-admin/extensions", () => ({ - ...jest.requireActual("firebase-admin/extensions"), - getExtensions: () => getExtensionsMock(), -})); - -jest.mock("../src/config", () => ({ - default: { - table: "", - dataset: "", - datasetLocation: "us", - collectionName: "27062023", - runInitialBackup: true, - bucketName: "dev-extensions-testing.appspot.com", - }, -})); - -/** Setup project config */ -process.env.FIRESTORE_EMULATOR_HOST = "127.0.0.1:8080"; -process.env.FIREBASE_FIRESTORE_EMULATOR_ADDRESS = "127.0.0.1:8080"; -process.env.FIREBASE_AUTH_EMULATOR_HOST = "127.0.0.1:9099"; -process.env.PUBSUB_EMULATOR_HOST = "127.0.0.1:8085"; -process.env.GOOGLE_CLOUD_PROJECT = "demo-test"; -process.env.FIREBASE_STORAGE_EMULATOR_HOST = "127.0.0.1:9199"; - -/** Global vars */ - -describe("backupDatabase", () => { - test("Can backup database", async () => { - /** Run the function */ - await runInitialSetup(); - }); -}); diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/firestoreSerializer.test.ts b/kits/firestore-incremental-capture/legacy/functions/__tests__/firestoreSerializer.test.ts deleted file mode 100644 index 3e5783a8d..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/__tests__/firestoreSerializer.test.ts +++ /dev/null @@ -1,418 +0,0 @@ -/** - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as admin from "firebase-admin"; - -const { Timestamp, GeoPoint } = require("@google-cloud/firestore"); - -import { verifySchema } from "./helpers"; - -process.env.FIRESTORE_EMULATOR_HOST = "127.0.0.1:8080"; -process.env.FIREBASE_FIRESTORE_EMULATOR_ADDRESS = "127.0.0.1:8080"; -process.env.FIREBASE_AUTH_EMULATOR_HOST = "127.0.0.1:9099"; -process.env.PUBSUB_EMULATOR_HOST = "127.0.0.1:8085"; -process.env.GOOGLE_CLOUD_PROJECT = "demo-project"; - -admin.initializeApp({ projectId: "demo-project" }); - -const db = admin.firestore(); - -/** - * TODO: Handle binary examples - */ - -describe("generateSchema", () => { - test("should handle an string value", async () => { - const documentPath = "products/stringExample"; - const sampleDocData = { - stringValue: "Hello, Firestore!", - }; - - await db.doc(documentPath).set(sampleDocData); - - await verifySchema(documentPath, { - stringValue: { type: "string", value: "Hello, Firestore!" }, - }); - }, 12000); - - test("should handle an boolean value", async () => { - const documentPath = "products/booleanExample"; - const sampleDocData = { - booleanExample: true, - }; - - await db.doc(documentPath).set(sampleDocData); - - await verifySchema(documentPath, { - booleanExample: { type: "boolean", value: true }, - }); - }, 12000); - - test("should handle a geopoint value", async () => { - const documentPath = "products/geoPointExample"; - const sampleDocData = { - geopointValue: new GeoPoint(52.379189, 4.899431), - }; - - await db.doc(documentPath).set(sampleDocData); - - await verifySchema(documentPath, { - geopointValue: { - type: "geopoint", - value: { - latitude: { - type: "number", - value: 52.379189, - }, - longitude: { - type: "number", - value: 4.899431, - }, - }, - }, - }); - }, 12000); - - test("should handle a document reference value", async () => { - const documentPath = "products/documentReferenceExample"; - const ref = db.doc("products/stringExample"); - const sampleDocData = { - documentReferenceValue: ref, - }; - - await db.doc(documentPath).set(sampleDocData); - - await verifySchema(documentPath, { - documentReferenceValue: { - type: "documentReference", - value: ref.path, // Assuming you want to store the path of the document reference - }, - }); - }, 12000); - - test("should handle a timestamp reference value", async () => { - const documentPath = "products/timestampExample"; - const timestampValue = Timestamp.now(); - const sampleDocData = { - timestampValue, - }; - - await db.doc(documentPath).set(sampleDocData); - - await verifySchema(documentPath, { - timestampValue: { - type: "timestamp", - value: timestampValue.toDate().toISOString(), - }, - }); - }, 12000); - - test("should handle an objectValue value", async () => { - const documentPath = "products/objectValueExample"; - const sampleDocData = { - objectValue: { foo: "bar" }, - }; - - await db.doc(documentPath).set(sampleDocData); - - await verifySchema(documentPath, { - objectValue: { - type: "map", - value: { - foo: { - type: "string", - value: "bar", - }, - }, - }, - }); - }, 12000); - - test("should handle an multiple objectValue values", async () => { - const documentPath = "products/objectValueExample"; - const sampleDocData = { - objectValue: { foo: "bar", foo2: "bar2" }, - }; - - await db.doc(documentPath).set(sampleDocData); - - await verifySchema(documentPath, { - objectValue: { - type: "map", - value: { - foo: { - type: "string", - value: "bar", - }, - foo2: { - type: "string", - value: "bar2", - }, - }, - }, - }); - }, 12000); - - test("should handle an empty array value", async () => { - const documentPath = "products/arrayValueExample"; - const sampleDocData = { - arrayValue: [], - }; - - await db.doc(documentPath).set(sampleDocData); - - await verifySchema(documentPath, { - arrayValue: { - type: "array", - value: [], - }, - }); - }, 12000); - - test("should handle an array with basic data types", async () => { - const documentPath = "products/complexExample"; - - const sampleDocData = { - arrayValue: [ - { - stringValue: "test", - integerValue: 42, - floatValue: 42.42, - booleanValue: true, - nullValue: null, - }, - ], - }; - - await db.doc(documentPath).set(sampleDocData); - - // Define the expected result according to the behavior of the flattenData function. - const expectedData = { - arrayValue: { - type: "array", - value: [ - { - stringValue: { - type: "string", - value: "test", - }, - integerValue: { - type: "number", - value: 42, - }, - floatValue: { - type: "number", - value: 42.42, - }, - booleanValue: { - type: "boolean", - value: true, - }, - nullValue: { - type: "null", - value: null, - }, - }, - ], - }, - }; - - await verifySchema(documentPath, expectedData); - }, 12000); - - test("should handle an array with complex data types", async () => { - const documentPath = "products/complexArrayExample"; - const timestampValue = Timestamp.now(); - - const sampleDocData = { - arrayValue: [ - { - nestedString: "nestedTest", - nestedNumber: 42, - nestedObject: { - deepNestedValue: "deepValue", - }, - geoPointValue: new GeoPoint(52.379189, 4.899431), - timestampValue, - }, - ], - }; - - await db.doc(documentPath).set(sampleDocData); - - // Define the expected result according to the behavior of the flattenData function. - const expectedData = { - arrayValue: { - type: "array", - value: [ - { - nestedString: { - type: "string", - value: "nestedTest", - }, - nestedNumber: { - type: "number", - value: 42, - }, - nestedObject: { - type: "map", - value: { - deepNestedValue: { - type: "string", - value: "deepValue", - }, - }, - }, - geoPointValue: { - type: "geopoint", - value: { - latitude: { - type: "number", - value: 52.379189, - }, - longitude: { - type: "number", - value: 4.899431, - }, - }, - }, - timestampValue: { - type: "timestamp", - value: timestampValue.toDate().toISOString(), - }, - }, - ], - }, - }; - - await verifySchema(documentPath, expectedData); - }, 12000); - - test("should handle arrays with mixed data types", async () => { - const documentPath = "products/mixedArrayExample"; - const sampleDocData = { - mixedArray: ["string", 42, true], - }; - - await db.doc(documentPath).set(sampleDocData); - - await verifySchema(documentPath, { - mixedArray: { - type: "array", - value: [ - { type: "string", value: "string" }, - { type: "number", value: 42 }, - { type: "boolean", value: true }, - ], - }, - }); - }, 12000); - - test("should handle standalone numbers", async () => { - const documentPath = "products/numberExample"; - const sampleDocData = { - numberValue: 42.42, - }; - - await db.doc(documentPath).set(sampleDocData); - - await verifySchema(documentPath, { - numberValue: { - type: "number", - value: 42.42, - }, - }); - }, 12000); - - test("should handle binary blob data", async () => { - const documentPath = "products/blobExample"; - - const sampleDocData = { - blobValue: Buffer.from("some sample data", "utf8"), - }; - - db.doc(documentPath).set(sampleDocData); - await db.doc(documentPath).set(sampleDocData); - - await verifySchema(documentPath, { - blobValue: { - type: "binary", - value: Buffer.from("some sample data").toString("base64"), // Modified this line to directly use a Buffer - }, - }); - }, 12000); - - test("should handle an integer value", async () => { - const documentPath = "products/integerExample"; - - // Sample data with an integer value - const sampleDocData = { - integerValue: 12345, - }; - - // Set the data in Firestore - await db.doc(documentPath).set(sampleDocData); - - // Verify if the value in the schema (or retrieved value) matches the set value - await verifySchema(documentPath, { - integerValue: { - type: "number", - value: 12345, - }, - }); - }, 12000); - - test("should handle a floating point value", async () => { - const documentPath = "products/floatingPointExample"; - - // Sample data with a floating point value - const sampleDocData = { - floatValue: 123.45, - }; - - // Set the data in Firestore - await db.doc(documentPath).set(sampleDocData); - - // Verify if the value in the schema (or retrieved value) matches the set value - await verifySchema(documentPath, { - floatValue: { - type: "number", - value: 123.45, - }, - }); - }, 12000); - - test("should handle a null value", async () => { - const documentPath = "products/nullValueExample"; - const sampleDocData = { - nullableField: null, - }; - - // Set the data in Firestore - await db.doc(documentPath).set(sampleDocData); - - // Prepare the update using the helper function (assuming this is how you're setting up your other tests) - await db.doc(documentPath).set(sampleDocData); - - // Check against the expected schema - await verifySchema(documentPath, { - nullableField: { - type: "null", // null is a type of object in JavaScript - value: null, - }, - }); - }, 12000); -}); diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/functions.test.ts b/kits/firestore-incremental-capture/legacy/functions/__tests__/functions.test.ts deleted file mode 100644 index 0dd80b91c..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/__tests__/functions.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as admin from "firebase-admin"; -import * as functions from "firebase-functions-test"; -import { syncData } from "../src/index"; -import { getTable, initialize } from "../src/bigquery"; - -import config from "../src/config"; -import { Table } from "@google-cloud/bigquery"; -import { clearBQTables } from "./helpers"; - -jest.mock("../src/config", () => ({ - default: { - table: "", - dataset: "", - datasetLocation: "us", - syncCollectionPath: "testing", - }, -})); - -/** Setup project config */ -const projectId = "dev-extensions-testing"; -const fft = functions({ projectId }); - -process.env.FIRESTORE_EMULATOR_HOST = "127.0.0.1:8080"; -process.env.FIREBASE_FIRESTORE_EMULATOR_ADDRESS = "127.0.0.1:8080"; -process.env.FIREBASE_AUTH_EMULATOR_HOST = "127.0.0.1:9099"; -process.env.PUBSUB_EMULATOR_HOST = "127.0.0.1:8085"; -process.env.GOOGLE_CLOUD_PROJECT = "demo-test"; -process.env.FIREBASE_STORAGE_EMULATOR_HOST = "127.0.0.1:9199"; - -/** Global vars */ -const { makeDocumentSnapshot } = fft.firestore; - -const db = admin.firestore(); -const collection = db.collection(config.syncCollectionPath); -let randomId = ""; - -xdescribe("functions", () => { - beforeAll(async () => { - /** clear all datasets */ - await clearBQTables(); - }); - beforeEach(async () => { - /** generate random id */ - randomId = (Math.random() + 1).toString(36).substring(7); - - config.table = randomId; - config.dataset = randomId; - - await initialize(); - }); - - xit("Can sync data with BQ", async () => { - /** Set document data */ - const doc = await collection.add({}); - const path = `${config.syncCollectionPath}/${doc.id}`; - const snap = makeDocumentSnapshot({ foo: "bar" }, path); - - /** Run the function */ - const wrapped = fft.wrap(syncData); - await wrapped(snap); - - /** check data has synced */ - const table: Table = await getTable(config.dataset, config.table); - - /** wait for 2 seconds */ - await new Promise((resolve) => setTimeout(resolve, 2000)); - - const [query] = await table.createQueryJob({ - query: `Select * from ${config.dataset}.${config.table}`, - }); - - const [results] = await query.getQueryResults(); - const { foo } = JSON.parse(results[0].data); - - expect(foo).toEqual("bar"); - }); - - it("Can replay data", async () => { - /** Set document data */ - const doc = await collection.add({}); - const path = `${config.syncCollectionPath}/${doc.id}`; - - /** Make an array of 10 items */ - const snapshots = Array.from(Array(10).keys()); - - /** Write snapshots to the database */ - const wrapped = fft.wrap(syncData); - for await (const snapshot of snapshots) { - /** Run the function */ - const bs = makeDocumentSnapshot({}, path); - const as = makeDocumentSnapshot({ foo: snapshot }, path); - const change = fft.makeChange(bs, as); - await wrapped(change); - } - - /** check data has synced */ - const table: Table = await getTable(config.dataset, config.table); - - const [query] = await table.createQueryJob({ - query: `Select * from ${config.dataset}.${config.table}`, - }); - - const [results] = await query.getQueryResults(); - const $ = JSON.parse(results[0].data); - - expect($).toEqual({ foo: 0 }); - - /** */ - }); -}); diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/helpers.ts b/kits/firestore-incremental-capture/legacy/functions/__tests__/helpers.ts deleted file mode 100644 index 5bfbe4434..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/__tests__/helpers.ts +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { DocumentReference, DocumentSnapshot } from "firebase-admin/firestore"; -import { WrappedFirebaseFunction } from "./types"; -import { FeaturesList } from "firebase-functions-test/lib/features"; - -const { BigQuery } = require("@google-cloud/bigquery"); -const bq = new BigQuery({ projectId: "dev-extensions-testing" }); - -export const simulateFunctionTriggered = - ( - module: FeaturesList, - wrappedFunction: WrappedFirebaseFunction, - collectionName: string - ) => - async (ref: DocumentReference, before?: DocumentSnapshot) => { - const data = (await ref.get()).data() as { [key: string]: any }; - const beforeFunctionExecution = module.firestore.makeDocumentSnapshot( - data, - `${collectionName}/${ref.id}` - ) as DocumentSnapshot; - const change = module.makeChange(before, beforeFunctionExecution); - await wrappedFunction(change); - return beforeFunctionExecution; - }; - -export const clearBQTables = async () => { - const [datasets] = await bq.getDatasets({ - projectId: "dev-extensions-testing", - }); - - for await (const dataset of datasets) { - try { - await dataset.delete({ force: true }); - console.log(`Dataset ${dataset.id} deleted.`); - } catch (ex) { - console.log((ex as Error).message); - } - } -}; diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/backup-test.js b/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/backup-test.js deleted file mode 100644 index ef2d19f39..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/backup-test.js +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const { Spanner } = require("@google-cloud/spanner"); -const { PreciseDate } = require("@google-cloud/precise-date"); - -(async () => { - const projectId = "dev-extensions-testing"; - const instanceId = "my-instance"; - const databaseId = "my-database"; - const backupId = "my-backup"; - const versionTime = Date.now() - 1000 * 60 * 60 * 24; // One day ago - - const spanner = new Spanner({ - projectId: projectId, - }); - - // Gets a reference to a Cloud Spanner instance and database - const instance = spanner.instance(instanceId); - const database = instance.database(databaseId); - - const backup = instance.backup(backupId); - - // Creates a new backup of the database - try { - console.log(`Creating backup of database ${database.formattedName_}.`); - const databasePath = database.formattedName_; - // Expire backup 14 days in the future - const expireTime = Date.now() + 1000 * 60 * 60 * 24 * 14; - // Create a backup of the state of the database at the current time. - const [, operation] = await backup.create({ - databasePath: databasePath, - expireTime: expireTime, - versionTime: versionTime, - }); - - console.log(`Waiting for backup ${backup.formattedName_} to complete...`); - await operation.promise(); - - // Verify backup is ready - const [backupInfo] = await backup.getMetadata(); - if (backupInfo.state === "READY") { - console.log( - `Backup ${backupInfo.name} of size ` + - `${backupInfo.sizeBytes} bytes was created at ` + - `${new PreciseDate(backupInfo.createTime).toISOString()} ` + - "for version of database at " + - `${new PreciseDate(backupInfo.versionTime).toISOString()}` - ); - } else { - console.error("ERROR: Backup is not ready."); - } - } catch (err) { - console.error("ERROR:", err); - } finally { - // Close the database when finished. - await database.close(); - } -})(); diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/createTestData.js b/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/createTestData.js deleted file mode 100644 index 286fe69c5..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/createTestData.js +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const admin = require("firebase-admin"); - -admin.initializeApp({ projectId: "dev-extensions-testing" }); - -/** add a root level collections */ -const rootCollection = admin.firestore().collection("sync"); - -(async () => { - const rootCollectionDocument = await rootCollection.add({ - name: "Sample Document", - description: "This is a sample document for reference.", - }); - - /** Set the reference and Wait 5 seconds */ - const subCollectionRef = rootCollection.doc().collection("subCollection"); - await new Promise((resolve) => setTimeout(resolve, 5000)); - - /** Add a sub collection document */ - let i = "first update"; - - await subCollectionRef.add({ - stringField: `This is a string ${i}`, - numberField: 12345 + i, - booleanField: i % 2 === 0, // Will alternate between true and false - arrayField: ["apple", "banana", "cherry"], - dateField: new Date(), - nullField: null, - objectField: { - subString: `Sub object string ${i}`, - subNumber: 67890 + i, - }, - geopointField: new admin.firestore.GeoPoint(34.0522, -118.2437), // This represents LA latitude and longitude - referenceField: rootCollectionDocument, - }); - - /** Wait 5 seconds */ - await new Promise((resolve) => setTimeout(resolve, 5000)); - - i = "second update"; - - await subCollectionRef.add({ - stringField: `This is a string ${i}`, - numberField: 12345 + i, - booleanField: i % 2 === 0, // Will alternate between true and false - arrayField: ["apple", "banana", "cherry"], - dateField: new Date(), - nullField: null, - objectField: { - subString: `Sub object string ${i}`, - subNumber: 67890 + i, - }, - geopointField: new admin.firestore.GeoPoint(34.0522, -118.2437), // This represents LA latitude and longitude - referenceField: rootCollectionDocument, - }); - - /** Wait 30 seconds */ - await new Promise((resolve) => setTimeout(resolve, 5000)); -})(); diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/exportfromBQ.js b/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/exportfromBQ.js deleted file mode 100644 index 8231920c8..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/__tests__/manualTesting/exportfromBQ.js +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const { BigQuery } = require("@google-cloud/bigquery"); - -const bq = new BigQuery({ projectId: "dev-extensions-testing" }); - -(async () => { - /** Get all the records from before 2023-08-22 13:23 */ - const query = - "SELECT * FROM `dev-extensions-testing.syncData.syncData` WHERE timestamp < TIMESTAMP('2023-08-22 13:23:00')"; - - /** Execute the query */ - await bq.query(query).then((data) => { - console.log(data); - }); -})(); diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/tsconfig.json b/kits/firestore-incremental-capture/legacy/functions/__tests__/tsconfig.json deleted file mode 100644 index 379a994d8..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/__tests__/tsconfig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "../tsconfig.json", - "include": ["."] -} diff --git a/kits/firestore-incremental-capture/legacy/functions/__tests__/types.ts b/kits/firestore-incremental-capture/legacy/functions/__tests__/types.ts deleted file mode 100644 index 655f93a1d..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/__tests__/types.ts +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { DocumentSnapshot } from "firebase-admin/firestore"; -import { WrappedFunction } from "firebase-functions-test/lib/v1"; -import { Change } from "firebase-functions/v1"; - -export type WrappedFirebaseFunction = WrappedFunction< - Change, - void ->; diff --git a/kits/firestore-incremental-capture/legacy/functions/jest.config.js b/kits/firestore-incremental-capture/legacy/functions/jest.config.js deleted file mode 100644 index d936ac9c5..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/jest.config.js +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const packageJson = require("./package.json"); - -module.exports = { - name: packageJson.name, - displayName: packageJson.name, - rootDir: "./", - globals: { - "ts-jest": { - tsConfig: "/__tests__/tsconfig.json", - }, - }, - testMatch: ["**/__tests__/*.test.ts"], - testPathIgnorePatterns: ["manualTesting"], - testEnvironment: "node", - preset: "ts-jest", -}; diff --git a/kits/firestore-incremental-capture/legacy/functions/package.json b/kits/firestore-incremental-capture/legacy/functions/package.json deleted file mode 100644 index f646de75d..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/package.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "name": "functions", - "scripts": { - "prepare": "npm run build", - "lint": "eslint --ext .js,.ts .", - "build": "tsc", - "build:watch": "tsc --watch", - "serve": "npm run build && firebase emulators:start --only functions", - "shell": "npm run build && firebase functions:shell", - "start": "npm run shell", - "deploy": "firebase deploy --only functions", - "logs": "firebase functions:log", - "generate-readme": "firebase ext:info .. --markdown > ../README.md", - "publish-from-main": "firebase ext:dev:upload googlecloud/firestore-incremental-capture --repo=https://github.com/googlecloudplatform/firebase-extensions --root=firestore-incremental-capture --ref=main --project pub-ext-gcloud" - }, - "engines": { - "node": "22" - }, - "main": "lib/index.js", - "dependencies": { - "@google-cloud/bigquery": "^7.1.1", - "@google-cloud/cloudbuild": "^4.0.1", - "@google-cloud/dataflow": "^3.0.1", - "@types/traverse": "^0.6.37", - "firebase-admin": "^12.2.0", - "firebase-functions": "^4.3.1", - "jest": "^29.6.2", - "traverse": "^0.6.11", - "ts-jest": "^29.4.0" - }, - "devDependencies": { - "@google-cloud/firestore": "^7.11.2", - "@google-cloud/precise-date": "^4.0.0", - "@google-cloud/spanner": "^7.0.0", - "@typescript-eslint/eslint-plugin": "^8.54.0", - "@typescript-eslint/parser": "^8.54.0", - "dotenv": "^16.3.1", - "eslint": "^9.39.2", - "eslint-config-google": "^0.14.0", - "eslint-plugin-import": "^2.32.0", - "firebase-functions-test": "^3.4.1", - "typescript": "^4.9.0" - }, - "overrides": { - "fast-xml-parser": "^5.3.4" - }, - "private": true -} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/config.ts b/kits/firestore-incremental-capture/legacy/functions/src/config.ts deleted file mode 100644 index 7ab342130..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/config.ts +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as admin from "firebase-admin"; - -admin.initializeApp(); - -const projectId = process.env.PROJECT_ID!; -const instanceId = process.env.EXT_INSTANCE_ID!; -const location = process.env.LOCATION!; -const backupInstance = process.env.BACKUP_INSTANCE_ID!; -const backupInstanceFullId = `projects/${projectId}/databases/${backupInstance}`; - -const getDefaultBucket = (): string => { - try { - // Try to get the default bucket from Firebase Admin - const defaultBucket = admin.storage().bucket().name; - console.log(`Using detected default bucket: ${defaultBucket}`); - return process.env.BUCKET_NAME || defaultBucket; - } catch (error) { - // Fallback to the environment variable or construct using project ID - console.log( - `Could not detect default bucket, using fallback: ${projectId}.appspot.com` - ); - return process.env.BUCKET_NAME || `${projectId}.appspot.com`; - } -}; - -const bucketName = getDefaultBucket(); - -export { admin }; // Export admin to use in other files - -export default { - projectId, - instanceId, - bucketName, - location, - bucketPath: "backups", - datasetLocation: "us", - runInitialBackup: true, - - instanceCollection: `_ext-${process.env.EXT_INSTANCE_ID!}`, - statusDoc: `_ext-${process.env.EXT_INSTANCE_ID!}/status`, - backupDoc: `_ext-${process.env.EXT_INSTANCE_ID!}/backups`, - restoreDoc: `_ext-${process.env.EXT_INSTANCE_ID!}/restore`, - cloudBuildDoc: `_ext-${process.env.EXT_INSTANCE_ID!}/cloudBuild`, - syncCollectionPath: process.env.SYNC_COLLECTION_PATH!, - - bqDataset: process.env.SYNC_DATASET!, - bqtable: process.env.SYNC_TABLE!, - - backupInstanceName: backupInstanceFullId, - - stagingLocation: `gs://${bucketName}/${instanceId}/staging`, - templateLocation: `gs://${bucketName}/${instanceId}/templates/myTemplate`, - dataflowRegion: - process.env.DATAFLOW_REGION || process.env.LOCATION || "us-central1", -}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/constants/bq_backup_schema.ts b/kits/firestore-incremental-capture/legacy/functions/src/constants/bq_backup_schema.ts deleted file mode 100644 index c1fb6934e..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/constants/bq_backup_schema.ts +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export const bqBackupSchema = [ - { name: "documentId", type: "STRING", mode: "REQUIRED" }, - { name: "documentPath", type: "STRING", mode: "REQUIRED" }, - { name: "beforeData", type: "JSON" }, - { name: "afterData", type: "JSON" }, - { name: "changeType", type: "STRING", mode: "REQUIRED" }, - { name: "timestamp", type: "TIMESTAMP", mode: "REQUIRED" }, -]; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/dataflow/build_flex_template.ts b/kits/firestore-incremental-capture/legacy/functions/src/dataflow/build_flex_template.ts deleted file mode 100644 index c3ac1d932..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/dataflow/build_flex_template.ts +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import config from "../config"; - -import { exec } from "child_process"; - -/** - * This function builds the flex template for the dataflow job, - * but it is not used in the current implementation. - * The reason is that gcloud CLI is not available in the cloud functions runtime, - * hence the build process cannot be automated. - * - * It is included here for reference purposes. - */ -export async function buildFlexTemplateHandler() { - const projectId = config.projectId; - const bucketName = config.bucketName; - const location = config.location; - const instanceId = config.instanceId; - - // Building JAR: mvn clean package -DskipTests -Dexec.mainClass=com.pipeline.RestorationPipeline - - exec( - `gcloud dataflow flex-template build gs://${bucketName}/dataflow-templates/${instanceId} \ - --image-gcr-path "${location}-docker.pkg.dev/${projectId}/${instanceId}/dataflow/restore:latest" \ - --sdk-language "JAVA" \ - --flex-template-base-image JAVA11 \ - --jar "path/to/pipeline.jar" \ - --env FLEX_TEMPLATE_JAVA_MAIN_CLASS="com.pipeline.RestorationPipeline" \ - --project ${projectId}`, - (err, stdout) => { - if (err) { - console.log(err); - Promise.reject(err); - } - - Promise.resolve(stdout); - } - ); -} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/dataflow/cloud_build.ts b/kits/firestore-incremental-capture/legacy/functions/src/dataflow/cloud_build.ts deleted file mode 100644 index 4bfc3d57c..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/dataflow/cloud_build.ts +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { logger } from "firebase-functions/v1"; -import config from "../config"; -import { CloudBuildClient } from "@google-cloud/cloudbuild"; - -const cloneStep = { - name: "gcr.io/cloud-builders/git", - args: [ - "clone", - "https://github.com/GoogleCloudPlatform/firebase-extensions.git", - ], -}; - -const checkoutStep = { - name: "gcr.io/cloud-builders/git", - args: ["checkout", "@invertase/firestore-incremental-capture"], - dir: "firebase-extensions", -}; - -const buildStep = { - name: "maven:3.8.1-openjdk-11", - args: [ - "mvn", - "compile", - "exec:java", - "-Dexec.mainClass=com.pipeline.RestorationPipeline", - `-Dexec.args=--runner=DataflowRunner --project=${config.projectId} --stagingLocation=${config.stagingLocation} --templateLocation=${config.templateLocation} --region=${config.dataflowRegion}`, - ], - dir: "firebase-extensions/firestore-incremental-capture/functions/pipeline", -}; - -// const notifyStep = { -// name: 'gcr.io/cloud-builders/curl', -// entrypoint: 'bash', -// args: [ -// '-c', -// `curl -X POST -H "Content-Type: application/json" -d \'{"status": "$BUILD_STATUS", "build_id": "$BUILD_ID"}\' https://${config.instanceId}/onCloudBuildComplete`, -// ], -// }; - -const client = new CloudBuildClient(); - -/** - * Builds the template for the dataflow pipeline, return the LROperation name - */ -export const stageTemplate = async () => { - logger.info("Staging template"); - const build_id = `${config.instanceId}-dataflow-template-${Date.now()}`; - - const [operation] = await client.createBuild({ - projectId: config.projectId, - build: { - name: build_id, - id: build_id, - steps: [cloneStep, checkoutStep, buildStep], - }, - }); - - logger.info(`Build created: ${operation.name}`); - - if (operation.error) { - throw new Error(operation.error.message); - } - return operation; -}; - -/** - * Regularly ping the import operation to check for completion - */ -export async function WaitForCreateBuildCompletion(name: string) { - logger.log("Checking for create build progress: ", name); - const response = await client.checkCreateBuildProgress(name); - if (!response.done) { - // Wait for 1 minute retrying - await new Promise((resolve) => setTimeout(resolve, 60000)); - /** try again */ - await WaitForCreateBuildCompletion(name); - } - return Promise.resolve(response); -} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/dataflow/on_complete_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/dataflow/on_complete_handler.ts deleted file mode 100644 index 08681ea87..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/dataflow/on_complete_handler.ts +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as admin from "firebase-admin"; -import { logger } from "firebase-functions/v1"; - -import config from "../config"; - -export async function onCompleteHandler(payload: any) { - logger.info("build event completed!"); - logger.info(`Message ===> ${JSON.stringify(payload)}`); - - await admin - .firestore() - .doc(config.cloudBuildDoc) - .update({ status: "staged", ...payload }); -} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/dataflow/trigger_dataflow_job.ts b/kits/firestore-incremental-capture/legacy/functions/src/dataflow/trigger_dataflow_job.ts deleted file mode 100644 index d32016fff..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/dataflow/trigger_dataflow_job.ts +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as admin from "firebase-admin"; -import { logger } from "firebase-functions/v1"; -import { FlexTemplatesServiceClient } from "@google-cloud/dataflow"; -import { Timestamp } from "firebase-admin/firestore"; - -import config from "../config"; - -const dataflowClient = new FlexTemplatesServiceClient(); - -export async function launchJob(timestamp: number) { - const projectId = config.projectId; - const serverTimestamp = Timestamp.now().toMillis(); - const { syncCollectionPath } = config; - - const runId = `${config.instanceId}-dataflow-run-${serverTimestamp}`; - - logger.info(`Launching job ${runId}`, { - labels: { run_id: runId }, - }); - - const runDoc = admin.firestore().doc(`restore/${runId}`); - - // Extract the database name from the backup instance name - const values = config.backupInstanceName.split("/"); - const firestoreDb = values[values.length - 1]; - - /** Select the correct collection Id for apache beam */ - const firestoreCollectionId = - syncCollectionPath === "{document=**}" ? "*" : syncCollectionPath; - - const [response] = await dataflowClient.launchFlexTemplate({ - projectId, - location: config.location, - launchParameter: { - jobName: runId, - parameters: { - timestamp: timestamp.toString(), - firestoreCollectionId, - firestoreDb, - bigQueryDataset: config.bqDataset, - bigQueryTable: config.bqtable, - }, - containerSpecGcsPath: `gs://${config.bucketName}/${config.instanceId}-dataflow-restore`, - }, - }); - - await runDoc.set({ status: "export triggered", runId: runId }); - - logger.info(`Launched job named ${response.job?.name} successfully`, { - job_response: response, - labels: { run_id: runId }, - }); - - return response; -} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/index.ts b/kits/firestore-incremental-capture/legacy/functions/src/index.ts deleted file mode 100644 index dc1045424..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/index.ts +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as functions from "firebase-functions"; - -import config from "./config"; - -import { syncDataHandler } from "./tasks/on_sync_data_handler"; -import { onCompleteHandler } from "./dataflow/on_complete_handler"; -import { syncDataTaskHandler } from "./tasks/sync_data_task_handler"; -import { buildFlexTemplateHandler } from "./dataflow/build_flex_template"; -import { onBackupRestoreHandler } from "./tasks/on_backup_restore_handler"; -import { runInitialSetupHandler } from "./tasks/on_run_initial_setup_handler"; -import { onHttpRunRestorationHandler } from "./tasks/on_http_run_restoration_handler"; -import { onFirestoreBackupInitHandler } from "./tasks/on_firestore_backup_init_handler"; - -/** - * Sync data to BigQuery, triggered by any change to a Firestore document - * */ -export const syncData = functions.firestore - .document(config.syncCollectionPath) - .onWrite(syncDataHandler); - -/** - * Cloud task to handle data sync - * */ -export const syncDataTask = functions.tasks - .taskQueue() - .onDispatch(syncDataTaskHandler); - -/** - * Backup the entire database on initial deployment - * */ -export const runInitialSetup = async () => await runInitialSetupHandler(); - -/** - * Run a backup restoration. - * */ -export const onHttpRunRestoration = functions.https.onRequest( - onHttpRunRestorationHandler -); - -export const onBackupRestore = functions.tasks - .taskQueue() - .onDispatch(onBackupRestoreHandler); - -/** - * Cloud task for handling database restoration - * */ -export const onFirestoreBackupInit = functions.tasks - .taskQueue() - .onDispatch(onFirestoreBackupInitHandler); - -/** - * Cloud task for staging the dataflow template - * */ -export const buildFlexTemplate = functions.tasks - .taskQueue() - .onDispatch(buildFlexTemplateHandler); - -export const onCloudBuildComplete = - functions.https.onRequest(onCompleteHandler); diff --git a/kits/firestore-incremental-capture/legacy/functions/src/logs.ts b/kits/firestore-incremental-capture/legacy/functions/src/logs.ts deleted file mode 100644 index 7cc69c452..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/logs.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { logger } from "firebase-functions"; - -export const bigQueryDatasetExists = (dataset: string) => { - logger.log(`${dataset} already exists`); -}; - -export const bigQueryTableExists = (dataset: string) => { - logger.log(`${dataset} already exists`); -}; - -export const bigQueryDatasetCreating = (dataset: string) => { - logger.log(`Creating dataset: ${dataset}`); -}; - -export const bigQueryTableCreating = (table: string) => { - logger.log(`Creating table: ${table}`); -}; - -export const bigQueryDatasetCreated = (dataset: string) => { - logger.log(`successfully created dataset: ${dataset}`); -}; - -export const bigQueryTableCreated = (table: string) => { - logger.log(`successfully created table: ${table}`); -}; - -export const tableCreationError = (table: string, message: string) => { - logger.log(`error creating table: ${table}, ${message}`); -}; - -export const datasetCeationError = (dataset: string) => { - logger.log(`error creatign dataset: ${dataset}`); -}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_backup_restore_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_backup_restore_handler.ts deleted file mode 100644 index 75941ca70..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_backup_restore_handler.ts +++ /dev/null @@ -1,143 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { logger } from "firebase-functions/v1"; - -import { launchJob } from "../dataflow/trigger_dataflow_job"; - -export const onBackupRestoreHandler = async (data: any) => { - const timestamp = data.timestamp as number; - - if (!isValidUnixTimestamp(timestamp)) { - logger.error( - '"timestamp" field is missing, please ensure that you are sending a valid timestamp in the request body, is in seconds since epoch and is not in the future.' - ); - return Promise.resolve(); - } - - logger.info(`Running backup restoration at PIT ${timestamp}`); - - // const importDoc = await db - // .doc(config.backupDoc) - // .collection('imports') - // .add({}); - - // // Get the latest backup collection - // const backupExportsCollection = db - // .doc(config.backupDoc) - // .collection('exports'); - - // const completedExports = await backupExportsCollection - // .where('status', '==', 'Completed') - // .get(); - - // const documents = completedExports.docs.map(doc => ({ - // id: doc.id, - // data: doc.data(), - // })); - - // // Sort documents by timestamp in descending order - // const sortedDocuments = documents.sort( - // (a, b) => b.data.timestamp.toDate() - a.data.timestamp.toDate() - // ); - - // Get the most recent document - // const doc = sortedDocuments.length > 0 ? sortedDocuments[0] : null; - - //TODO: use this version in the future, index creation is needed. - // const backupDocuments = await db - // .doc(config.backupDoc) - // .collection('exports') - // .where('status', '==', 'Completed') - // .orderBy('timestamp', 'desc') - // .limit(1) - // .get(); - - // Get the latest backup - // const backupId = doc?.id; - - // // If no backup - // if (!backupId) { - // logger.info('No backup found'); - // return Promise.resolve(); - // } - - try { - // // Export the Firestore db to storage - // const {id, operation} = await createImport(backupId); - - // // Update Firestore for tracking - // await importDoc.set({ - // id, - // status: 'Running import...', - // operation: operation.name, - // timestamp: FieldValue.serverTimestamp(), - // }); - - // // Wait for import completion - // await waitForImportCompletion(operation.name || ''); - - // await importDoc.set({ - // id, - // status: 'Initial backup restored, replaying final updates...', - // operation: operation.name, - // timestamp: FieldValue.serverTimestamp(), - // }); - - // Run DataFLow updates - await launchJob(timestamp); - - // await importDoc.set({ - // id, - // status: 'Completed', - // operation: operation.name, - // timestamp: FieldValue.serverTimestamp(), - // }); - } catch (ex: any) { - logger.error("Error restoring backup", ex); - - // await db.doc(config.backupDoc).collection('exports').add({ - // error: ex.message, - // status: 'Failed', - // timestamp: FieldValue.serverTimestamp(), - // }); - - return Promise.resolve(); - } -}; - -/** - * Checks if a long integer is a valid UNIX timestamp in seconds. - * - * @param timestamp The timestamp to check. - * @returns Whether the timestamp is valid. - */ -function isValidUnixTimestamp(timestamp: number): boolean { - // Ensure it's a non-negative integer - if (!timestamp || timestamp < 0 || !Number.isInteger(timestamp)) { - return false; - } - - // Get the current UNIX timestamp - const currentTimestamp: number = Math.floor(Date.now() / 1000); - - // Ensure the timestamp isn't in the future - if (timestamp > currentTimestamp) { - return false; - } - - return true; -} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_firestore_backup_init_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_firestore_backup_init_handler.ts deleted file mode 100644 index a2bb7648e..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_firestore_backup_init_handler.ts +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { getExtensions } from "firebase-admin/extensions"; - -import { logger } from "firebase-functions/v1"; -import { updateBackup, updateStatus } from "../utils/database"; - -import { waitForExportCompletion } from "../utils/import_export"; -import { FieldValue } from "firebase-admin/firestore"; - -export const onFirestoreBackupInitHandler = async (data: any) => { - const { id, name } = data; - const runtime = getExtensions().runtime(); - - // Update the status - await runtime.setProcessingState( - "NONE", - "Waiting for the export to be completed" - ); - - try { - // Update the Firestore status - await updateStatus(id, { - status: "Exporting initial backup", - }); - - // Start polling for updates - await waitForExportCompletion(name); - - // Set status to completed - await updateStatus(id, { - status: "Completed", - }); - - // Update the current backup - updateBackup(id, { - status: "Completed", - timestamp: FieldValue.serverTimestamp(), - }); - - // Update the status - await runtime.setProcessingState( - "PROCESSING_COMPLETE", - "Successfully backed up to Firestore" - ); - } catch (ex: any) { - logger.error("Error backing up to BQ", ex); - - await updateStatus(id, { - status: "Error", - error: ex.message, - }); - - await runtime.setProcessingState( - "PROCESSING_FAILED", - "Error backing up to Firestore" - ); - - return Promise.resolve(); - } - - return Promise.resolve(); -}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_http_run_restoration_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_http_run_restoration_handler.ts deleted file mode 100644 index ad02288e1..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_http_run_restoration_handler.ts +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { getFunctions } from "firebase-admin/functions"; -import { Request, Response, logger } from "firebase-functions/v1"; - -import config from "../config"; - -export const onHttpRunRestorationHandler = async ( - request: Request, - response: Response -) => { - const timestamp = request.body.timestamp; - if (!timestamp) { - logger.error( - '"timestamp" field is missing, please ensure that you are sending a valid timestamp in the request body' - ); - return Promise.resolve(); - } - - const now = new Date().getTime(); - - if (timestamp >= now) { - logger.error("The timestamp is in the future, aborting"); - return Promise.resolve(); - } - - const taskName = `projects/${config.projectId}/locations/${config.location}/functions/onBackupRestore`; - - const queue = getFunctions().taskQueue(taskName, config.instanceId); - - logger.log( - `Enqueuing task ${taskName} with timestamp ${timestamp}`, - request.body - ); - - // Queue a restoration task - await queue.enqueue(request.body); - response.status(200).send("Restoration task enqueued"); -}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_initial_setup_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_initial_setup_handler.ts deleted file mode 100644 index c4698f77a..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_initial_setup_handler.ts +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { getExtensions } from "firebase-admin/extensions"; - -import config from "../config"; - -import { initialize } from "../utils/big_query"; -import { bqBackupSchema } from "../constants/bq_backup_schema"; - -export async function runInitialSetupHandler() { - // Setup runtime - const runtime = getExtensions().runtime(); - - await runtime.setProcessingState( - "NONE", - `Creating/updating dataset and table ${config.bqDataset}.${config.bqtable}` - ); - - // Setup sync dataset and tables - const [syncDataset, syncTable] = await initialize( - config.bqDataset, - config.bqtable, - bqBackupSchema - ); - - return runtime.setProcessingState( - "PROCESSING_COMPLETE", - `Initialized dataset and table ${syncDataset.id}.${syncTable.id}` - ); -} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_restoration_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_restoration_handler.ts deleted file mode 100644 index edab916f5..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_run_restoration_handler.ts +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { getFunctions } from "firebase-admin/functions"; - -import config from "../config"; -import { onBackupRestore } from "../index"; - -export const onRunRestorationHandler = async () => { - const queue = getFunctions().taskQueue( - `locations/${config.location}/functions/${onBackupRestore.name}`, - config.instanceId - ); - - // Queue a restoration task - return queue.enqueue({}); -}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_sync_data_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_sync_data_handler.ts deleted file mode 100644 index 387736e9e..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/tasks/on_sync_data_handler.ts +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as functions from "firebase-functions"; -import { getFunctions } from "firebase-admin/functions"; - -import config from "../config"; -import { firestoreSerializer } from "../utils/firestore_serializer"; - -const getState = ( - change: functions.Change -) => { - // return if created - if (!change.before?.exists) return "CREATE"; - - // return if deleted - if (!change.after?.exists) return "DELETE"; - - //else return updated - return "UPDATE"; -}; - -export const syncDataHandler = async ( - change: functions.Change, - ctx: functions.EventContext -) => { - const queue = getFunctions().taskQueue( - `locations/${config.location}/functions/syncDataTask`, - config.instanceId - ); - - //state whether the update is an CREATE, UPDATE or DELETE - const changeType = getState(change); - - // format data - const beforeData = change.before ? change.before.data() : null; - const afterData = change.after ? change.after.data() : null; - - // serialize data - const serializedBeforeData = await firestoreSerializer(beforeData); - const serializedAfterData = await firestoreSerializer(afterData); - - return queue.enqueue({ - beforeData: JSON.stringify(serializedBeforeData), - afterData: JSON.stringify(serializedAfterData), - documentId: change.before?.id || change.after.id, - documentPath: change.before?.ref?.path || change.after.ref.path, - timestamp: ctx.timestamp, - changeType, - }); -}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/tasks/sync_data_task_handler.ts b/kits/firestore-incremental-capture/legacy/functions/src/tasks/sync_data_task_handler.ts deleted file mode 100644 index b5d619282..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/tasks/sync_data_task_handler.ts +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { logger } from "firebase-functions"; - -import config from "../config"; -import { getTable } from "../utils/big_query"; - -export async function syncDataTaskHandler( - data: Record -): Promise { - const table = await getTable(config.bqDataset, config.bqtable); - - // Write the data to the database - await table.insert(data).catch((ex: any) => { - for (const error of ex.errors) { - for (const err of error.errors) { - logger.error(err); - } - } - }); -} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/utils/big_query.ts b/kits/firestore-incremental-capture/legacy/functions/src/utils/big_query.ts deleted file mode 100644 index 8bb7a3138..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/utils/big_query.ts +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { storage } from "firebase-admin"; -import { BigQuery, Dataset } from "@google-cloud/bigquery"; - -import config from "../config"; -import * as logs from "../logs"; - -const bq = new BigQuery({ projectId: config.projectId }); - -function bigqueryDataset(databaseId: string) { - return bq.dataset(databaseId, { - location: config.datasetLocation, - }); -} - -async function initializeDataset(databaseId: string) { - let dataset: Dataset = bigqueryDataset(databaseId); - const [datasetExists] = await dataset.exists(); - - if (datasetExists) { - logs.bigQueryDatasetExists(databaseId); - return dataset; - } - - /** Create table if does not exisst */ - try { - logs.bigQueryDatasetCreating(databaseId); - [dataset] = await bq.createDataset(databaseId, { - location: config.datasetLocation, - }); - logs.bigQueryDatasetCreated(databaseId); - return dataset; - } catch (ex: any) { - logs.datasetCeationError(databaseId); - return dataset; - } -} - -async function initializeTable( - databaseId: string, - tableId: string, - schema: Record[] | null = null -) { - let table; - const dataset: Dataset = bigqueryDataset(databaseId); - table = dataset.table(tableId); - const [tableExists] = await table.exists(); - - /** Return if table exists */ - logs.bigQueryTableExists(tableId); - if (tableExists) return table; - - /** Create a new table and return */ - try { - logs.bigQueryTableCreating(tableId); - - if (!dataset.id || !schema) - throw new Error("Dataset ID and schema must not be undefined"); - - /** - * TODO: Add time partitioning - * TODO: Include expirationMs for partitioning based on config - */ - [table] = await bq.dataset(dataset.id).createTable(tableId, { - schema, - location: config.datasetLocation, - }); - - logs.bigQueryTableCreated(tableId); - return table; - } catch (ex: any) { - logs.tableCreationError(config.bqDataset, ex.message); - return dataset; - } -} - -export async function initialize( - databaseId: string, - tableId: string, - schema: Record[] | null = null -) { - const dataset = await initializeDataset(databaseId); - const table = await initializeTable(databaseId, tableId, schema); - - return [dataset, table]; -} - -export async function getTable(datasetId: string, tableId: string) { - return bq.dataset(datasetId).table(tableId); -} - -/** - * Export the Firestore db to storage - * TODO: This may now be obsolete. We can restore a database, and then replay the data through dataflow. - */ -export const exportToBQ = async (id: string) => { - const name = config.instanceId; - const filename = `${config.bucketPath}/${id}/all_namespaces/kind_${name}/all_namespaces_kind_${name}.export_metadata`; - const bucket = storage().bucket(`gs://${config.bucketName}`); - const file = bucket.file(filename); - - /** - * writeDisposition to overwire the table, if exists - */ - - return bq.dataset(config.bqDataset).table(config.bqtable).load(file, { - writeDisposition: "WRITE_TRUNCATE", - }); -}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/utils/database.ts b/kits/firestore-incremental-capture/legacy/functions/src/utils/database.ts deleted file mode 100644 index dfc6f6997..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/utils/database.ts +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as admin from "firebase-admin"; -import config from "../config"; -import { logger } from "firebase-functions/v1"; - -export const updateStatus = (id: string, data: any) => { - // log update - logger.info(`Updating status for ${id}`, data); - - // Get the backup collection document - const document = admin.firestore().doc(config.statusDoc); - - // Update the document - return document.set({ ...data }, { merge: true }); -}; - -export const updateBackup = (id: string, data: any) => { - // log update - logger.info(`Updating backup for ${id}`, data); - - // Get the backup collection document - const document = admin - .firestore() - .doc(config.backupDoc) - .collection("exports") - .doc(`${id}`); - - // Update the document - return document.set({ ...data }, { merge: true }); -}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/utils/firestore_serializer.ts b/kits/firestore-incremental-capture/legacy/functions/src/utils/firestore_serializer.ts deleted file mode 100644 index 44efc9c89..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/utils/firestore_serializer.ts +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as traverse from "traverse"; - -import { - DocumentReference, - GeoPoint, - Timestamp, -} from "firebase-admin/firestore"; - -export const firestoreSerializer = (data: any) => { - return traverse(data).reduce(function (acc, property) { - if (this.isRoot) return acc; - - if (Buffer.isBuffer(property)) { - if (this.key) { - acc[this.key] = { - type: "binary", - value: property.toString("base64"), - }; - } - - this.delete(true); - return acc; - } - - /** Handle array types */ - if (Array.isArray(property)) { - if (this.key) - acc[this.key] = { - type: "array", - value: property.map((item) => { - // If the item is a primitive, return the serialized format - if (typeof item !== "object" || item === null) { - return { type: typeof item, value: item }; - } - // If the item is an object (including array), recursively serialize it - return firestoreSerializer(item); - }), - }; - this.delete(true); - return acc; - } - - // Handle GeoPoint special type - if (property instanceof GeoPoint) { - if (this.key) - acc[this.key] = { - type: "geopoint", - value: { - latitude: { - type: "number", - value: property.latitude, - }, - longitude: { - type: "number", - value: property.longitude, - }, - }, - }; - this.delete(true); // Delete this node and halt further traversal for its children - return acc; - } - - // Handle Timestamp special type - if (property instanceof Timestamp) { - const date = property.toDate(); // Convert Timestamp to JavaScript Date - if (this.key) - acc[this.key] = { - type: "timestamp", - value: date.toISOString(), // Convert Date to ISO string - }; - this.delete(true); - return acc; - } - - // Handle DocumentReference special type - if (property instanceof DocumentReference) { - if (this.key) - acc[this.key] = { - type: "documentReference", - value: property.path, // Assuming DocumentReference has a 'path' property - }; - this.delete(true); - return acc; - } - - if (property === null) { - if (this.key) - acc[this.key] = { - type: "null", // Set the type as 'null' - value: null, - }; - return acc; - } - - // Handle object type nodes - if (!this.isLeaf) { - /** Handle array types */ - if (Array.isArray(property)) { - if (this.key) - acc[this.key] = { - type: "array", - value: property.map((item) => firestoreSerializer(item)), - }; - this.delete(true); - return acc; - } - - // If it's an object but not a special type, serialize it - else if (typeof property === "object" && property !== null) { - if (this.key) - acc[this.key] = { - type: "map", - value: firestoreSerializer(property), // Recursive serialization - }; - this.delete(true); - return acc; - } - - return acc; - } - - // Decide the accumulator context based on the parent node type - const context = - this.parent?.node && this.parent.node.type === "object" - ? this.parent?.node.value - : acc; - - if (this.key) - context[this.key] = { type: typeof property, value: property }; - - return acc; - }, {}); -}; diff --git a/kits/firestore-incremental-capture/legacy/functions/src/utils/import_export.ts b/kits/firestore-incremental-capture/legacy/functions/src/utils/import_export.ts deleted file mode 100644 index 58de329d8..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/utils/import_export.ts +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// eslint-disable-next-line node/no-unpublished-import -import * as firestore from "@google-cloud/firestore"; -import config from "../config"; -import { logger } from "firebase-functions/v1"; - -const client = new firestore.v1.FirestoreAdminClient({ - projectId: config.projectId, -}); - -/** - * Regularly ping the export operation to check for completion - */ -export async function waitForExportCompletion(name: string) { - logger.log("Checking for export progress: ", name); - const response = await client.checkExportDocumentsProgress(name); - if (!response.done) { - // Wait for 1 minute retrying - await new Promise((resolve) => setTimeout(resolve, 60000)); - //try again - await waitForExportCompletion(name); - } - return Promise.resolve(response); -} - -/** - * Regularly ping the import operation to check for completion - */ -export async function waitForImportCompletion(name: string) { - logger.log("Checking for import progress: ", name); - const response = await client.checkImportDocumentsProgress(name); - if (!response.done) { - // Wait for 1 minute retrying - await new Promise((resolve) => setTimeout(resolve, 60000)); - // try again - await waitForImportCompletion(name); - } - return Promise.resolve(response); -} - -/** - * Imports data from GCS to the specified Firestore backup instance - */ -export async function createImport(id: string) { - const { projectId, syncCollectionPath, bucketName } = config; - - // Extract the database name from the backup instance name - const values = config.backupInstanceName.split("/"); - const database = values[values.length - 1]; - - const name = client.databasePath(projectId, database); - - // Start backup - const [operation] = await client.importDocuments({ - name, - inputUriPrefix: `gs://${bucketName}/backups/${id}`, - collectionIds: - syncCollectionPath === "**" ? [] : syncCollectionPath.split(","), - }); - - return { id, operation }; -} diff --git a/kits/firestore-incremental-capture/legacy/functions/src/utils/serialize.ts b/kits/firestore-incremental-capture/legacy/functions/src/utils/serialize.ts deleted file mode 100644 index e2146b1c2..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/src/utils/serialize.ts +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { DocumentReference } from "firebase-admin/firestore"; - -import * as traverse from "traverse"; - -export const serializeData = (eventData: any) => { - if (typeof eventData === "undefined") { - return undefined; - } - - const data = traverse>(eventData).map(function ( - property: any - ) { - if (property && property.constructor) { - if (property.constructor.name === "Buffer") { - this.remove(); - } - - if (property.constructor.name === DocumentReference.name) { - this.update(property.path); - } - } - }); - - return data; -}; diff --git a/kits/firestore-incremental-capture/legacy/functions/tsconfig.dev.json b/kits/firestore-incremental-capture/legacy/functions/tsconfig.dev.json deleted file mode 100644 index c0f990d78..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/tsconfig.dev.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "include": [".eslintrc.js"] -} diff --git a/kits/firestore-incremental-capture/legacy/functions/tsconfig.json b/kits/firestore-incremental-capture/legacy/functions/tsconfig.json deleted file mode 100644 index 2e24641f7..000000000 --- a/kits/firestore-incremental-capture/legacy/functions/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "compilerOptions": { - "module": "commonjs", - "noImplicitReturns": true, - "noUnusedLocals": false, - "outDir": "lib", - "skipLibCheck": true, - "sourceMap": true, - "strict": true, - "target": "es2017" - }, - "compileOnSave": true, - "include": ["src"] -} diff --git a/kits/firestore-incremental-capture/legacy/install/functions/build_dataflow_template.sh b/kits/firestore-incremental-capture/legacy/install/functions/build_dataflow_template.sh deleted file mode 100644 index bcaa9d834..000000000 --- a/kits/firestore-incremental-capture/legacy/install/functions/build_dataflow_template.sh +++ /dev/null @@ -1,14 +0,0 @@ -echo -e "${YELLOW}Step 6: Building Dataflow Flex Template...${NC}" -if gcloud dataflow flex-template build gs://$BUCKET_NAME/$EXT_INSTANCE_ID-dataflow-restore \ - --image-gcr-path $LOCATION-docker.pkg.dev/$PROJECT_ID/$EXT_INSTANCE_ID/dataflow/restore:latest \ - --sdk-language JAVA \ - --flex-template-base-image JAVA11 \ - --jar $JAR_PATH \ - --env FLEX_TEMPLATE_JAVA_MAIN_CLASS="com.pipeline.RestorationPipeline" \ - --project $PROJECT_ID; then - echo -e "${GREEN}Dataflow Flex Template built successfully.${NC}" - SUCCESS_TASKS+=("${GREEN}${TICK} Dataflow Flex Template built successfully.") -else - echo -e "${RED}Failed to build Dataflow Flex Template.${NC}" - FAILED_TASKS+=("${RED}${CROSS} Failed to build Dataflow Flex Template.") -fi \ No newline at end of file diff --git a/kits/firestore-incremental-capture/legacy/install/functions/download_restore_firestore.sh b/kits/firestore-incremental-capture/legacy/install/functions/download_restore_firestore.sh deleted file mode 100644 index 640094ec7..000000000 --- a/kits/firestore-incremental-capture/legacy/install/functions/download_restore_firestore.sh +++ /dev/null @@ -1,42 +0,0 @@ -echo -e "${YELLOW}Downloading the JAR file...${NC}" - -# Use the correct URL -if curl -L -o restore-firestore.jar "https://github.com/GoogleCloudPlatform/firebase-extensions/raw/main/firestore-incremental-capture-pipeline/target/restore-firestore.jar"; then - # Check if the file is actually a JAR and not HTML - if file restore-firestore.jar | grep -q "HTML"; then - echo -e "${YELLOW}The downloaded file appears to be an HTML page, not a JAR file. The file may not exist at that location.${NC}" - - # Try alternative sources - echo -e "${YELLOW}Trying alternative locations...${NC}" - - # Alternative 1: Try Firebase Extensions GitHub repo directly - if curl -L -o restore-firestore.jar "https://github.com/firebase/extensions/raw/main/firestore-incremental-capture-pipeline/target/restore-firestore.jar"; then - if ! file restore-firestore.jar | grep -q "HTML"; then - echo -e "${GREEN}JAR file downloaded successfully from alternative location.${NC}" - SUCCESS_TASKS+=("${GREEN}${TICK} Successfully downloaded assets.") - exit 0 - fi - fi - - # Alternative 2: Try checking Google Cloud Storage - echo -e "${YELLOW}Trying to download from Cloud Storage...${NC}" - if gcloud storage cp gs://firebase-preview-drop/extension-builds/firestore-incremental-capture/restore-firestore.jar ./restore-firestore.jar 2>/dev/null; then - echo -e "${GREEN}JAR file downloaded successfully from Cloud Storage.${NC}" - SUCCESS_TASKS+=("${GREEN}${TICK} Successfully downloaded assets.") - exit 0 - fi - - # If all attempts fail - echo -e "${RED}Failed to download a valid JAR file from all known locations.${NC}" - echo -e "${YELLOW}You may need to build the JAR from source or contact Firebase support.${NC}" - FAILED_TASKS+=("${RED}${CROSS} Failed to download assets.") - exit 1 - else - echo -e "${GREEN}JAR file downloaded successfully.${NC}" - SUCCESS_TASKS+=("${GREEN}${TICK} Successfully downloaded assets.") - fi -else - echo -e "${RED}Failed to download JAR file.${NC}" - FAILED_TASKS+=("${RED}${CROSS} Failed to download assets.") - exit 1 -fi \ No newline at end of file diff --git a/kits/firestore-incremental-capture/legacy/install/functions/enable_pitr.sh b/kits/firestore-incremental-capture/legacy/install/functions/enable_pitr.sh deleted file mode 100644 index d60033560..000000000 --- a/kits/firestore-incremental-capture/legacy/install/functions/enable_pitr.sh +++ /dev/null @@ -1,9 +0,0 @@ -echo -e "${YELLOW}Step 1: Enabling PITR as per Google Cloud Console guide${NC}" - -if gcloud alpha firestore databases update --project=$PROJECT_ID --enable-pitr; then - echo -e "${GREEN}PITR enabled successfully on (default) database.${NC}" - SUCCESS_TASKS+=("${GREEN}${TICK} Enabled PiTR on (default) database.") -else - echo -e "${RED}Failed to enable PITR on (default) database.${NC}" - FAILED_TASKS+=("${RED}${CROSS} Failed to enable PiTR on (default) database.") -fi \ No newline at end of file diff --git a/kits/firestore-incremental-capture/legacy/install/functions/setup_artifact_registry.sh b/kits/firestore-incremental-capture/legacy/install/functions/setup_artifact_registry.sh deleted file mode 100644 index 5b26e89db..000000000 --- a/kits/firestore-incremental-capture/legacy/install/functions/setup_artifact_registry.sh +++ /dev/null @@ -1,18 +0,0 @@ -# Configure Artifact Registry -echo -e "${YELLOW}Step 3: Configuring Artifact Registry...${NC}" - -ARTIFACT_EXISTS=$(gcloud artifacts repositories list --location=$LOCATION --project=$PROJECT_ID --format="value(name)") - -if echo "$ARTIFACT_EXISTS" | grep -q "$EXT_INSTANCE_ID"; then - echo -e "${YELLOW}Artifact Registry already exists, skipping creation.${NC}" - SUCCESS_TASKS+=("${GREEN}${TICK} Artifact Registry already exists, configuration skipped.") -else - if gcloud artifacts repositories create $EXT_INSTANCE_ID --repository-format=docker --location=$LOCATION --project=$PROJECT_ID --async && \ - gcloud auth configure-docker $LOCATION-docker.pkg.dev; then - echo -e "${GREEN}Artifact Registry configured successfully.${NC}" - SUCCESS_TASKS+=("${GREEN}${TICK} Artifact Registry configured successfully.") - else - echo -e "${RED}Failed to configure Artifact Registry.${NC}" - FAILED_TASKS+=("${RED}${CROSS} Failed to configure Artifact Registry.") - fi -fi \ No newline at end of file diff --git a/kits/firestore-incremental-capture/legacy/install/functions/setup_firestore.sh b/kits/firestore-incremental-capture/legacy/install/functions/setup_firestore.sh deleted file mode 100644 index c80c5feb9..000000000 --- a/kits/firestore-incremental-capture/legacy/install/functions/setup_firestore.sh +++ /dev/null @@ -1,18 +0,0 @@ -# Check if Firestore database already exists -echo -e "${YELLOW}Step 2: Setting up Firestore database${NC}" -DB_EXISTS=$(gcloud alpha firestore databases list --project=$PROJECT_ID --format="value(name)") - -if echo "$DB_EXISTS" | grep -q "projects/$PROJECT_ID/databases/$DATABASE_ID"; then - echo -e "${GREEN}Firestore database already exists, skipping creation.${NC}" - SUCCESS_TASKS+=("${GREEN}${TICK} Database already exists, setup skipped.") -else - # Create secondary Firestore database - echo -e "${YELLOW}Creating secondary Firestore database...${NC}" - if gcloud alpha firestore databases create --database=$DATABASE_ID --location=$DATABASE_LOCATION --type=firestore-native --project=$PROJECT_ID; then - echo -e "${GREEN}Firestore database created successfully.${NC}" - SUCCESS_TASKS+=("${GREEN}${TICK} Database created successfully.") - else - echo -e "${RED}Failed to create Firestore database.${NC}" - FAILED_TASKS+=("${RED}${CROSS} Failed to create Firestore database.") - fi -fi \ No newline at end of file diff --git a/kits/firestore-incremental-capture/legacy/install/functions/setup_service_account.sh b/kits/firestore-incremental-capture/legacy/install/functions/setup_service_account.sh deleted file mode 100644 index 445f21099..000000000 --- a/kits/firestore-incremental-capture/legacy/install/functions/setup_service_account.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# Find extension's service account email -echo -e "${YELLOW}Finding extension's service account email...${NC}" - -# Get all service accounts that match our filter -SA_EMAILS=$(gcloud iam service-accounts list --format="value(EMAIL)" --filter="displayName~'Firebase Extensions $EXT_INSTANCE_ID service account' AND DISABLED=False" --project="$PROJECT_ID") - -# Check if we found any service accounts -if [ -z "$SA_EMAILS" ]; then - echo -e "${RED}Failed to find extension's service account email.${NC}" - FAILED_TASKS+=("${RED}${CROSS} Failed to find extension's service account.") - exit 1 # Exit if no service account is found as the next steps need it -else - # Use the first service account from the list - export SA_EMAIL=$(echo "$SA_EMAILS" | head -n 1) - echo -e "${GREEN}Service account email found: $SA_EMAIL${NC}" - - # Show all found service accounts for debugging - echo "$SA_EMAILS" - - SUCCESS_TASKS+=("${GREEN}${TICK} Found extension's service account.") -fi - -# Add required policy binding for Artifact Registry -echo -e "${YELLOW}Step 4: Adding IAM policy binding for Artifact Registry...${NC}" -if gcloud artifacts repositories add-iam-policy-binding $EXT_INSTANCE_ID \ - --location=$LOCATION \ - --project=$PROJECT_ID \ - --member="serviceAccount:$SA_EMAIL" \ - --role=roles/artifactregistry.writer \ - --condition=None; then - echo -e "${GREEN}Policy binding added successfully.${NC}" - SUCCESS_TASKS+=("${GREEN}${TICK} Policy binding added successfully.") -else - echo -e "${RED}Failed to add policy binding.${NC}" - FAILED_TASKS+=("${RED}${CROSS} Failed to add policy binding.") -fi - -# Add roles for extension service account to trigger Dataflow -echo -e "${YELLOW}Step 5: Adding roles for service account to trigger Dataflow...${NC}" -ROLE_SUCCESS=true - -if ! gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:$SA_EMAIL" \ - --role=roles/dataflow.developer \ - --condition=None; then - ROLE_SUCCESS=false -fi - -if ! gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:$SA_EMAIL" \ - --role=roles/iam.serviceAccountUser \ - --condition=None; then - ROLE_SUCCESS=false -fi - -if ! gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:$SA_EMAIL" \ - --role=roles/artifactregistry.writer \ - --condition=None; then - ROLE_SUCCESS=false -fi - -if [ "$ROLE_SUCCESS" = true ]; then - echo -e "${GREEN}Roles added successfully.${NC}" - SUCCESS_TASKS+=("${GREEN}${TICK} Roles added successfully") -else - echo -e "${RED}Failed to add one or more roles.${NC}" - FAILED_TASKS+=("${RED}${CROSS} Failed to add one or more roles.") -fi \ No newline at end of file diff --git a/kits/firestore-incremental-capture/legacy/install/run.sh b/kits/firestore-incremental-capture/legacy/install/run.sh deleted file mode 100755 index b5654911d..000000000 --- a/kits/firestore-incremental-capture/legacy/install/run.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash - -# Define color codes for better readability -export RED='\033[0;31m' -export GREEN='\033[0;32m' -export YELLOW='\033[1;33m' -export NC='\033[0m' -export TICK="✓" -export CROSS="✗" - -# Initialize arrays to hold success and failure messages -export SUCCESS_TASKS=() -export FAILED_TASKS=() - -# Define variables at the top -export PROJECT_ID="" -export BUCKET_NAME="" -export DATABASE_ID="" -export DATABASE_LOCATION="nam5" -export LOCATION="us-central1" -export EXT_INSTANCE_ID="firestore-incremental-capture" -export JAR_PATH="restore-firestore.jar" - -# Detect default bucket automatically only if BUCKET_NAME is not set -detect_default_bucket() { - # Skip if BUCKET_NAME is already set - if [ -n "$BUCKET_NAME" ]; then - echo "Using user-specified bucket: $BUCKET_NAME" - return - fi - - echo "Detecting default storage bucket..." - - # Try to list buckets and check for default buckets - local buckets=$(gcloud storage buckets list --project=$PROJECT_ID --format="value(name)") - - # Check for both possible default bucket names - if echo "$buckets" | grep -q "$PROJECT_ID.appspot.com"; then - echo "Detected default bucket: $PROJECT_ID.appspot.com" - BUCKET_NAME="$PROJECT_ID.appspot.com" - elif echo "$buckets" | grep -q "$PROJECT_ID.firebasestorage.app"; then - echo "Detected default bucket: $PROJECT_ID.firebasestorage.app" - BUCKET_NAME="$PROJECT_ID.firebasestorage.app" - else - echo -e "${YELLOW}Warning: Could not detect default bucket, using fallback strategy${NC}" - # Use a fallback approach - newest default bucket format - BUCKET_NAME="$PROJECT_ID.firebasestorage.app" - fi - - echo "Using bucket: $BUCKET_NAME" -} - -# Call the detect function after PROJECT_ID is set -detect_default_bucket - -# Source all component scripts -source ./functions/download_restore_firestore.sh -source ./functions/enable_pitr.sh -source ./functions/setup_firestore.sh -source ./functions/setup_artifact_registry.sh -source ./functions/setup_service_account.sh -source ./functions/build_dataflow_template.sh - -# Print summary -echo -e "\n${GREEN}Setup process completed.${NC}" - -if [ ${#SUCCESS_TASKS[@]} -gt 0 ]; then - echo -e "\n${GREEN}Successful operations:${NC}" - for TASK in "${SUCCESS_TASKS[@]}"; do - echo -e "$TASK" - done -fi - -if [ ${#FAILED_TASKS[@]} -gt 0 ]; then - echo -e "\n${RED}Failed operations:${NC}" - for TASK in "${FAILED_TASKS[@]}"; do - echo -e "$TASK" - done - echo -e "\n${RED}Warning: Some operations failed. Please review the errors above.${NC}" - exit 1 -else - echo -e "\n${GREEN}All operations completed successfully!${NC}" -fi \ No newline at end of file diff --git a/kits/firestore-incremental-capture/vitest.config.ts b/kits/firestore-incremental-capture/vitest.config.ts index 52314463a..bb202f9ea 100644 --- a/kits/firestore-incremental-capture/vitest.config.ts +++ b/kits/firestore-incremental-capture/vitest.config.ts @@ -18,8 +18,7 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { - // Scoped to the kit's own tests: `pipeline/` is Java, and `legacy/` is the - // reference copy of the extension this kit was migrated from. + // Scoped to the kit's own tests; `pipeline/` is Java, built by Maven. include: ["tests/**/*.test.ts"], }, }); From 559ea1acfd128c3ca88b4720d0472f9dd8a8745f Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Tue, 11 Aug 2026 15:32:20 +0100 Subject: [PATCH 06/10] fix(kits): align incremental capture with kit conventions Rebasing onto the current kits branch surfaced a deploy-model change the implementation predated. A kit stanza deploys every export as kit--, so the two-hop capture path was enqueueing onto task queues that do not exist: locations//functions/syncChangelogTask rather than locations//functions/kit-default-syncChangelogTask. Every captured write would have failed to enqueue. queueName() now builds the deployed name, and INSTANCE_ID becomes required with no default, since it has to match this instance's key in the instances map and a wrong value fails the same silent way. setup.sh defaults it to "default" so the flex template it stages is the one the function launches. Conventions the branch settled on since: - Drop firebase.json, .gitignore, .env.example and vitest.config.ts. Kits ship four files plus src/tests; kits/.gitignore covers build output, and its .env.* pattern is why .env.example files went. The vitest config only existed to exclude legacy/, which is gone. - firebase-functions 7.3.2, build via tsc -b, and typescript/@types/node from the root rather than per-kit devDependencies. - Restructure the README to the standard section order, and document the kit stanza, the kit-- naming and multiple instances. --- .../.env.example | 32 - kits/firestore-incremental-capture/.gitignore | 2 - kits/firestore-incremental-capture/README.md | 337 +++++--- .../firebase.json | 10 - .../package-lock.json | 769 ++++++++---------- .../package.json | 6 +- .../scripts/setup.sh | 9 +- .../src/capture-config.ts | 28 +- .../src/config.ts | 8 +- .../src/tasks.ts | 20 +- .../tests/capture-config.test.ts | 10 +- .../tests/dataflow.test.ts | 11 +- .../tests/handlers.test.ts | 1 + .../tests/tasks.test.ts | 66 ++ .../vitest.config.ts | 24 - 15 files changed, 694 insertions(+), 639 deletions(-) delete mode 100644 kits/firestore-incremental-capture/.env.example delete mode 100644 kits/firestore-incremental-capture/.gitignore delete mode 100644 kits/firestore-incremental-capture/firebase.json create mode 100644 kits/firestore-incremental-capture/tests/tasks.test.ts delete mode 100644 kits/firestore-incremental-capture/vitest.config.ts diff --git a/kits/firestore-incremental-capture/.env.example b/kits/firestore-incremental-capture/.env.example deleted file mode 100644 index db37b4b6c..000000000 --- a/kits/firestore-incremental-capture/.env.example +++ /dev/null @@ -1,32 +0,0 @@ -# Copy to .env or .env.. Run scripts/setup.sh before deploying; it -# prints the values to use here. - -# Region for the functions. Must be one of the Dataflow flex template regions. -LOCATION=us-central1 - -# Collection to capture. A multi-segment wildcard is not supported: a Firestore -# trigger only accepts one as its final path segment. -SYNC_COLLECTION_PATH=posts - -# BigQuery changelog destination. -SYNC_DATASET=backup_dataset -SYNC_TABLE=backup_table -DATASET_LOCATION=us - -# Firestore database restorations are written into. Required, and must not be -# "(default)" - a restoration batch-writes over it. Created by setup.sh. -BACKUP_INSTANCE_ID= - -# Bucket the flex template was staged to. Defaults to the project's default -# bucket, which is what setup.sh stages to unless you override it there. -# BUCKET_NAME= - -# Region for Dataflow jobs. Defaults to LOCATION. -# DATAFLOW_REGION= - -# Namespaces the task queues, template object, job names and status documents. -# Must match the INSTANCE_ID passed to setup.sh. -# INSTANCE_ID=firestore-incremental-capture - -# debug | info | warn | error | silent -# LOG_LEVEL=info diff --git a/kits/firestore-incremental-capture/.gitignore b/kits/firestore-incremental-capture/.gitignore deleted file mode 100644 index 73c8594a9..000000000 --- a/kits/firestore-incremental-capture/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -lib -*.tsbuildinfo diff --git a/kits/firestore-incremental-capture/README.md b/kits/firestore-incremental-capture/README.md index 387178546..64dd8d46f 100644 --- a/kits/firestore-incremental-capture/README.md +++ b/kits/firestore-incremental-capture/README.md @@ -1,104 +1,46 @@ # @firebase/firestore-incremental-capture -Incremental point-in-time capture of Firestore changes, as a deployable Firebase Function. +Incremental point-in-time capture of Firestore changes. This is the Firestore +Incremental Backup Stream Firebase Extension as an npm package you add to your +own Firebase Functions codebase and deploy. -Migrated from the `firestore-incremental-capture` Firebase Extension. Every write to a watched -collection is serialized into a BigQuery changelog. A restoration then rebuilds a separate Firestore -database as it stood at a chosen second: a Dataflow pipeline reads a PITR snapshot of the source -database and replays the changelog on top of it. +Every write to a watched collection is serialized into a BigQuery changelog. A +restoration then rebuilds a separate Firestore database as it stood at a chosen +second: a Dataflow pipeline reads a point-in-time-recovery snapshot of the +source database and replays the changelog on top of it. The functions run in +your own Firebase project; there is no hosted version, so you deploy them +yourself. -## How it works +## Install -**Capture.** `syncData` fires on every document write, serializes the before/after data, and queues -it. `syncChangelogTask` inserts the queued row into BigQuery. The insert is a separate hop so a -BigQuery outage retries on the task queue's schedule instead of holding the Firestore trigger open. - -**Restore.** `onHttpRunRestoration` validates a timestamp and queues the work. `runRestorationTask` -launches the Dataflow flex template in `pipeline/`, which writes the PITR baseline into the backup -database and then replays every changelog row up to the timestamp. - -**Provisioning.** `initIncrementalCapture` runs after first deploy and after each redeploy, creating -the BigQuery dataset and changelog table. Everything restoration needs beyond that is provisioned by -`scripts/setup.sh` - see below. - -## Security - -`onHttpRunRestoration` is **unauthenticated**, matching the extension it was migrated from. Anyone -who can reach the URL can start a Dataflow job that batch-writes over the backup database. Before -deploying to production, restrict it: set Cloud Run ingress, apply an IAM invoker policy, or drop the -endpoint and have your own authorized code enqueue `runRestorationTask` directly. - -## Setup - -Restoration needs a PITR-enabled source database, an existing backup database, and a staged Dataflow -flex template. None can be provisioned from the functions runtime, which has neither gcloud nor -Maven. Run the setup script once, before deploying: - -```bash -PROJECT_ID=my-project BACKUP_INSTANCE_ID=my-backup ./scripts/setup.sh +```sh +npm install @firebase/firestore-incremental-capture ``` -It enables the required APIs, turns on PITR, creates the backup database, creates an Artifact -Registry repository, grants the Dataflow roles, builds the pipeline jar from `pipeline/`, and stages -the flex template. Every step is idempotent. See the header of `scripts/setup.sh` for the optional -variables. - -PITR only covers writes made after it is enabled, so restoration can only target a point in time -after setup ran. - -## Configuration - -Set these in `.env` or `.env.`. - -| Param | Default | Description | -| ---------------------- | ------------------------------- | ----------------------------------------------------------- | -| `LOCATION` | `us-central1` | Region for the functions. | -| `SYNC_COLLECTION_PATH` | `posts` | Collection to capture. | -| `SYNC_DATASET` | `backup_dataset` | BigQuery dataset for the changelog. | -| `SYNC_TABLE` | `backup_table` | BigQuery changelog table. | -| `BACKUP_INSTANCE_ID` | _required_ | Firestore database to restore into. Must not be `(default)`. | -| `DATASET_LOCATION` | `us` | BigQuery dataset location. | -| `DATAFLOW_REGION` | `LOCATION` | Region for Dataflow jobs. | -| `BUCKET_NAME` | the project's default bucket | Bucket the flex template was staged to. | -| `INSTANCE_ID` | `firestore-incremental-capture` | Namespaces the queues, template, jobs and status documents. | -| `LOG_LEVEL` | `info` | `debug`, `info`, `warn`, `error` or `silent`. | - -**Only the `(default)` database can be captured.** The restoration pipeline reads its PITR baseline -from `FirestoreOptions.getDefaultInstance()` (`RestorationPipeline.java`), so a non-default source -database would be captured to the changelog but absent from the restored baseline. There is -deliberately no param for it. - -**Only a single collection can be captured.** A Firestore trigger takes a multi-segment wildcard only -as its final path segment, so `SYNC_COLLECTION_PATH={document=**}` produces the undeployable pattern -`{document=**}/{documentId}`. Whole-database capture is not available. - -`BUCKET_NAME` is read from the project's default bucket when unset, rather than guessed from the -project id - the default bucket is `.firebasestorage.app` for projects created after -September 2024 and `.appspot.com` for older ones, and a wrong guess means restoration -launches against a template that was never staged there. - ## Required IAM -The package declares the roles below with `requiresRole(...)`. Firebase CLI 15.23.0 or later creates a -managed runtime service account for the codebase, grants it these roles, and attaches it to every -function. Declarative security cannot be combined with a custom runtime service account. - -| Role | Why | -| ----------------------------- | --------------------------------------------------------------- | -| `roles/bigquery.dataEditor` | create the changelog dataset/table; insert rows | -| `roles/bigquery.user` | run BigQuery jobs | -| `roles/datastore.user` | write the restoration run-status document | -| `roles/dataflow.developer` | launch the restoration job | -| `roles/iam.serviceAccountUser`| act as the Dataflow worker service account when launching | -| `roles/storage.objectViewer` | read the staged flex template spec | - -`scripts/setup.sh` cannot grant these: the managed account does not exist until the first deploy. What -the script does grant is the separate set of roles the **Dataflow worker** service account needs -(`dataflow.worker`, `datastore.user`, BigQuery read, staging bucket access). +Deploy needs these Google Cloud roles for the function's service account. +Firebase CLI 15.23.0 or later creates that account, grants the roles below, and +attaches the account to every function in this kit. Do not set a custom runtime +service account for this codebase — it conflicts with that automatic setup. + +| Role | Why | +| ------------------------------ | --------------------------------------------------------- | +| `roles/bigquery.dataEditor` | create the changelog dataset/table; insert rows | +| `roles/bigquery.user` | run BigQuery jobs | +| `roles/datastore.user` | write the restoration run-status document | +| `roles/dataflow.developer` | launch the restoration job | +| `roles/iam.serviceAccountUser` | act as the Dataflow worker service account when launching | +| `roles/storage.objectViewer` | read the staged flex template spec | + +`scripts/setup.sh` cannot grant these, because the managed account does not +exist until the first deploy. What the script grants instead is the separate set +of roles the **Dataflow worker** service account needs (`dataflow.worker`, +`datastore.user`, BigQuery read, staging bucket access). ## Usage -Re-export the functions from your own functions codebase entry: +Re-export the five wired functions from your functions codebase entry: ```ts export { @@ -110,49 +52,204 @@ export { } from "@firebase/firestore-incremental-capture"; ``` +- `syncData` is the Firestore trigger. It serializes each write and queues it. +- `syncChangelogTask` inserts a queued row into BigQuery. It is a separate hop so + a BigQuery outage retries on the queue's schedule rather than holding the + trigger open. +- `onHttpRunRestoration` validates a timestamp and queues a restoration. +- `runRestorationTask` launches the Dataflow pipeline. +- `initIncrementalCapture` is the first-deploy and redeploy provisioning + lifecycle task. + +Importing the package without exporting its functions deploys nothing — the CLI +only deploys what your entry file exports. + Trigger a restoration with a whole number of seconds since the Unix epoch: -```bash -curl -X POST https://-.cloudfunctions.net/onHttpRunRestoration \ +```sh +curl -X POST https://-.cloudfunctions.net/kit-default-onHttpRunRestoration \ -H 'Content-Type: application/json' \ -d '{"timestamp": 1700000000}' ``` -To own trigger registration yourself, import the handlers from the side-effect-free surface: +> **`onHttpRunRestoration` is unauthenticated**, matching the extension it was +> migrated from. Anyone who can reach the URL can start a Dataflow job that +> batch-writes over the backup database. Before deploying to production, +> restrict it: set Cloud Run ingress, apply an IAM invoker policy, or drop the +> endpoint and have your own authorized code enqueue `runRestorationTask` +> directly. -```ts -import { handleDocumentWrite, resolveCaptureConfig } from "@firebase/firestore-incremental-capture/lib"; +## Deploy + +Restoration depends on setup the functions runtime cannot do for itself: it has +neither gcloud nor Maven. Run the setup script once, before deploying: + +```sh +PROJECT_ID=my-project BACKUP_INSTANCE_ID=my-backup ./scripts/setup.sh ``` -## Restoration gaps +It enables the required APIs, turns on PITR for the source database, creates the +backup database, creates an Artifact Registry repository, grants the Dataflow +worker roles, builds the pipeline jar from `pipeline/`, and stages the Dataflow +flex template. Every step is idempotent, so re-running after a partial failure +is safe. The script prints the config values to use below. + +PITR only covers writes made after it is enabled, so a restoration can only +target a point in time after setup ran. + +The package's `firebase.json` declares a `kit` stanza (Firebase CLI 15.25.1 or +later, behind the `kits` experiment): + +```json +{ + "functions": [ + { + "source": ".", + "kit": "firestore-incremental-capture", + "instances": { + "default": "." + } + } + ] +} +``` -The restoration pipeline in `pipeline/` is vendored from the original extension unchanged, and it -does not round trip everything the capture side records. `FirestoreReconstructor.buildFirestoreMap` -switches on each value's type tag and **silently drops any field whose tag it does not handle**: - -- **`binary` and `null` fields are dropped.** The pipeline has no case for either, so a restored - document loses them. -- **Arrays of primitives do not survive.** `buildFirestoreList` rebuilds every element by passing it - to `buildFirestoreMap`, which reads field names at the top level, so `[1, 2]` restores as a list of - empty maps. Arrays of maps do round trip - see the note in `src/serializer.ts` on why array - elements are encoded differently from map fields. -- **Changelog replay writes to a malformed path.** `IncrementalCaptureLog.convertToFirestoreValue` - applies `createDocumentName` to a path that has already been through it, producing a doubled - `projects/…/databases/…/documents/` prefix. -- **Documents sharing an id across collections collide.** The replay query ranks with - `ROW_NUMBER() OVER(PARTITION BY documentId …)`, partitioning by document id rather than path, so - only one of `users/x` and `orders/x` is replayed. +`instances` maps each instance id to the directory (relative to +`firebase.json`) holding that instance's `.env`. The CLI prefixes every +function and task queue name with `kit--`, so the functions above +deploy as `kit-default-syncData`, `kit-default-syncChangelogTask`, +`kit-default-onHttpRunRestoration`, `kit-default-runRestorationTask`, and +`kit-default-initIncrementalCapture`. + +```sh +firebase experiments:enable kits +firebase deploy --only functions +``` -The PITR baseline half of a restoration is unaffected; these apply to the changelog replay on top of -it. Fixing them means changing the Java, which is out of scope for this migration. +Deploy a single instance with `firebase deploy --only functions:`. -## Development +## Configuration -```bash -npm install -npm run build -npm test +Set these values in a `.env` (or `.env.`) file. The Firebase CLI +loads them at deploy time and prompts for any required values that are missing. +`PROJECT_ID` is supplied by the Firebase CLI. + +| Field | Env var | Required | Default | Description | +| -------------------- | ---------------------- | -------- | ---------------- | ----------------------------------------------------------- | +| `instanceId` | `INSTANCE_ID` | yes | — | Must match this instance's key in the `instances` map | +| `backupInstanceId` | `BACKUP_INSTANCE_ID` | yes | — | Firestore database to restore into; must not be `(default)` | +| `syncCollectionPath` | `SYNC_COLLECTION_PATH` | no | `posts` | Collection to capture | +| `datasetId` | `SYNC_DATASET` | no | `backup_dataset` | BigQuery dataset for the changelog | +| `tableId` | `SYNC_TABLE` | no | `backup_table` | BigQuery changelog table | +| `location` | `LOCATION` | no | `us-central1` | Region for the functions and task queues | +| `datasetLocation` | `DATASET_LOCATION` | no | `us` | BigQuery dataset location | +| `dataflowRegion` | `DATAFLOW_REGION` | no | `LOCATION` | Region for Dataflow jobs | +| `bucketName` | `BUCKET_NAME` | no | default bucket | Bucket the flex template was staged to | +| `logLevel` | `LOG_LEVEL` | no | `info` | `debug`, `info`, `warn`, `error`, `silent` | + +Three constraints are worth stating outright, because each one fails silently if +you assume otherwise: + +- **Only the `(default)` database can be captured.** The pipeline reads its PITR + baseline from `FirestoreOptions.getDefaultInstance()` + (`RestorationPipeline.java`), so a non-default source would be captured to the + changelog but absent from the restored baseline. There is deliberately no + param for it. +- **Only a single collection can be captured.** A Firestore trigger accepts a + multi-segment wildcard only as its final path segment, so + `SYNC_COLLECTION_PATH={document=**}` produces the undeployable pattern + `{document=**}/{documentId}`. Whole-database capture is not available. +- **`BUCKET_NAME` is read from the project's default bucket when unset**, rather + than guessed from the project id. The default bucket is + `.firebasestorage.app` for projects created after September 2024 + and `.appspot.com` for older ones, and a wrong guess means + launching against a template that was never staged there. + +## Multiple instances + +To capture several collections, add one entry per instance to the `instances` +map, each pointing at its own config directory with its own `.env`: + +```json +{ + "functions": [ + { + "source": ".", + "kit": "firestore-incremental-capture", + "instances": { + "users": "instances/users", + "orders": "instances/orders" + } + } + ] +} ``` -`pipeline/` is the Java/Beam restoration pipeline, built by `scripts/setup.sh`. To work on it -directly, see `pipeline/README.md`. +Instance ids must be unique across all kit stanzas in the project, and every +instance's function names are namespaced by its `kit--` prefix, so +the instances cannot collide. Set `INSTANCE_ID` in each config directory to that +instance's key — the kit uses it to address its own task queues, and a mismatch +enqueues onto a queue that does not exist. + +Give each instance its own `SYNC_DATASET`/`SYNC_TABLE` or its own +`BACKUP_INSTANCE_ID`. Two instances sharing a changelog table would replay each +other's documents on restore. + +## Provisioning + +`initIncrementalCapture` runs after first deploy and after each redeploy. It +creates the BigQuery dataset and changelog table if they are missing, running in +the function's own identity so it has the runtime service account the creation +needs. It is idempotent, and Cloud Tasks retries it on a transient BigQuery +error so a blip does not leave the changelog unprovisioned. + +It does not provision the restoration prerequisites — PITR, the backup database +and the flex template all need gcloud. That is what `scripts/setup.sh` is for. + +## Restoration gaps + +The restoration pipeline in `pipeline/` is vendored from the original extension +unchanged, and it does not round trip everything the capture side records. +`FirestoreReconstructor.buildFirestoreMap` switches on each value's type tag and +silently drops any field whose tag it does not handle: + +- **`binary` and `null` fields are dropped.** The pipeline has no case for + either, so a restored document loses them. +- **Arrays of primitives do not survive.** `buildFirestoreList` rebuilds every + element by passing it to `buildFirestoreMap`, which reads field names at the + top level, so `[1, 2]` restores as a list of empty maps. Arrays of maps do + round trip — see the note in `src/serializer.ts` on why array elements are + encoded differently from map fields. +- **Changelog replay writes to a malformed path.** + `IncrementalCaptureLog.convertToFirestoreValue` applies `createDocumentName` + to a path that has already been through it, producing a doubled + `projects/…/databases/…/documents/` prefix. +- **Documents sharing an id across collections collide.** The replay query ranks + with `ROW_NUMBER() OVER(PARTITION BY documentId …)`, partitioning by document + id rather than path, so only one of `users/x` and `orders/x` is replayed. + +The PITR baseline half of a restoration is unaffected; these apply to the +changelog replay on top of it. Fixing them means changing the Java, which is out +of scope for this migration. + +## API surface + +- **Main entry** (`@firebase/firestore-incremental-capture`): exports the five + wired functions listed under Usage, and registers the first-deploy / redeploy + provisioning hooks. Runtime config is resolved lazily on first invocation. Use + this entry from Firebase deploy/emulator/runtime. For your own triggers, + import from `./lib` instead. +- **Library entry** (`./lib`): the raw handlers for owning trigger registration + yourself (`handleDocumentWrite`, `handleChangelogTask`, + `handleRestorationRequest`, `handleRestorationTask`), the config types and + helpers (`CaptureConfig`, `resolveCaptureConfig`) for building their injected + `HandlerContext`, and the changelog wire format (`CHANGELOG_SCHEMA`, + `ChangelogRow`, `serializeDocument`) for reading the changelog or + reimplementing the restoration side. Safe to import anywhere. + +`pipeline/` is the Java/Beam restoration pipeline, built by `scripts/setup.sh`. +To work on it directly, see `pipeline/README.md`. + +## License + +Apache-2.0 diff --git a/kits/firestore-incremental-capture/firebase.json b/kits/firestore-incremental-capture/firebase.json deleted file mode 100644 index 0abf0a395..000000000 --- a/kits/firestore-incremental-capture/firebase.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "functions": [ - { - "source": ".", - "codebase": "firestore-incremental-capture", - "ignore": ["node_modules", ".git", "src", "pipeline", "*.local"], - "predeploy": ["npm --prefix \"$RESOURCE_DIR\" run build"] - } - ] -} diff --git a/kits/firestore-incremental-capture/package-lock.json b/kits/firestore-incremental-capture/package-lock.json index 27b8e04aa..20fb0704a 100644 --- a/kits/firestore-incremental-capture/package-lock.json +++ b/kits/firestore-incremental-capture/package-lock.json @@ -12,11 +12,9 @@ "@google-cloud/bigquery": "^7.6.0", "@google-cloud/dataflow": "^3.2.0", "firebase-admin": "^14.1.0", - "firebase-functions": "7.3.0" + "firebase-functions": "7.3.2" }, "devDependencies": { - "@types/node": "^22.0.0", - "typescript": "^6.0.0", "vitest": "^3.2.4" }, "engines": { @@ -677,24 +675,6 @@ "node": ">=6" } }, - "node_modules/@google-cloud/firestore/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "optional": true, - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/@google-cloud/firestore/node_modules/gaxios": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.3.0.tgz", @@ -821,13 +801,6 @@ "node": ">= 14" } }, - "node_modules/@google-cloud/firestore/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT", - "optional": true - }, "node_modules/@google-cloud/firestore/node_modules/node-fetch": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", @@ -1693,21 +1666,20 @@ "license": "MIT" }, "node_modules/@types/express": { - "version": "4.17.25", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", - "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.6.tgz", + "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==", "license": "MIT", "dependencies": { "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "^1" + "@types/express-serve-static-core": "^5.0.0", + "@types/serve-static": "^2" } }, "node_modules/@types/express-serve-static-core": { - "version": "4.19.9", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.9.tgz", - "integrity": "sha512-QP2ESEe/ImWY0HDwNAnK9PvEffUyhLTnWkk7KXzHfyeWAnlrDe1fN77bXl6ia8KT3wPlmA7t9/VPRpnf4Ex9sg==", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.3.tgz", + "integrity": "sha512-dPfW8NFiOF4wOHc7+N/QSxlY9cfSsenewGbAz8C8U/MULPd/YZ27LvJUIlzaXie7e6Ove9YunJGgC9tbHD2cKw==", "license": "MIT", "dependencies": { "@types/node": "*", @@ -1738,12 +1710,6 @@ "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==", "license": "MIT" }, - "node_modules/@types/mime": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", - "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", - "license": "MIT" - }, "node_modules/@types/ms": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", @@ -1793,23 +1759,12 @@ } }, "node_modules/@types/serve-static": { - "version": "1.15.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz", - "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==", "license": "MIT", "dependencies": { "@types/http-errors": "*", - "@types/node": "*", - "@types/send": "<1" - } - }, - "node_modules/@types/serve-static/node_modules/@types/send": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz", - "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==", - "license": "MIT", - "dependencies": { - "@types/mime": "^1", "@types/node": "*" } }, @@ -1947,18 +1902,43 @@ } }, "node_modules/accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", "license": "MIT", "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" }, "engines": { "node": ">= 0.6" } }, + "node_modules/accepts/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -2005,12 +1985,6 @@ "license": "MIT", "optional": true }, - "node_modules/array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", - "license": "MIT" - }, "node_modules/arrify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", @@ -2096,27 +2070,40 @@ } }, "node_modules/body-parser": { - "version": "1.20.6", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.6.tgz", - "integrity": "sha512-p5tAzS57i5MV9fZFDj9LeIiTZEufbSe2eDozP+ElheSUq1m74CRq1jI4mYNDdVs9vQztXFLuk/Gd6BWTdwRJ5g==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", "license": "MIT", "dependencies": { - "bytes": "~3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "~1.2.0", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "on-finished": "~2.4.1", - "qs": "~6.15.1", - "raw-body": "~2.5.3", - "type-is": "~1.6.18", - "unpipe": "~1.0.0" + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/brace-expansion": { @@ -2255,15 +2242,16 @@ } }, "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", "license": "MIT", - "dependencies": { - "safe-buffer": "5.2.1" - }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/content-type": { @@ -2285,10 +2273,13 @@ } }, "node_modules/cookie-signature": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", - "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", - "license": "MIT" + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } }, "node_modules/cors": { "version": "2.8.6", @@ -2332,12 +2323,20 @@ } }, "node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { - "ms": "2.0.0" + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, "node_modules/deep-eql": { @@ -2368,16 +2367,6 @@ "node": ">= 0.8" } }, - "node_modules/destroy": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", - "license": "MIT", - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -2598,45 +2587,67 @@ } }, "node_modules/express": { - "version": "4.22.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", - "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", - "license": "MIT", - "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "~1.20.5", - "content-disposition": "~0.5.4", - "content-type": "~1.0.4", - "cookie": "~0.7.1", - "cookie-signature": "~1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "~1.3.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.0", - "merge-descriptors": "1.0.3", - "methods": "~1.1.2", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "~0.1.12", - "proxy-addr": "~2.0.7", - "qs": "~6.15.1", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "~0.19.0", - "serve-static": "~1.16.2", - "setprototypeof": "1.2.0", - "statuses": "~2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.10.0" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" }, "funding": { "type": "opencollective", @@ -2750,21 +2761,24 @@ } }, "node_modules/finalhandler": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", - "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", "license": "MIT", "dependencies": { - "debug": "2.6.9", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "statuses": "~2.0.2", - "unpipe": "~1.0.0" + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" }, "engines": { - "node": ">= 0.8" + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/firebase-admin": { @@ -2862,15 +2876,15 @@ } }, "node_modules/firebase-functions": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/firebase-functions/-/firebase-functions-7.3.0.tgz", - "integrity": "sha512-S3JhjESOWMq13iDodwgUPWNQ3gLAu7oTLDwF7hTtisyjVanEeQzYezgW+JTfd8I0kCJQzjGPC/wHQ19IL7CgaA==", + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/firebase-functions/-/firebase-functions-7.3.2.tgz", + "integrity": "sha512-DTa5CtCNxFE1gPjkjlhl5uh8AcNWXLKORAMRenfkecP8zGstbje+r2TaNh4Ehdxp0qTPvTUe/sS23VESW32Gbg==", "license": "MIT", "dependencies": { "@types/cors": "^2.8.5", - "@types/express": "^4.17.21", + "@types/express": "^5.0.0", "cors": "^2.8.5", - "express": "^4.21.0", + "express": "^5.2.1", "protobufjs": "^7.2.2" }, "bin": { @@ -2953,12 +2967,12 @@ } }, "node_modules/fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/fsevents": { @@ -3271,29 +3285,6 @@ "node": ">= 6.0.0" } }, - "node_modules/http-proxy-agent/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/http-proxy-agent/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, "node_modules/https-proxy-agent": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", @@ -3307,39 +3298,20 @@ "node": ">= 14" } }, - "node_modules/https-proxy-agent/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/https-proxy-agent/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", "license": "MIT", "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" }, "engines": { "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/inherits": { @@ -3375,6 +3347,12 @@ "node": ">=8" } }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, "node_modules/is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", @@ -3470,12 +3448,6 @@ "npm": ">=6" } }, - "node_modules/jsonwebtoken/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, "node_modules/jwa": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", @@ -3504,29 +3476,6 @@ "node": "^20.19.0 || ^22.12.0 || >= 23.0.0" } }, - "node_modules/jwks-rsa/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/jwks-rsa/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, "node_modules/jws": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", @@ -3648,32 +3597,30 @@ } }, "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/merge-descriptors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", "license": "MIT", + "engines": { + "node": ">=18" + }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/mime": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", @@ -3735,9 +3682,9 @@ } }, "node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, "node_modules/nanoid": { @@ -3760,9 +3707,9 @@ } }, "node_modules/negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", "license": "MIT", "engines": { "node": ">= 0.6" @@ -3942,10 +3889,14 @@ "optional": true }, "node_modules/path-to-regexp": { - "version": "0.1.13", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", - "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", - "license": "MIT" + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } }, "node_modules/pathe": { "version": "2.0.3", @@ -4078,27 +4029,31 @@ } }, "node_modules/range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", "license": "MIT", "engines": { "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/raw-body": { - "version": "2.5.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", - "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", "license": "MIT", "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", + "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" }, "engines": { - "node": ">= 0.8" + "node": ">= 0.10" } }, "node_modules/readable-stream": { @@ -4210,6 +4165,22 @@ "fsevents": "~2.3.2" } }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -4249,60 +4220,73 @@ } }, "node_modules/send": { - "version": "0.19.2", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", - "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", - "license": "MIT", - "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.1", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "~2.4.1", - "range-parser": "~1.2.1", - "statuses": "~2.0.2" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/send/node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "node_modules/send/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "license": "MIT", - "bin": { - "mime": "cli.js" - }, "engines": { - "node": ">=4" + "node": ">= 0.6" } }, - "node_modules/send/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" + "node_modules/send/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } }, "node_modules/serve-static": { - "version": "1.16.3", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", - "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", "license": "MIT", "dependencies": { - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "~0.19.1" + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/setprototypeof": { @@ -4602,23 +4586,6 @@ "node": ">= 6.0.0" } }, - "node_modules/teeny-request/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, "node_modules/teeny-request/node_modules/https-proxy-agent": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", @@ -4632,12 +4599,6 @@ "node": ">= 6" } }, - "node_modules/teeny-request/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -4721,30 +4682,59 @@ "license": "0BSD" }, "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", "license": "MIT", "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, - "node_modules/typescript": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", - "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" + "node_modules/type-is/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" }, "engines": { - "node": ">=14.17" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/undici-types": { @@ -4768,15 +4758,6 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "license": "MIT" }, - "node_modules/utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", @@ -4898,31 +4879,6 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/vite-node/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/vite-node/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, "node_modules/vitest": { "version": "3.2.7", "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.7.tgz", @@ -4996,31 +4952,6 @@ } } }, - "node_modules/vitest/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/vitest/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, "node_modules/web-streams-polyfill": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", diff --git a/kits/firestore-incremental-capture/package.json b/kits/firestore-incremental-capture/package.json index 100af95a1..8228a192b 100644 --- a/kits/firestore-incremental-capture/package.json +++ b/kits/firestore-incremental-capture/package.json @@ -19,7 +19,7 @@ "node": ">=22" }, "scripts": { - "build": "tsc -p tsconfig.json", + "build": "tsc -b", "clean": "tsc -b --clean", "test": "vitest run", "deploy": "pnpm build && firebase deploy --only functions", @@ -29,11 +29,9 @@ "@google-cloud/bigquery": "^7.6.0", "@google-cloud/dataflow": "^3.2.0", "firebase-admin": "^14.1.0", - "firebase-functions": "7.3.0" + "firebase-functions": "7.3.2" }, "devDependencies": { - "@types/node": "^22.0.0", - "typescript": "^6.0.0", "vitest": "^3.2.4" } } diff --git a/kits/firestore-incremental-capture/scripts/setup.sh b/kits/firestore-incremental-capture/scripts/setup.sh index 159952ab3..3004de742 100755 --- a/kits/firestore-incremental-capture/scripts/setup.sh +++ b/kits/firestore-incremental-capture/scripts/setup.sh @@ -34,9 +34,10 @@ # Default "us-central1". # BUCKET_NAME Bucket holding the flex template. Defaults to the # project's default bucket. -# INSTANCE_ID Namespace for the deployed resources. Must match the -# kit's INSTANCE_ID param. Default -# "firestore-incremental-capture". +# INSTANCE_ID This instance's key in the `instances` map of the kit +# stanza, and the kit's INSTANCE_ID param. Must match +# both: it names the flex template object the deployed +# function launches. Default "default". # WORKER_SERVICE_ACCOUNT Service account the Dataflow workers run as. # Defaults to the Compute Engine default service account. # @@ -58,7 +59,7 @@ readonly BACKUP_INSTANCE_ID="${BACKUP_INSTANCE_ID:-}" readonly SOURCE_DATABASE="(default)" readonly DATABASE_LOCATION="${DATABASE_LOCATION:-nam5}" readonly LOCATION="${LOCATION:-us-central1}" -readonly INSTANCE_ID="${INSTANCE_ID:-firestore-incremental-capture}" +readonly INSTANCE_ID="${INSTANCE_ID:-default}" readonly JAR_NAME="restore-firestore.jar" readonly GREEN='\033[0;32m' diff --git a/kits/firestore-incremental-capture/src/capture-config.ts b/kits/firestore-incremental-capture/src/capture-config.ts index 60fcb46b5..82d227682 100644 --- a/kits/firestore-incremental-capture/src/capture-config.ts +++ b/kits/firestore-incremental-capture/src/capture-config.ts @@ -54,12 +54,14 @@ export interface CaptureConfig { */ bucketName: string; /** - * Namespaces the deployed resources: the task queues, the flex template - * object, the Dataflow job names and the Firestore status documents. Deploy - * the kit twice under one project by giving each deployment its own value. - * Defaults to `firestore-incremental-capture`. + * This instance's key in the `instances` map of the kit stanza. Required, and + * must match exactly: the CLI deploys every function as + * `kit--`, so a mismatch makes the kit enqueue onto + * task queues that do not exist. It also namespaces the flex template object, + * the Dataflow job names and the run-status documents, which is what keeps two + * instances in one project from colliding. */ - instanceId?: string; + instanceId: string; /** Defaults to `info`. */ logLevel?: LogLevel; } @@ -88,7 +90,6 @@ export interface ResolvedCaptureConfig /** The only database the restoration pipeline can read a PITR baseline from. */ const SOURCE_DATABASE_ID = "(default)"; -const DEFAULT_INSTANCE_ID = "firestore-incremental-capture"; const DEFAULT_LOCATION = "us-central1"; const DEFAULT_DATASET_LOCATION = "us"; @@ -97,15 +98,15 @@ const DEFAULT_DATASET_LOCATION = "us"; * * @param config - Caller-supplied configuration. * @returns The fully resolved configuration. - * @throws If `backupInstanceId` is empty or is the captured database, either of - * which would make a restoration write over the source it restores from; or if + * @throws If `instanceId` is empty, which would misname every task queue; if + * `backupInstanceId` is empty or is the captured database, either of which + * would make a restoration write over the source it restores from; or if * `bucketName` is empty, which would leave the flex template path unresolvable. */ export function resolveCaptureConfig( config: CaptureConfig ): ResolvedCaptureConfig { const location = config.location || DEFAULT_LOCATION; - const instanceId = config.instanceId || DEFAULT_INSTANCE_ID; const invalid = (detail: string): never => { throw new Error( @@ -113,6 +114,15 @@ export function resolveCaptureConfig( ); }; + if (!config.instanceId) { + invalid( + "INSTANCE_ID is required. It must match this instance's key in the " + + "`instances` map of the kit stanza in firebase.json." + ); + } + + const instanceId = config.instanceId; + if (!config.backupInstanceId) { invalid("BACKUP_INSTANCE_ID is required."); } diff --git a/kits/firestore-incremental-capture/src/config.ts b/kits/firestore-incremental-capture/src/config.ts index d7eaa023f..b3a4bcf5b 100644 --- a/kits/firestore-incremental-capture/src/config.ts +++ b/kits/firestore-incremental-capture/src/config.ts @@ -108,9 +108,9 @@ const params = { }), dataflowRegion: defineString("DATAFLOW_REGION", { default: "" }), bucketName: defineString("BUCKET_NAME", { default: "" }), - instanceId: defineString("INSTANCE_ID", { - default: "firestore-incremental-capture", - }), + // No default: it has to match this instance's key in the `instances` map, and + // a wrong value silently misnames the task queues. + instanceId: defineString("INSTANCE_ID"), logLevel: defineString("LOG_LEVEL", { default: "info", input: select([...LOG_LEVEL_OPTIONS]), @@ -163,7 +163,7 @@ export function configFromEnv(defaultBucketName?: string): CaptureConfig { location: optional(params.location.value()), dataflowRegion: optional(params.dataflowRegion.value()), bucketName: optional(params.bucketName.value()) || defaultBucketName || "", - instanceId: optional(params.instanceId.value()), + instanceId: params.instanceId.value(), logLevel: normalizeLogLevel(params.logLevel.value()), }; } diff --git a/kits/firestore-incremental-capture/src/tasks.ts b/kits/firestore-incremental-capture/src/tasks.ts index 806e1c1d2..a68c485d4 100644 --- a/kits/firestore-incremental-capture/src/tasks.ts +++ b/kits/firestore-incremental-capture/src/tasks.ts @@ -17,22 +17,36 @@ import { getFunctions } from "firebase-admin/functions"; import type { ResolvedCaptureConfig } from "./capture-config"; -/** Names of the deployed task-queue functions. */ +/** + * Names the functions are exported under. The CLI renames them on deploy - see + * {@link queueName}. + */ export const CHANGELOG_TASK_FUNCTION = "syncChangelogTask"; export const RESTORATION_TASK_FUNCTION = "runRestorationTask"; /** * Builds the fully-qualified task queue name for a deployed function. * + * A kit stanza deploys every function under `kit--`, + * so the queue to enqueue onto is not named after the export. `instanceId` must + * match this instance's key in the `instances` map for the name to resolve; + * a mismatch enqueues onto a queue that does not exist. + * * @param config - The resolved capture configuration. - * @param functionName - The deployed function's name. + * @param functionName - The name the function is exported under. * @returns The queue resource name. */ export function queueName( config: ResolvedCaptureConfig, functionName: string ): string { - return `locations/${config.location}/functions/${functionName}`; + const region = config.location || process.env.FUNCTION_REGION; + + if (!region) { + throw new Error("A region is required to resolve task queues."); + } + + return `locations/${region}/functions/kit-${config.instanceId}-${functionName}`; } /** diff --git a/kits/firestore-incremental-capture/tests/capture-config.test.ts b/kits/firestore-incremental-capture/tests/capture-config.test.ts index 052beb827..c33be986c 100644 --- a/kits/firestore-incremental-capture/tests/capture-config.test.ts +++ b/kits/firestore-incremental-capture/tests/capture-config.test.ts @@ -28,6 +28,7 @@ function config(overrides: Partial = {}): CaptureConfig { backupInstanceId: "backup-db", datasetId: "backup_dataset", tableId: "backup_table", + instanceId: "default", bucketName: "test-project.firebasestorage.app", ...overrides, }; @@ -40,10 +41,15 @@ describe("resolveCaptureConfig", () => { expect(resolved.databaseId).toBe("(default)"); expect(resolved.location).toBe("us-central1"); expect(resolved.datasetLocation).toBe("us"); - expect(resolved.instanceId).toBe("firestore-incremental-capture"); expect(resolved.logLevel).toBe("info"); }); + test("requires an instance id, which must match the instances map key", () => { + expect(() => resolveCaptureConfig(config({ instanceId: "" }))).toThrow( + /INSTANCE_ID is required/ + ); + }); + test("defaults the Dataflow region to the functions location", () => { expect( resolveCaptureConfig(config({ location: "europe-west1" })).dataflowRegion @@ -77,7 +83,7 @@ describe("resolveCaptureConfig", () => { "projects/test-project/databases/backup-db" ); expect(resolved.flexTemplatePath).toBe( - "gs://test-project.firebasestorage.app/firestore-incremental-capture-dataflow-restore" + "gs://test-project.firebasestorage.app/default-dataflow-restore" ); }); diff --git a/kits/firestore-incremental-capture/tests/dataflow.test.ts b/kits/firestore-incremental-capture/tests/dataflow.test.ts index dc406c7a0..080f0a157 100644 --- a/kits/firestore-incremental-capture/tests/dataflow.test.ts +++ b/kits/firestore-incremental-capture/tests/dataflow.test.ts @@ -51,6 +51,7 @@ function config(overrides: Partial = {}) { backupInstanceId: "backup-db", datasetId: "ds", tableId: "tbl", + instanceId: "default", bucketName: "test-project.firebasestorage.app", ...overrides, }); @@ -81,7 +82,7 @@ describe("RestorationLauncher", () => { const [request] = client.launchFlexTemplate.mock.calls[0]; expect(request.launchParameter.containerSpecGcsPath).toBe( - "gs://test-project.firebasestorage.app/firestore-incremental-capture-dataflow-restore" + "gs://test-project.firebasestorage.app/default-dataflow-restore" ); expect(request.launchParameter.containerSpecGcsPath).toBe( cfg.flexTemplatePath @@ -146,7 +147,7 @@ describe("RestorationLauncher", () => { timestamp: 1700000000, }); - expect(a.runId).toBe("firestore-incremental-capture-restore-1700000000"); + expect(a.runId).toBe("default-restore-1700000000"); expect(b.runId).toBe(a.runId); expect( first.launchFlexTemplate.mock.calls[0][0].launchParameter.jobName @@ -173,11 +174,9 @@ describe("RestorationLauncher", () => { expect(getFirestore).toHaveBeenCalledWith("(default)"); expect(collection).toHaveBeenCalledWith(cfg.restoreCollection); - expect(doc).toHaveBeenCalledWith( - "firestore-incremental-capture-restore-1700000000" - ); + expect(doc).toHaveBeenCalledWith("default-restore-1700000000"); expect(set).toHaveBeenCalledWith({ - runId: "firestore-incremental-capture-restore-1700000000", + runId: "default-restore-1700000000", jobName: "job-1", timestamp: 1700000000, status: "launched", diff --git a/kits/firestore-incremental-capture/tests/handlers.test.ts b/kits/firestore-incremental-capture/tests/handlers.test.ts index 6a0fd7adc..3550dfbec 100644 --- a/kits/firestore-incremental-capture/tests/handlers.test.ts +++ b/kits/firestore-incremental-capture/tests/handlers.test.ts @@ -63,6 +63,7 @@ function makeCtx(): HandlerContext { backupInstanceId: "backup-db", datasetId: "ds", tableId: "tbl", + instanceId: "default", bucketName: "test-project.firebasestorage.app", }), enqueueChangelogRow: vi.fn().mockResolvedValue(undefined), diff --git a/kits/firestore-incremental-capture/tests/tasks.test.ts b/kits/firestore-incremental-capture/tests/tasks.test.ts new file mode 100644 index 000000000..df9d7ee22 --- /dev/null +++ b/kits/firestore-incremental-capture/tests/tasks.test.ts @@ -0,0 +1,66 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, expect, test, vi } from "vitest"; +import { + type CaptureConfig, + resolveCaptureConfig, +} from "../src/capture-config"; + +vi.mock("firebase-admin/functions", () => ({ + getFunctions: vi.fn(), +})); + +const { CHANGELOG_TASK_FUNCTION, queueName, RESTORATION_TASK_FUNCTION } = + await import("../src/tasks"); + +function config(overrides: Partial = {}) { + return resolveCaptureConfig({ + projectId: "test-project", + syncCollectionPath: "users", + backupInstanceId: "backup-db", + datasetId: "ds", + tableId: "tbl", + instanceId: "default", + bucketName: "test-project.firebasestorage.app", + ...overrides, + }); +} + +describe("queueName", () => { + test("carries the kit-- prefix the CLI deploys under", () => { + // A kit stanza renames every export to kit--. + // Without the prefix the enqueue targets a queue that does not exist. + expect(queueName(config(), CHANGELOG_TASK_FUNCTION)).toBe( + "locations/us-central1/functions/kit-default-syncChangelogTask" + ); + expect(queueName(config(), RESTORATION_TASK_FUNCTION)).toBe( + "locations/us-central1/functions/kit-default-runRestorationTask" + ); + }); + + test("namespaces the queue by instance id", () => { + expect( + queueName(config({ instanceId: "orders" }), "syncChangelogTask") + ).toBe("locations/us-central1/functions/kit-orders-syncChangelogTask"); + }); + + test("uses the configured region", () => { + expect( + queueName(config({ location: "europe-west1" }), "syncChangelogTask") + ).toBe("locations/europe-west1/functions/kit-default-syncChangelogTask"); + }); +}); diff --git a/kits/firestore-incremental-capture/vitest.config.ts b/kits/firestore-incremental-capture/vitest.config.ts deleted file mode 100644 index bb202f9ea..000000000 --- a/kits/firestore-incremental-capture/vitest.config.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { defineConfig } from "vitest/config"; - -export default defineConfig({ - test: { - // Scoped to the kit's own tests; `pipeline/` is Java, built by Maven. - include: ["tests/**/*.test.ts"], - }, -}); From a528b109a36d8a40a04be9938abdc3794c4cb766 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Tue, 11 Aug 2026 15:42:45 +0100 Subject: [PATCH 07/10] fix(kits): stringify BigInt in the incremental capture serializer JSON.stringify throws a TypeError on a BigInt, so a document with one failed handleDocumentWrite outright rather than losing a single field, and the Firestore trigger retried it forever. The tag was already declared in SerializedType; only the encoding was missing. Introduced by this PR, not inherited from the extension. --- kits/firestore-incremental-capture/src/serializer.ts | 7 +++++++ .../tests/wire-format.test.ts | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/kits/firestore-incremental-capture/src/serializer.ts b/kits/firestore-incremental-capture/src/serializer.ts index 678e3abeb..8636812e0 100644 --- a/kits/firestore-incremental-capture/src/serializer.ts +++ b/kits/firestore-incremental-capture/src/serializer.ts @@ -122,6 +122,13 @@ function serializeValue(value: unknown): SerializedValue { return { type: "map", value: serializeDocument(value) }; } + // Stringified because JSON.stringify throws on a BigInt, which would fail the + // capture for this document permanently rather than dropping one field. The + // Firestore client only returns BigInt when configured with `useBigInt`. + if (typeof value === "bigint") { + return { type: "bigint", value: value.toString() }; + } + return { type: typeof value as SerializedType, value }; } diff --git a/kits/firestore-incremental-capture/tests/wire-format.test.ts b/kits/firestore-incremental-capture/tests/wire-format.test.ts index dc17fff9f..005d8796b 100644 --- a/kits/firestore-incremental-capture/tests/wire-format.test.ts +++ b/kits/firestore-incremental-capture/tests/wire-format.test.ts @@ -198,6 +198,13 @@ describe("changelog wire format", () => { }); }); + test("stringifies a BigInt so the row survives JSON.stringify", () => { + const serialized = serializeDocument({ big: 10n }); + + expect(serialized).toEqual({ big: { type: "bigint", value: "10" } }); + expect(() => JSON.stringify(serialized)).not.toThrow(); + }); + test("tags null rather than omitting the field", () => { expect(serializeDocument({ nullValue: null })).toEqual({ nullValue: { type: "null", value: null }, From f3ee8b95a1e4d5e930dc094da6c3ca3749e7b1f9 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Tue, 11 Aug 2026 15:42:56 +0100 Subject: [PATCH 08/10] fix(kits): keep incremental capture setup progress off stdout build_jar returns the jar path on stdout, but step/ok and Maven wrote there too, so main() parsed it back out with tail -n 1. Progress and Maven output now go to stderr and the parsing hack is gone. Introduced by this PR, not inherited from the extension. --- kits/firestore-incremental-capture/scripts/setup.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kits/firestore-incremental-capture/scripts/setup.sh b/kits/firestore-incremental-capture/scripts/setup.sh index 3004de742..c06c845dc 100755 --- a/kits/firestore-incremental-capture/scripts/setup.sh +++ b/kits/firestore-incremental-capture/scripts/setup.sh @@ -67,8 +67,9 @@ readonly YELLOW='\033[1;33m' readonly RED='\033[0;31m' readonly NC='\033[0m' -step() { echo -e "\n${YELLOW}==> $*${NC}"; } -ok() { echo -e "${GREEN} $*${NC}"; } +# Progress goes to stderr so stdout carries only a function's return value. +step() { echo -e "\n${YELLOW}==> $*${NC}" >&2; } +ok() { echo -e "${GREEN} $*${NC}" >&2; } die() { echo -e "${RED}Error: $*${NC}" >&2 exit 1 @@ -208,7 +209,8 @@ grant_worker_roles() { # extension fetched from GitHub is not a durable artifact. build_jar() { step "Building the restoration pipeline" - mvn -q -f "${PIPELINE_DIR}/pom.xml" clean package -DskipTests + # Maven to stderr as well: stdout is this function's return value. + mvn -q -f "${PIPELINE_DIR}/pom.xml" clean package -DskipTests >&2 local jar="${PIPELINE_DIR}/target/${JAR_NAME}" [[ -f "${jar}" ]] || die "Expected ${jar} after the Maven build." ok "Built ${jar}." @@ -246,7 +248,7 @@ main() { create_backup_database create_artifact_registry grant_worker_roles "${worker_service_account}" - jar="$(build_jar | tail -n 1)" + jar="$(build_jar)" stage_flex_template "${jar}" "${bucket}" echo -e "\n${GREEN}Setup complete.${NC}" From ee24cbaeb205bbfbeb6fb22a9abbb7cf63f8e7d3 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Tue, 11 Aug 2026 15:48:21 +0100 Subject: [PATCH 09/10] fix(kits): bump vendored Beam to 2.75.0 for the CVE backlog Pre-existing: the pipeline was vendored verbatim from GoogleCloudPlatform/firebase-extensions@68ef3fa, and Beam 2.51.0 (October 2023) is where all 29 high and 4 medium Wiz vulnerability findings on this PR come from. Its 195-dependency tree carried commons-compress 1.8.1 (2014), avro 1.8.2 (2017), jackson-core/mapper-asl 1.9.13 (Jackson 1.x, end of life), snakeyaml 1.33, netty 4.1.87 and protobuf-java 3.23.2. None of it is a porting regression; the npm side reports no high or critical findings. 2.75.0 moves commons-compress to 1.26.2, avro to 1.12.0, snakeyaml to 2.2, protobuf-java to 4.33.2, netty to 4.1.132 and jackson-databind to 2.18.6, and drops the Jackson 1.x artifacts from the tree entirely. The version is now a single beam.version property. Compiler source/target moves from 1.8 to 11, since Beam requires 11 and the flex template already builds on the JAVA11 base image. Verified by build only: all 7 sources compile and the shaded jar links against 2.75.0. Nothing here has been run on Dataflow, so a semantic change across the 24 minor versions would not have been caught. --- kits/firestore-incremental-capture/pipeline/pom.xml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/kits/firestore-incremental-capture/pipeline/pom.xml b/kits/firestore-incremental-capture/pipeline/pom.xml index fc7d15ce0..e7b424859 100644 --- a/kits/firestore-incremental-capture/pipeline/pom.xml +++ b/kits/firestore-incremental-capture/pipeline/pom.xml @@ -9,8 +9,9 @@ 1.0 - 1.8 - 1.8 + 11 + 11 + 2.75.0 restore-firestore @@ -97,23 +98,23 @@ org.apache.beam beam-sdks-java-core - 2.51.0 + ${beam.version} org.apache.beam beam-runners-google-cloud-dataflow-java - 2.51.0 + ${beam.version} org.apache.beam beam-sdks-java-io-google-cloud-platform - 2.51.0 + ${beam.version} org.apache.beam beam-runners-direct-java - 2.51.0 + ${beam.version} From 908576ea6e0225534f51536476908c2e891a4dab Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Tue, 11 Aug 2026 17:15:52 +0100 Subject: [PATCH 10/10] fix(kits): keep incremental capture working without a Storage bucket The bucket-detection fix coupled the capture path to Cloud Storage. The lazy context is shared by all five functions, and it resolved the bucket eagerly via getStorage().bucket().name, which throws on a project that never enabled Storage. That took down syncData, syncChangelogTask and initIncrementalCapture, none of which need a bucket, and with retry: true on the trigger every write then retried for days against a permanent failure. The extension captured fine on such projects: its capture path never read the bucket, and its lookup fell back rather than throwing. bucketName and flexTemplatePath are now optional. Resolution no longer throws when the bucket is unknown; the launcher raises an actionable error instead, so the failure lands on the restoration that needs it. Also from the same review: - Give syncChangelogTask and initIncrementalCapture the 512MiB/540s the extension allotted them. The v2 defaults of 256MiB/60s were a silent downgrade, and creating a BigQuery dataset and table can outlast 60s. - Document that a Timestamp, GeoPoint, DocumentReference or Buffer sitting directly in an array does not survive restoration. The extension preserved these as maps of their internals, restoring the wrong type but keeping the data. - Correct doc comments that still advertised {document=**} capture, which the trigger cannot deploy. Consumers of ./lib see the comment, not the README. --- kits/firestore-incremental-capture/README.md | 12 +++- .../src/capture-config.ts | 66 +++++++++++-------- .../src/dataflow.ts | 9 +++ .../src/index.ts | 34 ++++++++-- .../tests/capture-config.test.ts | 21 ++++-- .../tests/dataflow.test.ts | 13 ++++ 6 files changed, 115 insertions(+), 40 deletions(-) diff --git a/kits/firestore-incremental-capture/README.md b/kits/firestore-incremental-capture/README.md index 64dd8d46f..8392fa6a9 100644 --- a/kits/firestore-incremental-capture/README.md +++ b/kits/firestore-incremental-capture/README.md @@ -144,7 +144,7 @@ loads them at deploy time and prompts for any required values that are missing. | `location` | `LOCATION` | no | `us-central1` | Region for the functions and task queues | | `datasetLocation` | `DATASET_LOCATION` | no | `us` | BigQuery dataset location | | `dataflowRegion` | `DATAFLOW_REGION` | no | `LOCATION` | Region for Dataflow jobs | -| `bucketName` | `BUCKET_NAME` | no | default bucket | Bucket the flex template was staged to | +| `bucketName` | `BUCKET_NAME` | no | default bucket | Bucket the flex template was staged to; restoration only | | `logLevel` | `LOG_LEVEL` | no | `info` | `debug`, `info`, `warn`, `error`, `silent` | Three constraints are worth stating outright, because each one fails silently if @@ -163,7 +163,9 @@ you assume otherwise: than guessed from the project id. The default bucket is `.firebasestorage.app` for projects created after September 2024 and `.appspot.com` for older ones, and a wrong guess means - launching against a template that was never staged there. + launching against a template that was never staged there. Only restoration + reads it, so capture works on a project with no Storage bucket at all; the + error surfaces when a restoration is launched. ## Multiple instances @@ -220,6 +222,12 @@ silently drops any field whose tag it does not handle: top level, so `[1, 2]` restores as a list of empty maps. Arrays of maps do round trip — see the note in `src/serializer.ts` on why array elements are encoded differently from map fields. +- **A Timestamp, GeoPoint, DocumentReference or Buffer sitting _directly_ in an + array does not survive either**, for the same reason: its `{type, value}` + envelope is not a field map. The extension happened to preserve these as maps + of their internals (`_seconds`/`_nanoseconds` for a Timestamp), so they + restored with the wrong type but with the data present; here they restore as + empty maps. The same values nested inside a map element round trip correctly. - **Changelog replay writes to a malformed path.** `IncrementalCaptureLog.convertToFirestoreValue` applies `createDocumentName` to a path that has already been through it, producing a doubled diff --git a/kits/firestore-incremental-capture/src/capture-config.ts b/kits/firestore-incremental-capture/src/capture-config.ts index 82d227682..376c7801f 100644 --- a/kits/firestore-incremental-capture/src/capture-config.ts +++ b/kits/firestore-incremental-capture/src/capture-config.ts @@ -25,8 +25,11 @@ export interface CaptureConfig { /** GCP project holding the Firestore databases, BigQuery dataset and jobs. */ projectId: string; /** - * Collection to capture, relative to the database root. `{document=**}` - * captures every collection. + * Collection to capture, relative to the database root. + * + * A single collection only. A Firestore trigger accepts a multi-segment + * wildcard just as its final path segment, so `{document=**}` would build the + * undeployable pattern `{document=**}/{documentId}`. */ syncCollectionPath: string; /** @@ -45,14 +48,20 @@ export interface CaptureConfig { /** Region Dataflow jobs run in. Defaults to {@link CaptureConfig.location}. */ dataflowRegion?: string; /** - * Cloud Storage bucket holding the Dataflow flex template. Required, and not - * guessed: the default bucket is `.firebasestorage.app` for - * projects created after September 2024 and `.appspot.com` for - * older ones, and guessing wrong means restoration launches against a - * template that is not there. The entry point fills this from the project's - * actual default bucket when the `BUCKET_NAME` param is unset. + * Cloud Storage bucket holding the Dataflow flex template. + * + * Never guessed from the project id: the default bucket is + * `.firebasestorage.app` for projects created after September 2024 + * and `.appspot.com` for older ones, and guessing wrong means + * restoration launches against a template that is not there. The entry point + * fills this from the project's actual default bucket when the `BUCKET_NAME` + * param is unset. + * + * Optional because only restoration reads it. A project with no Storage + * bucket at all can still capture; the error surfaces when a restoration is + * launched, not on the write path. */ - bucketName: string; + bucketName?: string; /** * This instance's key in the `instances` map of the kit stanza. Required, and * must match exactly: the CLI deploys every function as @@ -68,8 +77,10 @@ export interface CaptureConfig { /** {@link CaptureConfig} with every default applied and paths derived. */ export interface ResolvedCaptureConfig - extends Required> { + extends Required> { dataflowRegion: string; + /** Absent when the project has no default bucket and none was configured. */ + bucketName?: string; /** * Database the changes are captured from. Always `(default)`: the restoration * pipeline reads its PITR baseline from `FirestoreOptions.getDefaultInstance()` @@ -81,9 +92,10 @@ export interface ResolvedCaptureConfig backupInstanceName: string; /** * Cloud Storage path of the Dataflow flex template spec. Built out-of-band by - * the setup script, which must write to this exact path. + * the setup script, which must write to this exact path. Absent when no + * bucket is known, which only blocks restoration. */ - flexTemplatePath: string; + flexTemplatePath?: string; /** Firestore document tracking the state of each restoration run. */ restoreCollection: string; } @@ -98,10 +110,13 @@ const DEFAULT_DATASET_LOCATION = "us"; * * @param config - Caller-supplied configuration. * @returns The fully resolved configuration. - * @throws If `instanceId` is empty, which would misname every task queue; if + * Deliberately does not require `bucketName`. Only restoration reads it, and + * throwing here would take the capture path down with it on a project that has + * no Storage bucket - which the extension captured on quite happily. + * + * @throws If `instanceId` is empty, which would misname every task queue, or if * `backupInstanceId` is empty or is the captured database, either of which - * would make a restoration write over the source it restores from; or if - * `bucketName` is empty, which would leave the flex template path unresolvable. + * would make a restoration write over the source it restores from. */ export function resolveCaptureConfig( config: CaptureConfig @@ -135,14 +150,7 @@ export function resolveCaptureConfig( ); } - if (!config.bucketName) { - invalid( - "BUCKET_NAME is required. It must name the bucket the Dataflow flex " + - "template was staged to by scripts/setup.sh." - ); - } - - const bucketName = config.bucketName; + const bucketName = config.bucketName || undefined; return { projectId: config.projectId, @@ -158,14 +166,20 @@ export function resolveCaptureConfig( instanceId, logLevel: config.logLevel || "info", backupInstanceName: `projects/${config.projectId}/databases/${config.backupInstanceId}`, - flexTemplatePath: `gs://${bucketName}/${instanceId}-dataflow-restore`, + flexTemplatePath: bucketName + ? `gs://${bucketName}/${instanceId}-dataflow-restore` + : undefined, restoreCollection: `_${instanceId}/runs/restorations`, }; } /** - * Collection id the Dataflow pipeline reads from. The pipeline takes `*` to - * mean every collection, where the Firestore trigger spells that `{document=**}`. + * Collection id the Dataflow pipeline reads from. The pipeline spells + * every-collection as `*`, where a Firestore trigger spells it `{document=**}`. + * + * The mapping is kept for callers that register their own trigger through + * `./lib`. It is unreachable from the wired `syncData` trigger, which cannot + * deploy a `{document=**}` pattern - see {@link CaptureConfig.syncCollectionPath}. * * @param syncCollectionPath - The configured collection path. * @returns The collection id in the pipeline's spelling. diff --git a/kits/firestore-incremental-capture/src/dataflow.ts b/kits/firestore-incremental-capture/src/dataflow.ts index c2ee0678f..613eacbc4 100644 --- a/kits/firestore-incremental-capture/src/dataflow.ts +++ b/kits/firestore-incremental-capture/src/dataflow.ts @@ -54,6 +54,15 @@ export class RestorationLauncher { */ async launch(request: RestorationRequest): Promise { const { config } = this; + + if (!config.flexTemplatePath) { + throw new Error( + "Cannot launch a restoration: no Cloud Storage bucket is configured. " + + "Set BUCKET_NAME to the bucket scripts/setup.sh staged the Dataflow " + + "flex template to." + ); + } + const runId = `${config.instanceId}-restore-${request.timestamp}`; logs.info(`Launching restoration job ${runId}`, { diff --git a/kits/firestore-incremental-capture/src/index.ts b/kits/firestore-incremental-capture/src/index.ts index 0dd2d8381..659ea4ba9 100644 --- a/kits/firestore-incremental-capture/src/index.ts +++ b/kits/firestore-incremental-capture/src/index.ts @@ -97,6 +97,25 @@ for (const role of REQUIRED_ROLES) { afterFirstDeploy({ task: { function: INIT_FUNCTION } }); afterRedeploy({ task: { function: INIT_FUNCTION } }); +/** + * The project's default Cloud Storage bucket, or `undefined` if it has none. + * + * Read from the initialized app rather than assembled from the project id: the + * default bucket is `.firebasestorage.app` for projects created after + * September 2024 and `.appspot.com` for older ones. + * + * Swallows the lookup failure because only restoration needs a bucket. A project + * that never enabled Storage has none, and letting this throw would take the + * capture path down with it - `getHandlerContext` is shared by every function. + */ +function defaultBucketName(): string | undefined { + try { + return getStorage().bucket().name; + } catch { + return undefined; + } +} + let ctx: HandlerContext | undefined; function getHandlerContext(): HandlerContext { @@ -108,12 +127,7 @@ function getHandlerContext(): HandlerContext { initializeApp(); } - // Read from the initialized app rather than assembled from the project id: - // the default bucket is .firebasestorage.app for projects created - // after September 2024 and .appspot.com for older ones. - const config = resolveCaptureConfig( - configFromEnv(getStorage().bucket().name) - ); + const config = resolveCaptureConfig(configFromEnv(defaultBucketName())); logs.setLogLevel(config.logLevel); @@ -165,6 +179,10 @@ export const syncData = onDocumentWritten( export const syncChangelogTask = onTaskDispatched( { ...functionOptions, + // Matches the extension's allowance for this function; the v2 defaults + // (256MiB/60s) would be a silent downgrade. + memory: "512MiB", + timeoutSeconds: 540, retryConfig: { maxAttempts: 15, minBackoffSeconds: 10 }, }, (request) => handleChangelogTask(request.data, getHandlerContext()) @@ -219,6 +237,10 @@ export const runRestorationTask = onTaskDispatched( export const initIncrementalCapture = onTaskDispatched( { ...functionOptions, + // As the extension's runInitialSetup: creating a dataset and table can be + // slow, and the v2 default 60s timeout would cut it short. + memory: "512MiB", + timeoutSeconds: 540, retryConfig: LIFECYCLE_RETRY_CONFIG, }, async () => { diff --git a/kits/firestore-incremental-capture/tests/capture-config.test.ts b/kits/firestore-incremental-capture/tests/capture-config.test.ts index c33be986c..7d0f357d2 100644 --- a/kits/firestore-incremental-capture/tests/capture-config.test.ts +++ b/kits/firestore-incremental-capture/tests/capture-config.test.ts @@ -64,12 +64,21 @@ describe("resolveCaptureConfig", () => { expect(resolved.dataflowRegion).toBe("us-central1"); }); - test("requires an explicit bucket rather than guessing one", () => { - // Guessing is unsafe: the default bucket domain differs by project age, and - // a wrong guess means launching against a template that is not staged there. - expect(() => resolveCaptureConfig(config({ bucketName: "" }))).toThrow( - /BUCKET_NAME is required/ - ); + test("resolves without a bucket, so capture works without Cloud Storage", () => { + // Only restoration needs a bucket. Throwing here would take the capture + // path down on a project that never enabled Storage. + const resolved = resolveCaptureConfig(config({ bucketName: "" })); + + expect(resolved.bucketName).toBeUndefined(); + expect(resolved.flexTemplatePath).toBeUndefined(); + }); + + test("never guesses a bucket name from the project id", () => { + // The default bucket domain differs by project age, and a wrong guess means + // launching against a template that was never staged there. + const resolved = resolveCaptureConfig(config({ bucketName: "" })); + + expect(JSON.stringify(resolved)).not.toContain("test-project."); }); test("pins the captured database to the only one the pipeline can restore", () => { diff --git a/kits/firestore-incremental-capture/tests/dataflow.test.ts b/kits/firestore-incremental-capture/tests/dataflow.test.ts index 080f0a157..bc488ca6a 100644 --- a/kits/firestore-incremental-capture/tests/dataflow.test.ts +++ b/kits/firestore-incremental-capture/tests/dataflow.test.ts @@ -183,6 +183,19 @@ describe("RestorationLauncher", () => { }); }); + test("fails with an actionable error when no bucket is configured", async () => { + const client = fakeClient(); + + await expect( + new RestorationLauncher( + config({ bucketName: "" }), + client as never + ).launch({ timestamp: 1700000000 }) + ).rejects.toThrow(/no Cloud Storage bucket is configured/); + + expect(client.launchFlexTemplate).not.toHaveBeenCalled(); + }); + test("records a null job name when Dataflow reports none", async () => { const client = fakeClient({});