feat(config,server,cli,watch): support global general.callback_url (#1481)

ChangeLog.md

@@ -1,8 +1,11 @@
 # v2.8.1
 NEW FEATURES
+- add `general.callback_url` (env `CALLBACK_URL`) and `general.callback_timeout` (env `CALLBACK_TIMEOUT`, default `5s`) — HTTP POST completion notification for API, one-shot CLI commands, and each `watch` iteration; API `?callback=` overrides the global URL when non-empty; the payload keeps the existing `status`/`error`/`operation_id` fields and adds `command` and `duration`; `status` is now also `cancel` when the operation was killed via `/backup/kill`; callbacks are sent asynchronously, failures are logged and never change the backup result; one-shot CLI commands and `watch` iterations are now registered in `/backup/status` like API operations, fix [#1481](https://github.com/Altinity/clickhouse-backup/issues/1481)
+- add `general.status_history_size` (env `STATUS_HISTORY_SIZE`, default `1000`) — upper bound on how many finished operations are kept in the in-memory status list exposed by `/backup/status` and `system.backup_actions`; needed because `watch` now records one operation per iteration, so the history would otherwise grow for as long as the process lives; operations still running are never dropped, whatever their age
 - `delete local|remote <backup_name>` and `POST /backup/delete/{where}/{name}` now refuse to delete a backup which other backups require via `required_backup` and report the dependent backup names, instead of silently breaking the incremental backups chain (the breakage surfaced only later, when a descendant was downloaded or restored, and for object disks the descendant `required` parts blobs were deleted together with the parent); pass `--force` (`force=1` for the API) to get the old behavior, or set `general.rebase_during_delete: true` (env `REBASE_DURING_DELETE`, default `false`) to rebase every dependent increment first (same as the `rebase` command) so the chain stays restorable — rebase copies the deleted backup parts into its dependents, so deletion time grows with the copied data size and a rebase failure aborts the delete. `backups_to_keep_local`/`backups_to_keep_remote` retention is not affected, fix [#1493](https://github.com/Altinity/clickhouse-backup/issues/1493)
 
 BUG FIXES
+- `GET /backup/kill` and `kill` in `POST /backup/actions` now actually cancel commands which were started through `POST /backup/actions` — such commands re-enter the CLI app in process with `--command-id` passed *before* the command name, so the value lands in the application flag set, but every command action read it with `c.Int("command-id")`, which in urfave/cli v1 only looks at the command's own flag set (each command re-declares the application flags via `Flags: append(cliapp.Flags, ...)`, so it saw the `-1` default). The command therefore ran with a fresh background context instead of the one owned by its status row and ignored cancellation; broken since [0a26ee69](https://github.com/Altinity/clickhouse-backup/commit/0a26ee69) (v2.1.0), which moved `command-id` from the commands to the application but kept the `c.Int` lookups
 - drop a stale `upload.state2` when `upload --resume` runs for a backup which doesn't exist on remote storage anymore — the resumable state survives a successful upload and is removed only together with the local backup, so `create_remote --resume` + `delete remote` + `upload --resume` skipped every data file and uploaded a backup containing `metadata.json` only, which looked valid in `list remote`; an interrupted upload still leaves the backup folder on remote, so a real resume is not affected, fix [#1492](https://github.com/Altinity/clickhouse-backup/issues/1492)
 
 # v2.8.0

ReadMe.md

@@ -172,6 +172,19 @@ general:
   retries_jitter: 30             # RETRIES_JITTER, percent of RETRIES_PAUSE for jitter to avoid same time retries from parallel operations
   delete_batch_size: 1000        # DELETE_BATCH_SIZE, default batch size for bulk DeleteObjects() requests in remote storages that support batch delete (e.g. S3); upper bound for one API call 
 
+  # callback_url - CALLBACK_URL, optional HTTP endpoint notified with POST application/json when a backup command completes
+  # (API, one-shot CLI commands, and each watch-loop iteration). API `?callback=` overrides this when non-empty.
+  # Payload always includes status (success|error|cancel), error (empty string on success), operation_id, command and duration.
+  # Read-only commands (list, tables, status, ...) never send a callback.
+  # Callbacks are sent asynchronously; failures are logged and never change the backup command exit code / result.
+  callback_url: ""
+  callback_timeout: 5s           # CALLBACK_TIMEOUT, max wait for the completion callback HTTP POST
+
+  # status_history_size - STATUS_HISTORY_SIZE, how many finished operations are kept in the in-memory
+  # status list exposed by `/backup/status` and `system.backup_actions`. `watch` records one operation
+  # per iteration, so the history needs an upper bound. Operations still running are never dropped.
+  status_history_size: 1000
+
   watch_interval: 1h       # WATCH_INTERVAL, use only for `watch` command, backup will create every 1h
   full_interval: 24h       # FULL_INTERVAL, use only for `watch` command, full backup will create every 24h
   watch_backup_name_template: "shard{shard}-{type}-{time:20060102150405}" # WATCH_BACKUP_NAME_TEMPLATE, used only for `watch` command, macros values will apply from `system.macros` for time:XXX, look format in https://go.dev/src/time/format.go
@@ -542,7 +555,7 @@ Create new backup: `curl -s localhost:7171/backup/create -X POST | jq .`
 - Optional boolean query argument `configs-only` or `configs_only` works the same as the `--configs-only` CLI argument (backup only configs).
 - Optional boolean query argument `skip-check-parts-columns` or `skip_check_parts_columns` works the same as the `--skip-check-parts-columns` CLI argument (allow backup inconsistent column types for data parts).
 - Optional boolean query argument `resume` works the same as the `--resume` CLI argument (resume upload for object disk data).
-- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success","error":"not empty when error happens", "operation_id" : "<random_uuid>"}`.
+- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success|cancel","error":"not empty when error happens", "operation_id" : "<random_uuid>", "command":"<full command line>", "duration":"<elapsed>"}`. When omitted or empty, falls back to `general.callback_url` if configured.
 
 Additional example: `curl -s 'localhost:7171/backup/create?table=default.billing&name=billing_test' -X POST`
 
@@ -566,7 +579,7 @@ Create new backup and upload to remote storage: `curl -s localhost:7171/backup/c
 - Optional string query argument `skip-projections` or `skip_projections` works the same as the `--skip-projections` CLI argument.
 - Optional boolean query argument `delete-source` or `delete_source` works the same as `--delete-source` CLI argument.
 - Optional boolean query argument `resume` works the same as the `--resume` CLI argument (resume upload for object disk data).
-- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success","error":"not empty when error happens", "operation_id" : "<random_uuid>"}`.
+- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success|cancel","error":"not empty when error happens", "operation_id" : "<random_uuid>", "command":"<full command line>", "duration":"<elapsed>"}`. When omitted or empty, falls back to `general.callback_url` if configured.
 
 Note: this operation is asynchronous, so the API will return once the operation has started. The response includes an `operation_id` field that can be used to track the operation status via `/backup/status?operationid=<operation_id>`.
 
@@ -618,7 +631,7 @@ Upload backup to remote storage: `curl -s localhost:7171/backup/upload/<BACKUP_N
 - Optional boolean query argument `configs-only` works the same as the `--configs-only` CLI argument (upload configs
   only).
 - Optional boolean query argument `resumable` works the same as the `--resumable` CLI argument (save intermediate upload state and resume upload if data already exists on remote storage).
-- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success","error":"not empty when error happens", "operation_id" : "<random_uuid>"}`.
+- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success|cancel","error":"not empty when error happens", "operation_id" : "<random_uuid>", "command":"<full command line>", "duration":"<elapsed>"}`. When omitted or empty, falls back to `general.callback_url` if configured.
 
 Note: this operation is asynchronous, so the API will return once the operation has started. The response includes an `operation_id` field that can be used to track the operation status via `/backup/status?operationid=<operation_id>`.
 
@@ -642,15 +655,15 @@ Download backup from remote storage: `curl -s localhost:7171/backup/download/<BA
 - Optional boolean query argument `configs-only` works the same as the `--configs-only` CLI argument (download configs
   only).
 - Optional boolean query argument `resumable` works the same as the `--resumable` CLI argument (save intermediate download state and resume download if it already exists on local storage).
-- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success","error":"not empty when error happens", "operation_id" : "<random_uuid>"}`.
+- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success|cancel","error":"not empty when error happens", "operation_id" : "<random_uuid>", "command":"<full command line>", "duration":"<elapsed>"}`. When omitted or empty, falls back to `general.callback_url` if configured.
 
 Note: this operation is asynchronous, so the API will return once the operation has started. The response includes an `operation_id` field that can be used to track the operation status via `/backup/status?operationid=<operation_id>`.
 
 ### POST /backup/rebase
 
 Copy required parts from the `required_backup` chain into remote backup and remove the `required_backup` dependency, so the incremental backup becomes a full one: `curl -s localhost:7171/backup/rebase/<BACKUP_NAME> -X POST | jq .`
 
-- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success","error":"not empty when error happens", "operation_id" : "<random_uuid>"}`.
+- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success|cancel","error":"not empty when error happens", "operation_id" : "<random_uuid>", "command":"<full command line>", "duration":"<elapsed>"}`. When omitted or empty, falls back to `general.callback_url` if configured.
 
 Note: this operation is asynchronous, so the API will return once the operation has started. The response includes an `operation_id` field that can be used to track the operation status via `/backup/status?operationid=<operation_id>`.
 
@@ -660,7 +673,7 @@ Move data parts inside local backup between disks to match the current `system.p
 
 - Optional string query argument `table` works the same as the `--tables value` CLI argument.
 - Optional boolean query argument `dry-run` works the same as the `--dry-run` CLI argument (only log which parts would move between disks, change nothing).
-- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success","error":"not empty when error happens", "operation_id" : "<random_uuid>"}`.
+- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success|cancel","error":"not empty when error happens", "operation_id" : "<random_uuid>", "command":"<full command line>", "duration":"<elapsed>"}`. When omitted or empty, falls back to `general.callback_url` if configured.
 
 Note: this operation is asynchronous, so the API will return once the operation has started. The response includes an `operation_id` field that can be used to track the operation status via `/backup/status?operationid=<operation_id>`.
 
@@ -684,7 +697,7 @@ Create schema and restore data from backup: `curl -s localhost:7171/backup/resto
 - Optional boolean query argument `resume` works the same as the `--resume` CLI argument (resume download for object disk data).
 - Optional boolean query argument `skip_empty_tables` or `skip-empty-tables` works the same as the `--skip-empty-tables` CLI argument (skip restoring tables that have no data).
 - Optional boolean query argument `rebind_replica_path_if_exists` or `rebind-replica-path-if-exists` works the same as the `--rebind-replica-path-if-exists` CLI argument (overrides `clickhouse.rebind_replica_path_if_exists` for this request, rebind a restored ReplicatedMergeTree to `default_replica_path` when the original ZK path still has leftover state but our replica entry is absent). WARNING: never set during a concurrent HA multi-replica restore.
-- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success","error":"not empty when error happens", "operation_id" : "<random_uuid>"}`.
+- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success|cancel","error":"not empty when error happens", "operation_id" : "<random_uuid>", "command":"<full command line>", "duration":"<elapsed>"}`. When omitted or empty, falls back to `general.callback_url` if configured.
 
 Note: this operation is asynchronous, so the API will return once the operation has started. The response includes an `operation_id` field that can be used to track the operation status via `/backup/status?operationid=<operation_id>`.
 
@@ -711,7 +724,7 @@ Download and restore data from remote backup: `curl -s localhost:7171/backup/res
 - Optional boolean query argument `hardlink_exists_files` or `hardlink-exists-files` works the same as the `--hardlink-exists-files` CLI argument (Create hardlinks for existing files instead of downloading).
 - Optional boolean query argument `skip_empty_tables` or `skip-empty-tables` works the same as the `--skip-empty-tables` CLI argument (skip restoring tables that have no data).
 - Optional boolean query argument `rebind_replica_path_if_exists` or `rebind-replica-path-if-exists` works the same as the `--rebind-replica-path-if-exists` CLI argument (overrides `clickhouse.rebind_replica_path_if_exists` for this request, rebind a restored ReplicatedMergeTree to `default_replica_path` when the original ZK path still has leftover state but our replica entry is absent). WARNING: never set during a concurrent HA multi-replica restore.
-- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success","error":"not empty when error happens", "operation_id" : "<random_uuid>"}`.
+- Optional string query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success|cancel","error":"not empty when error happens", "operation_id" : "<random_uuid>", "command":"<full command line>", "duration":"<elapsed>"}`. When omitted or empty, falls back to `general.callback_url` if configured.
 
 Note: this operation is asynchronous, so the API will return once the operation has started. The response includes an `operation_id` field that can be used to track the operation status via `/backup/status?operationid=<operation_id>`.
 

cmd/clickhouse-backup/cli_callback.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"strings"
+
+	"github.com/Altinity/clickhouse-backup/v2/pkg/config"
+	"github.com/Altinity/clickhouse-backup/v2/pkg/status"
+
+	"github.com/google/uuid"
+	"github.com/urfave/cli"
+)
+
+// registerCLIStatus wraps every command Action so a one-shot CLI run registers
+// itself in status.Current, exactly like an API request does. Completion
+// callbacks are then emitted from the single place which owns them,
+// status.AsyncStatus.Stop, instead of a CLI specific dispatcher.
+//
+// Which commands are worth notifying about is decided by status.CallbackEligible,
+// this package deliberately holds no list of command names.
+func registerCLIStatus(commands []cli.Command) {
+	for i := range commands {
+		cmd := &commands[i]
+		registerCLIStatus(cmd.Subcommands)
+		if cmd.Action == nil || !status.CallbackEligible(cmd.Name) {
+			continue
+		}
+		action := cmd.Action
+		name := cmd.Name
+		cmd.Action = func(c *cli.Context) error {
+			return runWithCLIStatus(c, name, action)
+		}
+	}
+}
+
+func runWithCLIStatus(c *cli.Context, name string, action interface{}) error {
+	// The API server re-enters this same cli.App in process, see
+	// APIServer.httpBackupActionsHandler. Such runs are already tracked and
+	// notified by the handler which started them — or deliberately untracked, when
+	// the command is listed in api.backup_actions_skip_commands, in which case
+	// --command-id is status.NotFromAPI and only the server mode marker tells the
+	// two apart.
+	if status.APIServerMode() || commandIdFromCli(c) != status.NotFromAPI {
+		return cli.HandleAction(action, c)
+	}
+	cfg := config.GetConfigFromCli(c)
+	commandId, _ := status.Current.StartWithCallback(cliFullCommand(c, name), uuid.NewString(), cliCallback(cfg))
+	err := cli.HandleAction(action, c)
+	status.Current.Stop(commandId, err)
+	return err
+}
+
+// cliFullCommand renders the command the way API handlers do, name first so
+// status.CallbackEligible and /backup/status filters see the same shape.
+func cliFullCommand(c *cli.Context, name string) string {
+	if args := c.Args(); args.Present() {
+		return name + " " + strings.Join(args, " ")
+	}
+	return name
+}
+
+func cliCallback(cfg *config.Config) *status.CallbackConfig {
+	if cfg == nil || cfg.General.CallbackURL == "" {
+		return nil
+	}
+	return &status.CallbackConfig{
+		URLs:    []string{cfg.General.CallbackURL},
+		Timeout: cfg.General.CallbackTimeoutDuration,
+	}
+}