From 12026d76e6934a5f678c56f33c1b393db79324d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20B=C4=85k?= Date: Thu, 11 Jun 2026 17:31:41 +0200 Subject: [PATCH] S3 auth via AWS profile / boto3 credential chain Add --s3auth=aws and --s3profile so s3gc can resolve temporary session-token credentials (e.g. from AWS SSO) instead of static keys; --s3-session-token for explicit temporary credentials --- README.md | 39 ++++++++++++ requirements.txt | 1 + s3gc.py | 163 +++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 184 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 4ed0c77..b1cabd7 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,22 @@ It is important to use `--s3diskname` if your disk name is not `s3` which is by WARNING!: Please use `--dry-run` to check and compare results of what is going to be deleted, just to be on the safe side. ## script invocation +### install +Install the Python dependencies, ideally in a virtualenv: +``` +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +``` +On Debian/Ubuntu/WSL you may first need `sudo apt install -y python3-pip python3-venv`. +Without a virtualenv, use `pip install --user -r requirements.txt` (add +`--break-system-packages` if pip refuses on an externally-managed Python). + ### help ``` python3 s3gc.py --help ``` + ### typical usage #### all together with dry-run for https://altinity-clickhouse-data-demo20565656565620663600000001.s3.amazonaws.com/github @@ -43,6 +55,30 @@ S3GC_S3PATH=github/ \ S3GC_S3SECURE_FLAG=true \ python3 ./s3gc.py --verbose --dry-run ``` +#### AWS SSO or AWS profile credentials +Authenticate with AWS CLI first, then let `s3gc` resolve temporary credentials through the boto3 credential chain. +``` +aws sso login --profile my-sso-profile + +S3GC_S3AUTH=aws \ +S3GC_S3PROFILE=my-sso-profile \ +S3GC_S3IP=s3.amazonaws.com \ +S3GC_S3PORT=443 \ +S3GC_S3REGION=us-east-1 \ +S3GC_S3BUCKET=altinity-clickhouse-data-demo20565656565620663600000001 \ +S3GC_S3PATH=github/ \ +S3GC_S3SECURE_FLAG=true \ +python3 ./s3gc.py --verbose --dry-run +``` + +`S3GC_S3ACCESSKEY` and `S3GC_S3SECRETKEY` are not used with `S3GC_S3AUTH=aws`. +The selected credentials must allow `s3:ListBucket` on the bucket for +`S3GC_S3PATH`, even for `--dry-run`. Verify the same profile with: +``` +aws sts get-caller-identity --profile my-sso-profile +aws s3api list-objects-v2 --bucket altinity-clickhouse-data-demo20565656565620663600000001 --prefix github/ --max-keys 1 --profile my-sso-profile +``` + #### GCS and object storage that do not support batch delete operations ``` S3GC_S3ACCESSKEY=GOOG1xxxxxxxxx \ @@ -94,6 +130,9 @@ sudo docker run --network="host" -e S3GC_S3PORT=19000 -e S3GC_S3ACCESSKEY=minio9 ### v_0.2 Fri Jan 31 2025 - added option to avoid batch deletion for services like GCS +### v_0.3 Mon Jun 15 2026 +- added s3 profile option + ## to do list ~~1. option to avoid `remove_objects` which is reportedly not supported by GCE~~ diff --git a/requirements.txt b/requirements.txt index 839b8f5..81a157d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ Minio +boto3 clickhouse_connect jsonargparse[all] diff --git a/s3gc.py b/s3gc.py index f668589..9ea461b 100644 --- a/s3gc.py +++ b/s3gc.py @@ -19,6 +19,7 @@ from io import StringIO from minio import Minio from minio.deleteobjects import DeleteObject +from minio.error import S3Error from contextlib import redirect_stdout import clickhouse_connect @@ -31,7 +32,6 @@ import urllib3 import logging import datetime -from distutils.util import strtobool usage = """ s3 garbage collector for ClickHouse @@ -102,6 +102,27 @@ default="", help="S3 secret key", ) +parser.add_argument( + "--s3-session-token", + "--s3sessiontoken", + dest="s3sessiontoken", + default="", + help="S3 session token for explicit temporary credentials", +) +parser.add_argument( + "--s3auth", + "--s3-auth", + dest="s3auth", + default="static", + help="S3 auth mode: static or aws. aws uses the boto3 credential chain, including AWS SSO profiles", +) +parser.add_argument( + "--s3profile", + "--s3-profile", + dest="s3profile", + default="", + help="AWS profile name for S3 auth. Setting this enables aws auth mode", +) parser.add_argument( "--s3secure", "--s3-secure", @@ -446,10 +467,14 @@ class LogFormatter(logging.Formatter): def get_filter_strings(): filter_strings = [] - if len(args.chpass) > 3: - filter_strings.append(args.chpass) - if len(args.s3secretkey) > 3: - filter_strings.append(args.s3secretkey) + for secret in [ + args.chpass, + args.s3accesskey, + args.s3secretkey, + args.s3sessiontoken, + ]: + if len(secret) > 3: + filter_strings.append(secret) return filter_strings filter_strings = get_filter_strings() @@ -500,6 +525,19 @@ def graceful_exit(): ch_client = None +class UserVisibleError(RuntimeError): + pass + + +def strtobool(value): + value = value.lower() + if value in ["y", "yes", "t", "true", "on", "1"]: + return True + if value in ["n", "no", "f", "false", "off", "0"]: + return False + raise ValueError(f"invalid truth value {value}") + + def connect_to_ch(): logger.info( f"Connecting to ClickHouse, host={args.chhost}, port={args.chport}, username={args.chuser}, password={args.chpass}, s3path={args.s3path}, bucket={args.s3bucket}, s3path={args.s3path}" @@ -514,25 +552,105 @@ def connect_to_ch(): ) +def resolve_static_s3_credentials(): + if bool(args.s3accesskey) != bool(args.s3secretkey): + raise ValueError("s3accesskey and s3secretkey must be specified together") + if args.s3sessiontoken and not args.s3accesskey: + raise ValueError("s3sessiontoken requires s3accesskey and s3secretkey") + + if args.s3accesskey: + return args.s3accesskey, args.s3secretkey, args.s3sessiontoken or None, args.s3region, "static" + + return None, None, None, args.s3region, "anonymous" + + +def resolve_aws_s3_credentials(): + if args.s3accesskey or args.s3secretkey or args.s3sessiontoken: + raise ValueError("s3auth=aws cannot be combined with explicit S3 access keys") + + try: + import boto3 + except ImportError as exc: + raise UserVisibleError("boto3 is required for s3auth=aws") from exc + + session = boto3.Session( + profile_name=args.s3profile or None, + region_name=args.s3region, + ) + credentials = session.get_credentials() + if credentials is None: + profile_hint = f" profile {args.s3profile}" if args.s3profile else "" + raise UserVisibleError(f"unable to resolve AWS credentials{profile_hint}") + + frozen_credentials = credentials.get_frozen_credentials() + if not frozen_credentials.access_key or not frozen_credentials.secret_key: + profile_hint = f" profile {args.s3profile}" if args.s3profile else "" + raise UserVisibleError(f"resolved AWS credentials{profile_hint} are incomplete") + + return ( + frozen_credentials.access_key, + frozen_credentials.secret_key, + frozen_credentials.token, + args.s3region or session.region_name, + "aws", + ) + + +def resolve_s3_credentials(): + auth_mode = args.s3auth.lower() + if auth_mode not in ["static", "aws"]: + raise ValueError("s3auth must be static or aws") + if args.s3profile: + auth_mode = "aws" + + if auth_mode == "aws": + return resolve_aws_s3_credentials() + + return resolve_static_s3_credentials() + + def connect_to_s3(): if args.s3secure_flag: logger.debug(f"using SSL certificate {args.s3sslcertfile}") os.environ["SSL_CERT_FILE"] = args.s3sslcertfile + access_key, secret_key, session_token, s3_region, s3_auth = resolve_s3_credentials() logger.info( - f"Connecting to S3, host:port={args.s3ip}:{args.s3port}, access_key={args.s3accesskey}, secret_key={args.s3secretkey}, secure={args.s3secure_flag}, region={args.s3region}" + f"Connecting to S3, host:port={args.s3ip}:{args.s3port}, auth={s3_auth}, secure={args.s3secure_flag}, region={s3_region}" ) global minio_client minio_client = Minio( f"{args.s3ip}:{args.s3port}", - access_key=args.s3accesskey, - secret_key=args.s3secretkey, + access_key=access_key, + secret_key=secret_key, + session_token=session_token, secure=args.s3secure_flag, - region=args.s3region, + region=s3_region, http_client=urllib3.PoolManager(cert_reqs="CERT_NONE"), ) +def format_s3_list_error(exc): + code = getattr(exc, "code", "unknown") + message = getattr(exc, "message", str(exc)) + profile_arg = f" --profile {args.s3profile}" if args.s3profile else "" + return ( + f"unable to list S3 objects for bucket={args.s3bucket!r}, prefix={args.s3path!r}: " + f"{code}: {message}. " + f"s3gc collection requires s3:ListBucket on arn:aws:s3:::{args.s3bucket} " + f"for this prefix, even with --dry-run. Verify the same credentials with: " + f"aws sts get-caller-identity{profile_arg}; " + f"aws s3api list-objects-v2 --bucket {args.s3bucket} --prefix {args.s3path} --max-keys 1{profile_arg}" + ) + + +def next_s3_object(objects): + try: + return next(objects) + except S3Error as exc: + raise UserVisibleError(format_s3_list_error(exc)) from exc + + def do_collect(): logger.debug(f"start_after {args.collectafter}") objects = minio_client.list_objects( @@ -568,7 +686,7 @@ def do_collect(): objs = [] for batch_element in range(0, args.collectbatchsize): try: - obj = next(objects) + obj = next_s3_object(objects) delta = datetime.datetime.now(datetime.timezone.utc) - obj.last_modified hours = int(delta.seconds / 3600) if hours >= args.age: @@ -713,15 +831,22 @@ def make_antijoin(calc_only=False, sample=None): def main(): - connect_to_ch() - if not (args.usecollected_flag and args.dryrun_flag): - connect_to_s3() - if not args.usecollected_flag: - do_collect() - if not args.collectonly_flag: - do_use() - - graceful_exit() + try: + connect_to_ch() + if not (args.usecollected_flag and args.dryrun_flag): + connect_to_s3() + if not args.usecollected_flag: + do_collect() + if not args.collectonly_flag: + do_use() + + graceful_exit() + except UserVisibleError as exc: + if args.debug_flag: + logger.exception(str(exc)) + else: + logger.error(str(exc)) + sys.exit(1) if __name__ == "__main__":