S3 auth via AWS profile / boto3 credential chain

README.md

@@ -25,10 +25,22 @@ It is important to use `--s3diskname` if your disk name is not `s3` which is by
 WARNING!: Please use `--dry-run` to check and compare results of what is going to be deleted, just to be on the safe side. 
 
 ## script invocation
+### install
+Install the Python dependencies, ideally in a virtualenv:
+```
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+On Debian/Ubuntu/WSL you may first need `sudo apt install -y python3-pip python3-venv`.
+Without a virtualenv, use `pip install --user -r requirements.txt` (add
+`--break-system-packages` if pip refuses on an externally-managed Python).
+
 ### help
 ```
 python3 s3gc.py --help
 ```
+
 ### typical usage
 #### all together with dry-run
 for https://altinity-clickhouse-data-demo20565656565620663600000001.s3.amazonaws.com/github
@@ -43,6 +55,30 @@ S3GC_S3PATH=github/ \
 S3GC_S3SECURE_FLAG=true \
 python3 ./s3gc.py --verbose --dry-run
 ```
+#### AWS SSO or AWS profile credentials
+Authenticate with AWS CLI first, then let `s3gc` resolve temporary credentials through the boto3 credential chain.
+```
+aws sso login --profile my-sso-profile
+
+S3GC_S3AUTH=aws \
+S3GC_S3PROFILE=my-sso-profile \
+S3GC_S3IP=s3.amazonaws.com \
+S3GC_S3PORT=443 \
+S3GC_S3REGION=us-east-1 \
+S3GC_S3BUCKET=altinity-clickhouse-data-demo20565656565620663600000001 \
+S3GC_S3PATH=github/ \
+S3GC_S3SECURE_FLAG=true \
+python3 ./s3gc.py --verbose --dry-run
+```
+
+`S3GC_S3ACCESSKEY` and `S3GC_S3SECRETKEY` are not used with `S3GC_S3AUTH=aws`.
+The selected credentials must allow `s3:ListBucket` on the bucket for
+`S3GC_S3PATH`, even for `--dry-run`. Verify the same profile with:
+```
+aws sts get-caller-identity --profile my-sso-profile
+aws s3api list-objects-v2 --bucket altinity-clickhouse-data-demo20565656565620663600000001 --prefix github/ --max-keys 1 --profile my-sso-profile
+```
+
 #### GCS and object storage that do not support batch delete operations
 ```
 S3GC_S3ACCESSKEY=GOOG1xxxxxxxxx \
@@ -94,6 +130,9 @@ sudo docker run --network="host" -e S3GC_S3PORT=19000 -e S3GC_S3ACCESSKEY=minio9
 ### v_0.2 Fri Jan 31 2025
 - added option to avoid batch deletion for services like GCS
 
+### v_0.3 Mon Jun 15 2026
+- added s3 profile option
+
 ## to do list
 ~~1. option to avoid `remove_objects` which is reportedly not supported by GCE~~
 

requirements.txt

@@ -1,3 +1,4 @@
 Minio
+boto3
 clickhouse_connect
 jsonargparse[all]

s3gc.py

@@ -19,6 +19,7 @@
 from io import StringIO
 from minio import Minio
 from minio.deleteobjects import DeleteObject
+from minio.error import S3Error
 from contextlib import redirect_stdout
 import clickhouse_connect
 
@@ -31,7 +32,6 @@
 import urllib3
 import logging
 import datetime
-from distutils.util import strtobool
 
 usage = """
     s3 garbage collector for ClickHouse
@@ -102,6 +102,27 @@
     default="",
     help="S3 secret key",
 )
+parser.add_argument(
+    "--s3-session-token",
+    "--s3sessiontoken",
+    dest="s3sessiontoken",
+    default="",
+    help="S3 session token for explicit temporary credentials",
+)
+parser.add_argument(
+    "--s3auth",
+    "--s3-auth",
+    dest="s3auth",
+    default="static",
+    help="S3 auth mode: static or aws. aws uses the boto3 credential chain, including AWS SSO profiles",
+)
+parser.add_argument(
+    "--s3profile",
+    "--s3-profile",
+    dest="s3profile",
+    default="",
+    help="AWS profile name for S3 auth. Setting this enables aws auth mode",
+)
 parser.add_argument(
     "--s3secure",
     "--s3-secure",
@@ -446,10 +467,14 @@ class LogFormatter(logging.Formatter):
 
     def get_filter_strings():
         filter_strings = []
-        if len(args.chpass) > 3:
-            filter_strings.append(args.chpass)
-        if len(args.s3secretkey) > 3:
-            filter_strings.append(args.s3secretkey)
+        for secret in [
+            args.chpass,
+            args.s3accesskey,
+            args.s3secretkey,
+            args.s3sessiontoken,
+        ]:
+            if len(secret) > 3:
+                filter_strings.append(secret)
         return filter_strings
 
     filter_strings = get_filter_strings()
@@ -500,6 +525,19 @@ def graceful_exit():
 ch_client = None
 
 
+class UserVisibleError(RuntimeError):
+    pass
+
+
+def strtobool(value):
+    value = value.lower()
+    if value in ["y", "yes", "t", "true", "on", "1"]:
+        return True
+    if value in ["n", "no", "f", "false", "off", "0"]:
+        return False
+    raise ValueError(f"invalid truth value {value}")
+
+
 def connect_to_ch():
     logger.info(
         f"Connecting to ClickHouse, host={args.chhost}, port={args.chport}, username={args.chuser}, password={args.chpass}, s3path={args.s3path}, bucket={args.s3bucket}, s3path={args.s3path}"
@@ -514,25 +552,105 @@ def connect_to_ch():
     )
 
 
+def resolve_static_s3_credentials():
+    if bool(args.s3accesskey) != bool(args.s3secretkey):
+        raise ValueError("s3accesskey and s3secretkey must be specified together")
+    if args.s3sessiontoken and not args.s3accesskey:
+        raise ValueError("s3sessiontoken requires s3accesskey and s3secretkey")
+
+    if args.s3accesskey:
+        return args.s3accesskey, args.s3secretkey, args.s3sessiontoken or None, args.s3region, "static"
+
+    return None, None, None, args.s3region, "anonymous"
+
+
+def resolve_aws_s3_credentials():
+    if args.s3accesskey or args.s3secretkey or args.s3sessiontoken:
+        raise ValueError("s3auth=aws cannot be combined with explicit S3 access keys")
+
+    try:
+        import boto3
+    except ImportError as exc:
+        raise UserVisibleError("boto3 is required for s3auth=aws") from exc
+
+    session = boto3.Session(
+        profile_name=args.s3profile or None,
+        region_name=args.s3region,
+    )
+    credentials = session.get_credentials()
+    if credentials is None:
+        profile_hint = f" profile {args.s3profile}" if args.s3profile else ""
+        raise UserVisibleError(f"unable to resolve AWS credentials{profile_hint}")
+
+    frozen_credentials = credentials.get_frozen_credentials()
+    if not frozen_credentials.access_key or not frozen_credentials.secret_key:
+        profile_hint = f" profile {args.s3profile}" if args.s3profile else ""
+        raise UserVisibleError(f"resolved AWS credentials{profile_hint} are incomplete")
+
+    return (
+        frozen_credentials.access_key,
+        frozen_credentials.secret_key,
+        frozen_credentials.token,
+        args.s3region or session.region_name,
+        "aws",
+    )
+
+
+def resolve_s3_credentials():
+    auth_mode = args.s3auth.lower()
+    if auth_mode not in ["static", "aws"]:
+        raise ValueError("s3auth must be static or aws")
+    if args.s3profile:
+        auth_mode = "aws"
+
+    if auth_mode == "aws":
+        return resolve_aws_s3_credentials()
+
+    return resolve_static_s3_credentials()
+
+
 def connect_to_s3():
     if args.s3secure_flag:
         logger.debug(f"using SSL certificate {args.s3sslcertfile}")
         os.environ["SSL_CERT_FILE"] = args.s3sslcertfile
 
+    access_key, secret_key, session_token, s3_region, s3_auth = resolve_s3_credentials()
     logger.info(
-        f"Connecting to S3, host:port={args.s3ip}:{args.s3port}, access_key={args.s3accesskey}, secret_key={args.s3secretkey}, secure={args.s3secure_flag}, region={args.s3region}"
+        f"Connecting to S3, host:port={args.s3ip}:{args.s3port}, auth={s3_auth}, secure={args.s3secure_flag}, region={s3_region}"
     )
     global minio_client
     minio_client = Minio(
         f"{args.s3ip}:{args.s3port}",
-        access_key=args.s3accesskey,
-        secret_key=args.s3secretkey,
+        access_key=access_key,
+        secret_key=secret_key,
+        session_token=session_token,
         secure=args.s3secure_flag,
-        region=args.s3region,
+        region=s3_region,
         http_client=urllib3.PoolManager(cert_reqs="CERT_NONE"),
     )
 
 
+def format_s3_list_error(exc):
+    code = getattr(exc, "code", "unknown")
+    message = getattr(exc, "message", str(exc))
+    profile_arg = f" --profile {args.s3profile}" if args.s3profile else ""
+    return (
+        f"unable to list S3 objects for bucket={args.s3bucket!r}, prefix={args.s3path!r}: "
+        f"{code}: {message}. "
+        f"s3gc collection requires s3:ListBucket on arn:aws:s3:::{args.s3bucket} "
+        f"for this prefix, even with --dry-run. Verify the same credentials with: "
+        f"aws sts get-caller-identity{profile_arg}; "
+        f"aws s3api list-objects-v2 --bucket {args.s3bucket} --prefix {args.s3path} --max-keys 1{profile_arg}"
+    )
+
+
+def next_s3_object(objects):
+    try:
+        return next(objects)
+    except S3Error as exc:
+        raise UserVisibleError(format_s3_list_error(exc)) from exc
+
+
 def do_collect():
     logger.debug(f"start_after {args.collectafter}")
     objects = minio_client.list_objects(
@@ -568,7 +686,7 @@ def do_collect():
         objs = []
         for batch_element in range(0, args.collectbatchsize):
             try:
-                obj = next(objects)
+                obj = next_s3_object(objects)
                 delta = datetime.datetime.now(datetime.timezone.utc) - obj.last_modified
                 hours = int(delta.seconds / 3600)
                 if hours >= args.age:
@@ -713,15 +831,22 @@ def make_antijoin(calc_only=False, sample=None):
 
 
 def main():
-    connect_to_ch()
-    if not (args.usecollected_flag and args.dryrun_flag):
-        connect_to_s3()
-    if not args.usecollected_flag:
-        do_collect()
-    if not args.collectonly_flag:
-        do_use()
-
-    graceful_exit()
+    try:
+        connect_to_ch()
+        if not (args.usecollected_flag and args.dryrun_flag):
+            connect_to_s3()
+        if not args.usecollected_flag:
+            do_collect()
+        if not args.collectonly_flag:
+            do_use()
+
+        graceful_exit()
+    except UserVisibleError as exc:
+        if args.debug_flag:
+            logger.exception(str(exc))
+        else:
+            logger.error(str(exc))
+        sys.exit(1)
 
 
 if __name__ == "__main__":