feat: add post3 s3 proxy for postgresql
Signed-off-by: kjuulh <contact@kjuulh.io>
This commit is contained in:
242
s3-compliance/run-s3-tests.sh
Executable file
242
s3-compliance/run-s3-tests.sh
Executable file
@@ -0,0 +1,242 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Run Ceph s3-tests against post3 (FS backend).
|
||||
#
|
||||
# Usage:
|
||||
# bash s3-compliance/run-s3-tests.sh # run tests
|
||||
# bash s3-compliance/run-s3-tests.sh --collect-only # dry-run: list matching tests
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
S3TESTS_DIR="$REPO_ROOT/s3-tests"
|
||||
SCRIPT_DIR="$REPO_ROOT/s3-compliance"
|
||||
|
||||
# --- Validate prerequisites ---------------------------------------------------
|
||||
|
||||
if [ ! -d "$S3TESTS_DIR" ]; then
|
||||
echo "ERROR: s3-tests submodule not found at $S3TESTS_DIR"
|
||||
echo "Run: git submodule update --init"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v python3 &>/dev/null; then
|
||||
echo "ERROR: python3 is required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- Pick a free port ---------------------------------------------------------
|
||||
|
||||
PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("",0)); print(s.getsockname()[1]); s.close()')
|
||||
echo "Using port $PORT"
|
||||
|
||||
# --- Temp data dir for FS backend ---------------------------------------------
|
||||
|
||||
DATA_DIR=$(mktemp -d)
|
||||
echo "Data dir: $DATA_DIR"
|
||||
|
||||
# --- Build post3-server -------------------------------------------------------
|
||||
|
||||
echo "Building post3-server (release)..."
|
||||
cargo build -p post3-server --release --quiet
|
||||
|
||||
BINARY="$REPO_ROOT/target/release/post3-server"
|
||||
if [ ! -x "$BINARY" ]; then
|
||||
echo "ERROR: binary not found at $BINARY"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- Generate s3tests.conf ----------------------------------------------------
|
||||
|
||||
CONF="$DATA_DIR/s3tests.conf"
|
||||
sed "s/__PORT__/$PORT/g" "$SCRIPT_DIR/s3tests.conf.template" > "$CONF"
|
||||
echo "Config: $CONF"
|
||||
|
||||
# --- Start the server ---------------------------------------------------------
|
||||
|
||||
export POST3_HOST="127.0.0.1:$PORT"
|
||||
"$BINARY" serve --backend fs --data-dir "$DATA_DIR/store" &
|
||||
SERVER_PID=$!
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "Stopping server (PID $SERVER_PID)..."
|
||||
kill "$SERVER_PID" 2>/dev/null || true
|
||||
wait "$SERVER_PID" 2>/dev/null || true
|
||||
echo "Cleaning up $DATA_DIR..."
|
||||
rm -rf "$DATA_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# --- Wait for the server to become ready --------------------------------------
|
||||
|
||||
echo "Waiting for server on port $PORT..."
|
||||
TRIES=0
|
||||
MAX_TRIES=60
|
||||
while ! curl -sf "http://127.0.0.1:$PORT/" >/dev/null 2>&1; do
|
||||
TRIES=$((TRIES + 1))
|
||||
if [ "$TRIES" -ge "$MAX_TRIES" ]; then
|
||||
echo "ERROR: server did not start within ${MAX_TRIES}s"
|
||||
exit 1
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
echo "Server is ready."
|
||||
|
||||
# --- Set up virtualenv for s3-tests -------------------------------------------
|
||||
|
||||
VENV_DIR="$S3TESTS_DIR/.venv"
|
||||
if [ ! -d "$VENV_DIR" ]; then
|
||||
echo "Creating virtualenv..."
|
||||
python3 -m venv "$VENV_DIR"
|
||||
fi
|
||||
source "$VENV_DIR/bin/activate"
|
||||
|
||||
# Install dependencies if needed
|
||||
if ! python3 -c "import boto3" 2>/dev/null; then
|
||||
echo "Installing s3-tests dependencies..."
|
||||
pip install --quiet -r "$S3TESTS_DIR/requirements.txt"
|
||||
fi
|
||||
|
||||
# --- Build the test filter expression -----------------------------------------
|
||||
|
||||
# Marker-based exclusions (features post3 doesn't implement)
|
||||
MARKER_EXCLUDE="not appendobject"
|
||||
MARKER_EXCLUDE+=" and not bucket_policy and not bucket_encryption"
|
||||
MARKER_EXCLUDE+=" and not bucket_logging and not checksum"
|
||||
MARKER_EXCLUDE+=" and not cloud_transition and not conditional_write"
|
||||
MARKER_EXCLUDE+=" and not cors and not encryption"
|
||||
MARKER_EXCLUDE+=" and not fails_strict_rfc2616"
|
||||
MARKER_EXCLUDE+=" and not iam_account and not iam_cross_account"
|
||||
MARKER_EXCLUDE+=" and not iam_role and not iam_tenant and not iam_user"
|
||||
MARKER_EXCLUDE+=" and not lifecycle and not lifecycle_expiration"
|
||||
MARKER_EXCLUDE+=" and not lifecycle_transition"
|
||||
MARKER_EXCLUDE+=" and not object_lock and not object_ownership"
|
||||
MARKER_EXCLUDE+=" and not role_policy and not session_policy"
|
||||
MARKER_EXCLUDE+=" and not user_policy and not group_policy"
|
||||
MARKER_EXCLUDE+=" and not s3select and not s3website"
|
||||
MARKER_EXCLUDE+=" and not s3website_routing_rules"
|
||||
MARKER_EXCLUDE+=" and not s3website_redirect_location"
|
||||
MARKER_EXCLUDE+=" and not sns and not sse_s3 and not storage_class"
|
||||
MARKER_EXCLUDE+=" and not tagging"
|
||||
MARKER_EXCLUDE+=" and not test_of_sts and not versioning and not delete_marker"
|
||||
MARKER_EXCLUDE+=" and not webidentity_test"
|
||||
MARKER_EXCLUDE+=" and not auth_aws2 and not auth_aws4 and not auth_common"
|
||||
|
||||
# Keyword-based exclusions (individual tests requiring unimplemented ops)
|
||||
KEYWORD_EXCLUDE="not anonymous and not presigned and not copy_object"
|
||||
KEYWORD_EXCLUDE+=" and not test_account_usage and not test_head_bucket_usage"
|
||||
KEYWORD_EXCLUDE+=" and not acl and not ACL and not grant"
|
||||
KEYWORD_EXCLUDE+=" and not logging and not notification"
|
||||
# Exclude features not yet implemented:
|
||||
# - access_bucket / bucket access control tests (require ACL/policy)
|
||||
KEYWORD_EXCLUDE+=" and not test_access_bucket"
|
||||
# - POST object (HTML form-based upload)
|
||||
KEYWORD_EXCLUDE+=" and not test_post_object"
|
||||
# - Ranged requests (Range header)
|
||||
KEYWORD_EXCLUDE+=" and not ranged_request"
|
||||
# - Conditional requests (If-Match, If-None-Match, If-Modified-Since)
|
||||
KEYWORD_EXCLUDE+=" and not ifmatch and not ifnonematch and not ifmodified and not ifunmodified"
|
||||
KEYWORD_EXCLUDE+=" and not ifnonmatch"
|
||||
# - Object copy tests not caught by copy_object keyword
|
||||
KEYWORD_EXCLUDE+=" and not object_copy"
|
||||
# - Multipart copy (UploadPartCopy)
|
||||
KEYWORD_EXCLUDE+=" and not multipart_copy"
|
||||
# - Public access block
|
||||
KEYWORD_EXCLUDE+=" and not public_block"
|
||||
# - Object attributes API
|
||||
KEYWORD_EXCLUDE+=" and not object_attributes"
|
||||
# - Auth-related tests
|
||||
KEYWORD_EXCLUDE+=" and not invalid_auth and not bad_auth and not authenticated_expired"
|
||||
# - Torrent
|
||||
KEYWORD_EXCLUDE+=" and not torrent"
|
||||
# - content_encoding aws_chunked
|
||||
KEYWORD_EXCLUDE+=" and not aws_chunked"
|
||||
# - GetBucketLocation (needs location constraint storage)
|
||||
KEYWORD_EXCLUDE+=" and not bucket_get_location"
|
||||
# - expected_bucket_owner (needs owner tracking)
|
||||
KEYWORD_EXCLUDE+=" and not expected_bucket_owner"
|
||||
# - bucket_recreate_not_overriding (needs data preservation on re-create)
|
||||
KEYWORD_EXCLUDE+=" and not bucket_recreate_not_overriding"
|
||||
# - object_read_unreadable (needs permission model)
|
||||
KEYWORD_EXCLUDE+=" and not object_read_unreadable"
|
||||
# - Versioned concurrent tests
|
||||
KEYWORD_EXCLUDE+=" and not versioned_concurrent"
|
||||
# - 100-continue
|
||||
KEYWORD_EXCLUDE+=" and not 100_continue"
|
||||
# - multipart_get_part (GetObjectPartNumber)
|
||||
KEYWORD_EXCLUDE+=" and not multipart_get_part and not multipart_single_get_part and not non_multipart_get_part"
|
||||
# - object_anon_put
|
||||
KEYWORD_EXCLUDE+=" and not object_anon_put"
|
||||
# - raw response headers / raw get/put tests (presigned-like)
|
||||
KEYWORD_EXCLUDE+=" and not object_raw"
|
||||
# - Object write headers (cache-control, expires)
|
||||
KEYWORD_EXCLUDE+=" and not object_write_cache_control and not object_write_expires"
|
||||
# - bucket_head_extended
|
||||
KEYWORD_EXCLUDE+=" and not bucket_head_extended"
|
||||
# - Restore/read-through
|
||||
KEYWORD_EXCLUDE+=" and not restore_object and not read_through and not restore_noncur"
|
||||
# - list_multipart_upload_owner (needs owner tracking)
|
||||
KEYWORD_EXCLUDE+=" and not list_multipart_upload_owner"
|
||||
# - bucket_create_exists (needs owner tracking)
|
||||
KEYWORD_EXCLUDE+=" and not bucket_create_exists"
|
||||
# - bucket_create_naming_dns (dots + hyphens adjacent)
|
||||
KEYWORD_EXCLUDE+=" and not bucket_create_naming_dns"
|
||||
# - object_requestid_matches_header_on_error
|
||||
KEYWORD_EXCLUDE+=" and not requestid_matches_header"
|
||||
# - unicode metadata
|
||||
KEYWORD_EXCLUDE+=" and not unicode_metadata"
|
||||
# - multipart_upload_on_a_bucket_with_policy
|
||||
KEYWORD_EXCLUDE+=" and not upload_on_a_bucket_with_policy"
|
||||
# - upload_part_copy_percent_encoded_key
|
||||
KEYWORD_EXCLUDE+=" and not part_copy"
|
||||
# - list_buckets_paginated (needs pagination support in list_buckets)
|
||||
KEYWORD_EXCLUDE+=" and not list_buckets_paginated and not list_buckets_invalid and not list_buckets_bad"
|
||||
# - multipart_resend_first_finishes_last
|
||||
KEYWORD_EXCLUDE+=" and not resend_first_finishes_last"
|
||||
# - ranged_big_request (Range header support)
|
||||
KEYWORD_EXCLUDE+=" and not ranged_big"
|
||||
# - encoding_basic (URL encoding in listing)
|
||||
KEYWORD_EXCLUDE+=" and not encoding_basic"
|
||||
# - maxkeys_invalid (needs proper 400 error for non-numeric maxkeys)
|
||||
KEYWORD_EXCLUDE+=" and not maxkeys_invalid"
|
||||
# - fetchowner (needs FetchOwner=true support in v2)
|
||||
KEYWORD_EXCLUDE+=" and not fetchowner"
|
||||
# - list_return_data (needs Owner data in old SDK format)
|
||||
KEYWORD_EXCLUDE+=" and not list_return_data"
|
||||
# - unordered listing tests (parallel create, needs strict ordering)
|
||||
KEYWORD_EXCLUDE+=" and not bucket_list_unordered and not bucket_listv2_unordered"
|
||||
# - block_public_policy/restrict tests (PutPublicAccessBlock not implemented)
|
||||
KEYWORD_EXCLUDE+=" and not block_public"
|
||||
# - multipart_upload_resend_part (uses Range header in _check_content_using_range)
|
||||
KEYWORD_EXCLUDE+=" and not upload_resend_part"
|
||||
|
||||
FILTER="$MARKER_EXCLUDE and $KEYWORD_EXCLUDE"
|
||||
|
||||
# --- Run the tests ------------------------------------------------------------
|
||||
|
||||
export S3TEST_CONF="$CONF"
|
||||
|
||||
EXTRA_ARGS=("${@}")
|
||||
|
||||
echo ""
|
||||
echo "Running s3-tests..."
|
||||
echo "Filter: $FILTER"
|
||||
echo ""
|
||||
|
||||
# --- Individual test deselections (can't use -k without affecting similarly-named tests)
|
||||
|
||||
DESELECT_ARGS=()
|
||||
# test_multipart_upload: uses Range requests + idempotent double-complete (Ceph-specific)
|
||||
DESELECT_ARGS+=(--deselect "s3tests/functional/test_s3.py::test_multipart_upload")
|
||||
# test_multipart_upload_small: idempotent double-complete (Ceph-specific behavior)
|
||||
DESELECT_ARGS+=(--deselect "s3tests/functional/test_s3.py::test_multipart_upload_small")
|
||||
|
||||
cd "$S3TESTS_DIR"
|
||||
python3 -m pytest s3tests/functional/test_s3.py \
|
||||
-k "$FILTER" \
|
||||
"${DESELECT_ARGS[@]}" \
|
||||
-v \
|
||||
--tb=short \
|
||||
"${EXTRA_ARGS[@]}" \
|
||||
|| true # don't fail the script on test failures — we want to see results
|
||||
Reference in New Issue
Block a user