From a248ed5ff52a3815625cb304188dd3e181899950 Mon Sep 17 00:00:00 2001 From: Mark Theunissen Date: Wed, 11 Dec 2024 21:51:34 +1100 Subject: [PATCH 1/4] Fixes for POST policy checks and the x-ignore implementation (#20674) --- .github/workflows/run-mint.sh | 3 + cmd/post-policy_test.go | 20 ++- cmd/postpolicyform.go | 105 ++++++------- cmd/postpolicyform_test.go | 269 ++++++++++++++++++++++++++-------- internal/http/headers.go | 4 + 5 files changed, 268 insertions(+), 133 deletions(-) diff --git a/.github/workflows/run-mint.sh b/.github/workflows/run-mint.sh index 6a64465c9e825..df1bec9b7a1e0 100755 --- a/.github/workflows/run-mint.sh +++ b/.github/workflows/run-mint.sh @@ -16,6 +16,9 @@ docker volume rm $(docker volume ls -f dangling=true) || true ## change working directory cd .github/workflows/mint +## always pull latest +docker pull docker.io/minio/mint:edge + docker-compose -f minio-${MODE}.yaml up -d sleep 1m diff --git a/cmd/post-policy_test.go b/cmd/post-policy_test.go index e2f74bc72a842..236e6dd108522 100644 --- a/cmd/post-policy_test.go +++ b/cmd/post-policy_test.go @@ -56,10 +56,12 @@ func newPostPolicyBytesV4WithContentRange(credential, bucketName, objectKey stri credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential) // Add the meta-uuid string, set to 1234 uuidConditionStr := fmt.Sprintf(`["eq", "$x-amz-meta-uuid", "%s"]`, "1234") + // Add the content-encoding string, set to gzip. + contentEncodingConditionStr := fmt.Sprintf(`["eq", "$content-encoding", "%s"]`, "gzip") // Combine all conditions into one string. - conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s, %s]`, bucketConditionStr, - keyConditionStr, contentLengthCondStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr) + conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s, %s, %s]`, bucketConditionStr, + keyConditionStr, contentLengthCondStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr, contentEncodingConditionStr) retStr := "{" retStr = retStr + expirationStr + "," retStr += conditionStr @@ -85,9 +87,11 @@ func newPostPolicyBytesV4(credential, bucketName, objectKey string, expiration t credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential) // Add the meta-uuid string, set to 1234 uuidConditionStr := fmt.Sprintf(`["eq", "$x-amz-meta-uuid", "%s"]`, "1234") + // Add the content-encoding string, set to gzip + contentEncodingConditionStr := fmt.Sprintf(`["eq", "$content-encoding", "%s"]`, "gzip") // Combine all conditions into one string. - conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s]`, bucketConditionStr, keyConditionStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr) + conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s, %s]`, bucketConditionStr, keyConditionStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr, contentEncodingConditionStr) retStr := "{" retStr = retStr + expirationStr + "," retStr += conditionStr @@ -331,7 +335,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr accessKey: credentials.AccessKey, secretKey: credentials.SecretKey, dates: []interface{}{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}, - policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"]]}`, + policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"],["eq", "$content-encoding", "gzip"]]}`, }, // Success case, no multipart filename. { @@ -341,7 +345,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr accessKey: credentials.AccessKey, secretKey: credentials.SecretKey, dates: []interface{}{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}, - policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"]]}`, + policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"],["eq", "$content-encoding", "gzip"]]}`, noFilename: true, }, // Success case, big body. @@ -352,7 +356,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr accessKey: credentials.AccessKey, secretKey: credentials.SecretKey, dates: []interface{}{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}, - policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"]]}`, + policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"],["eq", "$content-encoding", "gzip"]]}`, }, // Corrupted Base 64 result { @@ -447,7 +451,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr malformedBody: false, ignoreContentLength: false, }, - // Failed with Content-Length not specified. + // Success with Content-Length not specified. { objectName: "test", data: bytes.Repeat([]byte("a"), 1025), @@ -547,7 +551,7 @@ func testPostPolicyBucketHandlerRedirect(obj ObjectLayer, instanceType string, t rec := httptest.NewRecorder() dates := []interface{}{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)} - policy := `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], {"success_action_redirect":"` + redirectURL.String() + `"},["starts-with", "$key", "test/"], ["eq", "$x-amz-meta-uuid", "1234"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"]]}` + policy := `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], {"success_action_redirect":"` + redirectURL.String() + `"},["starts-with", "$key", "test/"], ["eq", "$x-amz-meta-uuid", "1234"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$content-encoding", "gzip"]]}` // Generate the final policy document policy = fmt.Sprintf(policy, dates...) diff --git a/cmd/postpolicyform.go b/cmd/postpolicyform.go index ad8aadc6a607a..74b5ef870a770 100644 --- a/cmd/postpolicyform.go +++ b/cmd/postpolicyform.go @@ -53,18 +53,6 @@ var startsWithConds = map[string]bool{ "$x-amz-date": false, } -var postPolicyIgnoreKeys = map[string]bool{ - "Policy": true, - xhttp.AmzSignature: true, - xhttp.ContentEncoding: true, - http.CanonicalHeaderKey(xhttp.AmzChecksumAlgo): true, - http.CanonicalHeaderKey(xhttp.AmzChecksumCRC32): true, - http.CanonicalHeaderKey(xhttp.AmzChecksumCRC32C): true, - http.CanonicalHeaderKey(xhttp.AmzChecksumSHA1): true, - http.CanonicalHeaderKey(xhttp.AmzChecksumSHA256): true, - http.CanonicalHeaderKey(xhttp.AmzChecksumMode): true, -} - // Add policy conditionals. const ( policyCondEqual = "eq" @@ -272,60 +260,57 @@ func checkPolicyCond(op string, input1, input2 string) bool { return false } +// S3 docs: "Each form field that you specify in a form (except x-amz-signature, file, policy, and field names +// that have an x-ignore- prefix) must appear in the list of conditions." +// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html +// keyInPolicyExceptions - list of keys that, when present in the form, can be missing in the conditions of the policy. +var keyInPolicyExceptions = map[string]bool{ + xhttp.AmzSignature: true, + "File": true, + "Policy": true, + + // MinIO specific exceptions to the general S3 rule above. + encrypt.SseKmsKeyID: true, + encrypt.SseEncryptionContext: true, + encrypt.SseCustomerAlgorithm: true, + encrypt.SseCustomerKey: true, + encrypt.SseCustomerKeyMD5: true, +} + // checkPostPolicy - apply policy conditions and validate input values. -// (http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html) +// Note that content-length-range is checked in the API handler function PostPolicyBucketHandler. +// formValues is the already-canonicalized form values from the POST request. func checkPostPolicy(formValues http.Header, postPolicyForm PostPolicyForm) error { // Check if policy document expiry date is still not reached if !postPolicyForm.Expiration.After(UTCNow()) { return fmt.Errorf("Invalid according to Policy: Policy expired") } - // check all formValues appear in postPolicyForm or return error. #https://github.com/minio/minio/issues/17391 - checkHeader := map[string][]string{} - ignoreKeys := map[string]bool{} - for key, value := range formValues { - switch { - case ignoreKeys[key], postPolicyIgnoreKeys[key], strings.HasPrefix(key, encrypt.SseGenericHeader): + + // mustFindInPolicy is a map to list all the keys that we must find in the policy as + // we process it below. At the end of checkPostPolicy function, if any key is left in + // this map, that's an error. + mustFindInPolicy := make(map[string][]string, len(formValues)) + for key, values := range formValues { + if keyInPolicyExceptions[key] || strings.HasPrefix(key, "X-Ignore-") { continue - case strings.HasPrefix(key, "X-Amz-Ignore-"): - ignoreKey := strings.Replace(key, "X-Amz-Ignore-", "", 1) - ignoreKeys[ignoreKey] = true - // if it have already - delete(checkHeader, ignoreKey) - default: - checkHeader[key] = value - } - } - // map to store the metadata - metaMap := make(map[string]string) - for _, policy := range postPolicyForm.Conditions.Policies { - if strings.HasPrefix(policy.Key, "$x-amz-meta-") { - formCanonicalName := http.CanonicalHeaderKey(strings.TrimPrefix(policy.Key, "$")) - metaMap[formCanonicalName] = policy.Value } + mustFindInPolicy[key] = values } - // Check if any extra metadata field is passed as input - for key := range formValues { - if strings.HasPrefix(key, "X-Amz-Meta-") { - if _, ok := metaMap[key]; !ok { - return fmt.Errorf("Invalid according to Policy: Extra input fields: %s", key) - } - } - } - - // Flag to indicate if all policies conditions are satisfied - var condPassed bool // Iterate over policy conditions and check them against received form fields for _, policy := range postPolicyForm.Conditions.Policies { // Form fields names are in canonical format, convert conditions names // to canonical for simplification purpose, so `$key` will become `Key` formCanonicalName := http.CanonicalHeaderKey(strings.TrimPrefix(policy.Key, "$")) + // Operator for the current policy condition op := policy.Operator - // Multiple values should not occur - if len(checkHeader[formCanonicalName]) >= 2 { - return fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]. FormValues have multiple values: [%s]", op, policy.Key, policy.Value, strings.Join(checkHeader[formCanonicalName], ", ")) + + // Multiple values are not allowed for a single form field + if len(mustFindInPolicy[formCanonicalName]) >= 2 { + return fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]. FormValues have multiple values: [%s]", op, policy.Key, policy.Value, strings.Join(mustFindInPolicy[formCanonicalName], ", ")) } + // If the current policy condition is known if startsWithSupported, condFound := startsWithConds[policy.Key]; condFound { // Check if the current condition supports starts-with operator @@ -333,35 +318,35 @@ func checkPostPolicy(formValues http.Header, postPolicyForm PostPolicyForm) erro return fmt.Errorf("Invalid according to Policy: Policy Condition failed") } // Check if current policy condition is satisfied - condPassed = checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value) - if !condPassed { + if !checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value) { return fmt.Errorf("Invalid according to Policy: Policy Condition failed") } } else if strings.HasPrefix(policy.Key, "$x-amz-meta-") || strings.HasPrefix(policy.Key, "$x-amz-") { // This covers all conditions X-Amz-Meta-* and X-Amz-* // Check if policy condition is satisfied - condPassed = checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value) - if !condPassed { + if !checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value) { return fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]", op, policy.Key, policy.Value) } } - delete(checkHeader, formCanonicalName) + delete(mustFindInPolicy, formCanonicalName) } - // For SignV2 - Signature/AWSAccessKeyId field will be ignored. + + // For SignV2 - Signature/AWSAccessKeyId fields do not need to be in the policy if _, ok := formValues[xhttp.AmzSignatureV2]; ok { - delete(checkHeader, xhttp.AmzSignatureV2) - for k := range checkHeader { + delete(mustFindInPolicy, xhttp.AmzSignatureV2) + for k := range mustFindInPolicy { // case-insensitivity for AWSAccessKeyId if strings.EqualFold(k, xhttp.AmzAccessKeyID) { - delete(checkHeader, k) + delete(mustFindInPolicy, k) break } } } - if len(checkHeader) != 0 { - logKeys := make([]string, 0, len(checkHeader)) - for key := range checkHeader { + // Check mustFindInPolicy to see if any key is left, if so, it was not found in policy and we return an error. + if len(mustFindInPolicy) != 0 { + logKeys := make([]string, 0, len(mustFindInPolicy)) + for key := range mustFindInPolicy { logKeys = append(logKeys, key) } return fmt.Errorf("Each form field that you specify in a form must appear in the list of policy conditions. %q not specified in the policy.", strings.Join(logKeys, ", ")) diff --git a/cmd/postpolicyform_test.go b/cmd/postpolicyform_test.go index 8095c4d0f9bed..0f86044b8f382 100644 --- a/cmd/postpolicyform_test.go +++ b/cmd/postpolicyform_test.go @@ -20,12 +20,12 @@ package cmd import ( "bytes" "encoding/base64" - "fmt" "net/http" "strings" "testing" minio "github.com/minio/minio-go/v7" + xhttp "github.com/minio/minio/internal/http" ) func TestParsePostPolicyForm(t *testing.T) { @@ -78,6 +78,28 @@ func TestParsePostPolicyForm(t *testing.T) { } } +type formValues struct { + http.Header +} + +func newFormValues() formValues { + return formValues{make(http.Header)} +} + +func (f formValues) Set(key, value string) formValues { + f.Header.Set(key, value) + return f +} + +func (f formValues) Add(key, value string) formValues { + f.Header.Add(key, value) + return f +} + +func (f formValues) Clone() formValues { + return formValues{f.Header.Clone()} +} + // Test Post Policy parsing and checking conditions func TestPostPolicyForm(t *testing.T) { pp := minio.NewPostPolicy() @@ -85,76 +107,193 @@ func TestPostPolicyForm(t *testing.T) { pp.SetContentType("image/jpeg") pp.SetUserMetadata("uuid", "14365123651274") pp.SetKeyStartsWith("user/user1/filename") - pp.SetContentLengthRange(1048579, 10485760) + pp.SetContentLengthRange(100, 999999) // not testable from this layer, condition is checked in the API handler. pp.SetSuccessStatusAction("201") + pp.SetCondition("eq", "X-Amz-Credential", "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request") + pp.SetCondition("eq", "X-Amz-Algorithm", "AWS4-HMAC-SHA256") + pp.SetCondition("eq", xhttp.AmzDate, "20160727T000000Z") + + defaultFormVals := newFormValues() + defaultFormVals.Set("Bucket", "testbucket") + defaultFormVals.Set("Content-Type", "image/jpeg") + defaultFormVals.Set(xhttp.AmzMetaUUID, "14365123651274") + defaultFormVals.Set("Key", "user/user1/filename/${filename}/myfile.txt") + defaultFormVals.Set("X-Amz-Credential", "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request") + defaultFormVals.Set("X-Amz-Algorithm", "AWS4-HMAC-SHA256") + defaultFormVals.Set(xhttp.AmzDate, "20160727T000000Z") + defaultFormVals.Set("Success_action_status", "201") + + policyCondFailedErr := "Invalid according to Policy: Policy Condition failed" type testCase struct { - Bucket string - Key string - XAmzDate string - XAmzAlgorithm string - XAmzCredential string - XAmzMetaUUID string - ContentType string - SuccessActionStatus string - Policy string - Expired bool - expectedErr error + name string + fv formValues + expired bool + wantErr string } + // Test case just contains fields we override from defaultFormVals. testCases := []testCase{ - // Everything is fine with this test - {Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: nil}, - // Expired policy document - {Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Expired: true, expectedErr: fmt.Errorf("Invalid according to Policy: Policy expired")}, - // Different AMZ date - {Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "2017T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")}, - // Key which doesn't start with user/user1/filename - {Bucket: "testbucket", Key: "myfile.txt", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")}, - // Incorrect bucket name. - {Bucket: "incorrect", Key: "user/user1/filename/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")}, - // Incorrect key name - {Bucket: "testbucket", Key: "incorrect", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")}, - // Incorrect date - {Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "incorrect", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")}, - // Incorrect ContentType - {Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "incorrect", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")}, - // Incorrect Metadata - {Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "151274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed: [eq, $x-amz-meta-uuid, 14365123651274]")}, + { + name: "happy path no errors", + fv: defaultFormVals.Clone(), + wantErr: "", + }, + { + name: "expired policy document", + fv: defaultFormVals.Clone(), + expired: true, + wantErr: "Invalid according to Policy: Policy expired", + }, + { + name: "different AMZ date", + fv: defaultFormVals.Clone().Set(xhttp.AmzDate, "2017T000000Z"), + wantErr: policyCondFailedErr, + }, + { + name: "incorrect date", + fv: defaultFormVals.Clone().Set(xhttp.AmzDate, "incorrect"), + wantErr: policyCondFailedErr, + }, + { + name: "key which doesn't start with user/user1/filename", + fv: defaultFormVals.Clone().Set("Key", "myfile.txt"), + wantErr: policyCondFailedErr, + }, + { + name: "incorrect key name", + fv: defaultFormVals.Clone().Set("Key", "incorrect"), + wantErr: policyCondFailedErr, + }, + { + name: "incorrect bucket name", + fv: defaultFormVals.Clone().Set("Bucket", "incorrect"), + wantErr: policyCondFailedErr, + }, + { + name: "incorrect ContentType", + fv: defaultFormVals.Clone().Set(xhttp.ContentType, "incorrect"), + wantErr: policyCondFailedErr, + }, + { + name: "incorrect X-Amz-Algorithm", + fv: defaultFormVals.Clone().Set(xhttp.AmzAlgorithm, "incorrect"), + wantErr: policyCondFailedErr, + }, + { + name: "incorrect X-Amz-Credential", + fv: defaultFormVals.Clone().Set(xhttp.AmzCredential, "incorrect"), + wantErr: policyCondFailedErr, + }, + { + name: "incorrect metadata uuid", + fv: defaultFormVals.Clone().Set(xhttp.AmzMetaUUID, "151274"), + wantErr: "Invalid according to Policy: Policy Condition failed: [eq, $x-amz-meta-uuid, 14365123651274]", + }, + { + name: "unknown key XAmzMetaName is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(xhttp.AmzMetaName, "my-name"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Meta-Name" not specified in the policy.`, + }, + { + name: "unknown key XAmzChecksumAlgo is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumAlgo), "algo-val"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Algorithm" not specified in the policy.`, + }, + { + name: "unknown key XAmzChecksumCRC32 is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumCRC32), "crc32-val"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Crc32" not specified in the policy.`, + }, + { + name: "unknown key XAmzChecksumCRC32C is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumCRC32C), "crc32c-val"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Crc32c" not specified in the policy.`, + }, + { + name: "unknown key XAmzChecksumSHA1 is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumSHA1), "sha1-val"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Sha1" not specified in the policy.`, + }, + { + name: "unknown key XAmzChecksumSHA256 is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumSHA256), "sha256-val"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Sha256" not specified in the policy.`, + }, + { + name: "unknown key XAmzChecksumMode is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumMode), "mode-val"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Mode" not specified in the policy.`, + }, + { + name: "unknown key Content-Encoding is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.ContentEncoding), "encoding-val"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "Content-Encoding" not specified in the policy.`, + }, + { + name: "many bucket values", + fv: defaultFormVals.Clone().Add("Bucket", "anotherbucket"), + wantErr: "Invalid according to Policy: Policy Condition failed: [eq, $bucket, testbucket]. FormValues have multiple values: [testbucket, anotherbucket]", + }, + { + name: "XAmzSignature does not have to appear in policy", + fv: defaultFormVals.Clone().Set(xhttp.AmzSignature, "my-signature"), + }, + { + name: "XIgnoreFoo does not have to appear in policy", + fv: defaultFormVals.Clone().Set("X-Ignore-Foo", "my-foo-value"), + }, + { + name: "File does not have to appear in policy", + fv: defaultFormVals.Clone().Set("File", "file-value"), + }, + { + name: "Signature does not have to appear in policy", + fv: defaultFormVals.Clone().Set(xhttp.AmzSignatureV2, "signature-value"), + }, + { + name: "AWSAccessKeyID does not have to appear in policy", + fv: defaultFormVals.Clone().Set(xhttp.AmzAccessKeyID, "access").Set(xhttp.AmzSignatureV2, "signature-value"), + }, + { + name: "any form value starting with X-Amz-Server-Side-Encryption- does not have to appear in policy", + fv: defaultFormVals.Clone(). + Set(xhttp.AmzServerSideEncryptionKmsContext, "context-val"). + Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, "algo-val"), + }, } - // Validate all the test cases. - for i, tt := range testCases { - formValues := make(http.Header) - formValues.Set("Bucket", tt.Bucket) - formValues.Set("Key", tt.Key) - formValues.Set("Content-Type", tt.ContentType) - formValues.Set("X-Amz-Date", tt.XAmzDate) - formValues.Set("X-Amz-Meta-Uuid", tt.XAmzMetaUUID) - formValues.Set("X-Amz-Algorithm", tt.XAmzAlgorithm) - formValues.Set("X-Amz-Credential", tt.XAmzCredential) - if tt.Expired { - // Expired already. - pp.SetExpires(UTCNow().AddDate(0, 0, -10)) - } else { - // Expires in 10 days. - pp.SetExpires(UTCNow().AddDate(0, 0, 10)) - } - - formValues.Set("Policy", base64.StdEncoding.EncodeToString([]byte(pp.String()))) - formValues.Set("Success_action_status", tt.SuccessActionStatus) - policyBytes, err := base64.StdEncoding.DecodeString(base64.StdEncoding.EncodeToString([]byte(pp.String()))) - if err != nil { - t.Fatal(err) - } - - postPolicyForm, err := parsePostPolicyForm(bytes.NewReader(policyBytes)) - if err != nil { - t.Fatal(err) - } - - err = checkPostPolicy(formValues, postPolicyForm) - if err != nil && tt.expectedErr != nil && err.Error() != tt.expectedErr.Error() { - t.Fatalf("Test %d:, Expected %s, got %s", i+1, tt.expectedErr.Error(), err.Error()) - } + + // Run tests + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + if tt.expired { + // Expired already. + pp.SetExpires(UTCNow().AddDate(0, 0, -10)) + } else { + // Expires in 10 days. + pp.SetExpires(UTCNow().AddDate(0, 0, 10)) + } + + tt.fv.Set("Policy", base64.StdEncoding.EncodeToString([]byte(pp.String()))) + + policyBytes, err := base64.StdEncoding.DecodeString(base64.StdEncoding.EncodeToString([]byte(pp.String()))) + if err != nil { + t.Fatal(err) + } + + postPolicyForm, err := parsePostPolicyForm(bytes.NewReader(policyBytes)) + if err != nil { + t.Fatal(err) + } + + errStr := "" + err = checkPostPolicy(tt.fv.Header, postPolicyForm) + if err != nil { + errStr = err.Error() + } + if errStr != tt.wantErr { + t.Errorf("test: '%s', want error: '%s', got error: '%s'", tt.name, tt.wantErr, errStr) + } + }) } } diff --git a/internal/http/headers.go b/internal/http/headers.go index d8968910f3542..edfca9d9bf4ef 100644 --- a/internal/http/headers.go +++ b/internal/http/headers.go @@ -177,6 +177,10 @@ const ( AmzChecksumSHA256 = "x-amz-checksum-sha256" AmzChecksumMode = "x-amz-checksum-mode" + // Post Policy related + AmzMetaUUID = "X-Amz-Meta-Uuid" + AmzMetaName = "X-Amz-Meta-Name" + // Delete special flag to force delete a bucket or a prefix MinIOForceDelete = "x-minio-force-delete" From d56ef8dbe1f5903d9dd76c5b3966aa569b19000e Mon Sep 17 00:00:00 2001 From: ebozduman Date: Wed, 11 Dec 2024 02:52:17 -0800 Subject: [PATCH 2/4] Adds AIstore documentation link (#20738) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 835a0216851dd..99498d369c534 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![MinIO](https://raw.githubusercontent.com/minio/minio/master/.github/logo.svg?sanitize=true)](https://min.io) -MinIO is a High Performance Object Storage released under GNU Affero General Public License v3.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads. +MinIO is a High Performance Object Storage released under GNU Affero General Public License v3.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads. To learn more about what MinIO is doing for AI storage, go to [AI storage documentation](https://min.io/solutions/object-storage-for-ai). This README provides quickstart instructions on running MinIO on bare metal hardware, including container-based installations. For Kubernetes environments, use the [MinIO Kubernetes Operator](https://github.com/minio/operator/blob/master/README.md). From 7b3eb9f7f8760b82ba903cfe8d8b2c5405a18c29 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 11 Dec 2024 16:23:28 +0530 Subject: [PATCH 3/4] fix: groups lookup performance issue with users with lots of groups (#20740) fixes https://github.com/minio/minio/issues/20717 --- cmd/iam-store.go | 98 ++++++++++++++++++++++++++++++++++++++++++------ cmd/iam.go | 7 +++- 2 files changed, 93 insertions(+), 12 deletions(-) diff --git a/cmd/iam-store.go b/cmd/iam-store.go index 943af171bb12f..6d17a7fa73d8f 100644 --- a/cmd/iam-store.go +++ b/cmd/iam-store.go @@ -26,6 +26,7 @@ import ( "path" "sort" "strings" + "sync" "time" jsoniter "github.com/json-iterator/go" @@ -37,6 +38,7 @@ import ( "github.com/minio/minio/internal/jwt" "github.com/minio/pkg/v3/env" "github.com/minio/pkg/v3/policy" + "github.com/minio/pkg/v3/sync/errgroup" "github.com/puzpuzpuz/xsync/v3" "golang.org/x/sync/singleflight" ) @@ -357,6 +359,68 @@ func (c *iamCache) removeGroupFromMembershipsMap(group string) { } } +func (c *iamCache) policyDBGetGroups(store *IAMStoreSys, userPolicyPresent bool, groups ...string) ([]string, error) { + var policies []string + for _, group := range groups { + if store.getUsersSysType() == MinIOUsersSysType { + g, ok := c.iamGroupsMap[group] + if !ok { + continue + } + + // Group is disabled, so we return no policy - this + // ensures the request is denied. + if g.Status == statusDisabled { + continue + } + } + + policy, ok := c.iamGroupPolicyMap.Load(group) + if !ok { + continue + } + + policies = append(policies, policy.toSlice()...) + } + + found := len(policies) > 0 + if found { + return policies, nil + } + + if userPolicyPresent { + // if user mapping present and no group policies found + // rely on user policy for access, instead of fallback. + return nil, nil + } + + var mu sync.Mutex + + // no mappings found, fallback for all groups. + g := errgroup.WithNErrs(len(groups)).WithConcurrency(10) // load like 10 groups at a time. + + for index := range groups { + index := index + g.Go(func() error { + err := store.loadMappedPolicy(context.TODO(), groups[index], regUser, true, c.iamGroupPolicyMap) + if err != nil && !errors.Is(err, errNoSuchPolicy) { + return err + } + if errors.Is(err, errNoSuchPolicy) { + return nil + } + policy, _ := c.iamGroupPolicyMap.Load(groups[index]) + mu.Lock() + policies = append(policies, policy.toSlice()...) + mu.Unlock() + return nil + }, index) + } + + err := errors.Join(g.Wait()...) + return policies, err +} + // policyDBGet - lower-level helper; does not take locks. // // If a group is passed, it returns policies associated with the group. @@ -676,7 +740,8 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) erro type IAMStoreSys struct { IAMStorageAPI - group *singleflight.Group + group *singleflight.Group + policy *singleflight.Group } // HasWatcher - returns if the storage system has a watcher. @@ -757,21 +822,32 @@ func (store *IAMStoreSys) PolicyDBGet(name string, groups ...string) ([]string, cache := store.rlock() defer store.runlock() - policies, _, err := cache.policyDBGet(store, name, false, false) - if err != nil { - return nil, err - } + getPolicies := func() ([]string, error) { + policies, _, err := cache.policyDBGet(store, name, false, false) + if err != nil { + return nil, err + } - userPolicyPresent := len(policies) > 0 - for _, group := range groups { - ps, _, err := cache.policyDBGet(store, group, true, userPolicyPresent) + userPolicyPresent := len(policies) > 0 + + groupPolicies, err := cache.policyDBGetGroups(store, userPolicyPresent, groups...) if err != nil { return nil, err } - policies = append(policies, ps...) - } - return policies, nil + policies = append(policies, groupPolicies...) + return policies, nil + } + if store.policy != nil { + val, err, _ := store.policy.Do(name, func() (interface{}, error) { + return getPolicies() + }) + if err != nil { + return nil, err + } + return val.([]string), nil + } + return getPolicies() } // AddUsersToGroup - adds users to group, creating the group if needed. diff --git a/cmd/iam.go b/cmd/iam.go index c809eb929f7f1..8964605991d19 100644 --- a/cmd/iam.go +++ b/cmd/iam.go @@ -178,13 +178,18 @@ func (sys *IAMSys) initStore(objAPI ObjectLayer, etcdClient *etcd.Client) { } if etcdClient == nil { - var group *singleflight.Group + var ( + group *singleflight.Group + policy *singleflight.Group + ) if env.Get("_MINIO_IAM_SINGLE_FLIGHT", config.EnableOn) == config.EnableOn { group = &singleflight.Group{} + policy = &singleflight.Group{} } sys.store = &IAMStoreSys{ IAMStorageAPI: newIAMObjectStore(objAPI, sys.usersSysType), group: group, + policy: policy, } } else { sys.store = &IAMStoreSys{IAMStorageAPI: newIAMEtcdStore(etcdClient, sys.usersSysType)} From 9cdd204ae484fd48afd7807724ddd81ca05077f5 Mon Sep 17 00:00:00 2001 From: "Cesar N." <11819101+cesnietor@users.noreply.github.com> Date: Wed, 11 Dec 2024 04:53:53 -0600 Subject: [PATCH 4/4] Upgrade Console version to v1.7.5 (#20748) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 09471939b4a2d..ed9bba1964515 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/lithammer/shortuuid/v4 v4.0.0 github.com/miekg/dns v1.1.62 github.com/minio/cli v1.24.2 - github.com/minio/console v1.7.4 + github.com/minio/console v1.7.5 github.com/minio/csvparser v1.0.0 github.com/minio/dnscache v0.1.1 github.com/minio/dperf v0.6.0 diff --git a/go.sum b/go.sum index ed94743be4f5f..2638298878569 100644 --- a/go.sum +++ b/go.sum @@ -432,8 +432,8 @@ github.com/minio/cli v1.24.2 h1:J+fCUh9mhPLjN3Lj/YhklXvxj8mnyE/D6FpFduXJ2jg= github.com/minio/cli v1.24.2/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= github.com/minio/colorjson v1.0.8 h1:AS6gEQ1dTRYHmC4xuoodPDRILHP/9Wz5wYUGDQfPLpg= github.com/minio/colorjson v1.0.8/go.mod h1:wrs39G/4kqNlGjwqHvPlAnXuc2tlPszo6JKdSBCLN8w= -github.com/minio/console v1.7.4 h1:4ho6GBGJ6ncTZpHqc/LBjEzaL5e0eHQ8T7sH30sxM2Y= -github.com/minio/console v1.7.4/go.mod h1:qbj8JxTq2XgbzunpcQEM3fJ7Sgf5gKXW3b6Zs7bbsf8= +github.com/minio/console v1.7.5 h1:dnPP++SfV93b/uhUqggoeOEhF9E+bLSOQrkh1CiH8+U= +github.com/minio/console v1.7.5/go.mod h1:gHeX9FFJlJjSOxfp/cjDCk8rkP7q8hNxARkzhM3wK9M= github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU= github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM= github.com/minio/dnscache v0.1.1 h1:AMYLqomzskpORiUA1ciN9k7bZT1oB3YZN4cEIi88W5o=