Merge pull request #70051 from mheler/wip-copyobject-replace

src/rgw/driver/d4n/rgw_sal_d4n.cc

@@ -909,6 +909,10 @@ int D4NFilterObject::copy_object(const ACLOwner& owner,
       bl_val.append(*version_id);
       baseAttrs[RGW_CACHE_ATTR_VERSION_ID] = std::move(bl_val); //populate destination version id
     }
+    auto titer = attrs.find(RGW_ATTR_TAGS);
+    if (titer != attrs.end()) {
+      baseAttrs[RGW_ATTR_TAGS] = titer->second;
+    }
   }
 
   //ATTRSMOD_MERGE - any conflicting meta keys on the source object's attributes are overwritten by values contained in attrs parameter.

src/rgw/driver/posix/rgw_sal_posix.cc

@@ -3392,7 +3392,13 @@ int POSIXObject::copy_object(const ACLOwner& owner,
     }
     break;
   case ATTRSMOD_NONE:
-    attrs = src_attrs;
+    {
+      auto tags = attrs.extract(RGW_ATTR_TAGS);
+      attrs = src_attrs;
+      if (!tags.empty()) {
+        attrs[RGW_ATTR_TAGS] = std::move(tags.mapped());
+      }
+    }
     ret = 0;
     break;
   }

src/rgw/driver/rados/rgw_rados.cc

@@ -3960,7 +3960,13 @@ static void set_copy_attrs(map<string, bufferlist>& src_attrs,
 {
   switch (attrs_mod) {
   case RGWRados::ATTRSMOD_NONE:
-    attrs = src_attrs;
+    {
+      auto tags = attrs.extract(RGW_ATTR_TAGS);
+      attrs = src_attrs;
+      if (!tags.empty()) {
+        attrs[RGW_ATTR_TAGS] = std::move(tags.mapped());
+      }
+    }
     break;
   case RGWRados::ATTRSMOD_REPLACE:
     if (!attrs[RGW_ATTR_ETAG].length()) {
@@ -3972,6 +3978,13 @@ static void set_copy_attrs(map<string, bufferlist>& src_attrs,
         attrs[RGW_ATTR_TAIL_TAG] = src_attrs[RGW_ATTR_TAIL_TAG];
       }
     }
+    // the tag-set is copied from the source unless the request replaced it
+    if (attrs.find(RGW_ATTR_TAGS) == attrs.end()) {
+      auto titer = src_attrs.find(RGW_ATTR_TAGS);
+      if (titer != src_attrs.end()) {
+        attrs[RGW_ATTR_TAGS] = titer->second;
+      }
+    }
     break;
   case RGWRados::ATTRSMOD_MERGE:
     for (map<string, bufferlist>::iterator it = src_attrs.begin(); it != src_attrs.end(); ++it) {

src/rgw/rgw_op.cc

@@ -6324,11 +6324,52 @@ int RGWCopyObj::verify_permission(optional_yield y)
     rgw_add_to_iam_environment(s->env, "s3:x-amz-metadata-directive",
                                *md_directive);
 
+  /*
+   * The destination object is tagged whether the tag-set is replaced (from the
+   * request) or copied (from the source), so both paths must authorize object
+   * tagging and expose the tags to policy conditions. For a copied tag-set the
+   * source is read here to learn whether the destination will carry tags.
+   */
+  std::optional<RGWObjTags> dest_obj_tags = obj_tags;
+  if (!dest_obj_tags && copy_source_tags &&
+      s->local_source && source_zone.empty()) {
+    op_ret = s->src_object->get_obj_attrs(y, this);
+    if (op_ret < 0) {
+      return op_ret;
+    }
+    const auto& src_attrs = s->src_object->get_attrs();
+    auto titer = src_attrs.find(RGW_ATTR_TAGS);
+    if (titer != src_attrs.end()) {
+      RGWObjTags tagset;
+      try {
+        auto bliter = titer->second.cbegin();
+        tagset.decode(bliter);
+      } catch (buffer::error& err) {
+        ldpp_dout(s, 0) << "ERROR: caught buffer::error, couldn't decode TagSet" << dendl;
+        return -EIO;
+      }
+      dest_obj_tags = std::move(tagset);
+    }
+  }
+
+  if (dest_obj_tags) {
+    for (const auto& kv : dest_obj_tags->get_tags()) {
+      rgw_add_to_iam_environment(s->env, "s3:RequestObjectTag/" + kv.first, kv.second);
+    }
+  }
+
   if (!verify_bucket_permission(this, s, ARN(s->object->get_obj()),
                                 rgw::IAM::s3PutObject)) {
     return -EACCES;
   }
 
+  // writing or clearing object tags requires the tagging permission too
+  if (dest_obj_tags &&
+      !verify_bucket_permission(this, s, ARN(s->object->get_obj()),
+                                rgw::IAM::s3PutObjectTagging)) {
+    return -EACCES;
+  }
+
   op_ret = init_dest_policy();
   if (op_ret < 0) {
     return op_ret;
@@ -6366,6 +6407,10 @@ int RGWCopyObj::init_common()
   }
   populate_with_generic_attrs(s, attrs);
 
+  if (obj_tags) {
+    obj_tags->encode(attrs[RGW_ATTR_TAGS]);
+  }
+
   return 0;
 }
 
@@ -6420,12 +6465,19 @@ void RGWCopyObj::execute(optional_yield y)
   if (init_common() < 0)
     return;
 
+  // expose replacement tags to the notification event payload
+  if (obj_tags) {
+    s->tagset = *obj_tags;
+  }
+
   // make reservation for notification if needed
   std::unique_ptr<rgw::sal::Notification> res
 				   = driver->get_notification(
 				     s->object.get(), s->src_object.get(),
 				     s, rgw::notify::ObjectCreatedCopy, y);
-  op_ret = res->publish_reserve(this);
+
+  // expose replacement tags to notification filtering
+  op_ret = res->publish_reserve(this, obj_tags ? &*obj_tags : nullptr);
   if (op_ret < 0) {
     return;
   }
@@ -6513,6 +6565,13 @@ void RGWCopyObj::execute(optional_yield y)
     return;
   }
 
+  if (copy_source_tags && attrs_mod == rgw::sal::ATTRSMOD_REPLACE) {
+    bufferlist tags_bl;
+    if (s->src_object->get_attr(RGW_ATTR_TAGS, tags_bl)) {
+      attrs[RGW_ATTR_TAGS] = std::move(tags_bl);
+    }
+  }
+
   /*
    * Compute the normalized destination compression for CopyObject.
    * When the effective destination will be encrypted and the zonegroup

src/rgw/rgw_op.h

@@ -1619,6 +1619,8 @@ protected:
   std::string_view copy_source;
   // Not actually required
   std::optional<std::string_view> md_directive;
+  std::optional<RGWObjTags> obj_tags;
+  bool copy_source_tags = false;
   std::map<std::string, std::string> crypt_http_responses;
 
   off_t ofs;
@@ -2409,7 +2411,8 @@ inline int rgw_get_request_metadata(const DoutPrefixProvider *dpp,
       "x-amz-storage-class",
       "x-amz-content-sha256",
       "x-amz-checksum-algorithm",
-      "x-amz-date"
+      "x-amz-date",
+      "x-amz-tagging"
   };
 
   size_t valid_meta_count = 0;

src/rgw/rgw_rest_s3.cc

@@ -3974,6 +3974,32 @@ int RGWCopyObj_ObjStore_S3::get_params(optional_yield y)
     md_directive = tmp_md_d;
   }
 
+  copy_source_tags = true;
+
+  auto tagging_directive = s->info.env->get("HTTP_X_AMZ_TAGGING_DIRECTIVE");
+  if (tagging_directive) {
+    if (strcasecmp(tagging_directive, "REPLACE") == 0) {
+      RGWObjTags new_tags;
+      auto tag_str = s->info.env->get("HTTP_X_AMZ_TAGGING");
+      if (tag_str) {
+        int r = new_tags.set_from_string(tag_str);
+        if (r < 0) {
+          ldpp_dout(this, 0) << "setting obj tags failed with " << r << dendl;
+          if (r == -ERR_INVALID_TAG) {
+            r = -EINVAL;
+          }
+          return r;
+        }
+      }
+      obj_tags = std::move(new_tags);
+      copy_source_tags = false;
+    } else if (strcasecmp(tagging_directive, "COPY") != 0) {
+      s->err.message = "Unknown tagging directive.";
+      ldpp_dout(this, 0) << s->err.message << dendl;
+      return -EINVAL;
+    }
+  }
+
   if (source_zone.empty() &&
       (s->bucket->get_tenant() == s->src_tenant_name) &&
       (s->bucket->get_name() == s->src_bucket_name) &&

src/test/rgw/rgw_multi/tests.py

@@ -4513,6 +4513,34 @@ def test_copy_object_same_bucket():
 
     zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name)
 
+@attr('copy_object')
+def test_copy_object_replacing_tagging():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    primary = zonegroup_conns.rw_zones[0]
+
+    bucket = primary.create_bucket(gen_bucket_name())
+    objname = 'dummy'
+
+    primary.s3_client.put_object(Bucket=bucket.name, Key=objname, Body='foo', Tagging='key1=value1&key2=value2')
+    zonegroup_meta_checkpoint(zonegroup)
+    zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name)
+
+    primary.s3_client.copy_object(
+        Bucket=bucket.name,
+        CopySource={'Bucket': bucket.name, 'Key': objname},
+        Key=objname + '-copy',
+        TaggingDirective='REPLACE',
+        Tagging='key3=value3&key4=value4'
+    )
+
+    zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name)
+
+    expected = [{'Key': 'key3', 'Value': 'value3'}, {'Key': 'key4', 'Value': 'value4'}]
+    for zone in zonegroup_conns.rw_zones:
+        response = zone.s3_client.get_object_tagging(Bucket=bucket.name, Key=objname + '-copy')
+        assert_equal(response['TagSet'], expected)
+
 @attr('copy_object')
 def test_copy_object_different_bucket():
     zonegroup = realm.master_zonegroup()

src/test/rgw/s3-tests/s3tests/functional/test_s3.py

@@ -5743,6 +5743,113 @@ def test_object_copy_replacing_metadata():
         assert metadata == response['Metadata']
         assert size == response['ContentLength']
 
+@pytest.mark.tagging
+@pytest.mark.fails_on_dbstore
+def test_object_copy_retaining_tagging():
+    bucket_name = get_new_bucket()
+    client = get_client()
+    client.put_object(Bucket=bucket_name, Key='foo123bar', Body='foo', Tagging='key1=value1&key2=value2')
+
+    copy_source = {'Bucket': bucket_name, 'Key': 'foo123bar'}
+    client.copy_object(Bucket=bucket_name, CopySource=copy_source, Key='bar321foo')
+
+    response = client.get_object_tagging(Bucket=bucket_name, Key='bar321foo')
+    assert response['TagSet'] == [{'Key': 'key1', 'Value': 'value1'}, {'Key': 'key2', 'Value': 'value2'}]
+
+@pytest.mark.tagging
+@pytest.mark.fails_on_dbstore
+def test_object_copy_tagging_directive_copy():
+    bucket_name = get_new_bucket()
+    client = get_client()
+    client.put_object(Bucket=bucket_name, Key='foo123bar', Body='foo', Tagging='key1=value1&key2=value2')
+
+    copy_source = {'Bucket': bucket_name, 'Key': 'foo123bar'}
+    client.copy_object(Bucket=bucket_name, CopySource=copy_source, Key='bar321foo', TaggingDirective='COPY')
+
+    response = client.get_object_tagging(Bucket=bucket_name, Key='bar321foo')
+    assert response['TagSet'] == [{'Key': 'key1', 'Value': 'value1'}, {'Key': 'key2', 'Value': 'value2'}]
+
+@pytest.mark.tagging
+@pytest.mark.fails_on_dbstore
+def test_object_copy_tagging_ignored_without_directive():
+    bucket_name = get_new_bucket()
+    client = get_client()
+    client.put_object(Bucket=bucket_name, Key='foo123bar', Body='foo', Tagging='key1=value1&key2=value2')
+
+    copy_source = {'Bucket': bucket_name, 'Key': 'foo123bar'}
+    client.copy_object(Bucket=bucket_name, CopySource=copy_source, Key='bar321foo', Tagging='key3=value3&key4=value4')
+
+    response = client.get_object_tagging(Bucket=bucket_name, Key='bar321foo')
+    assert response['TagSet'] == [{'Key': 'key1', 'Value': 'value1'}, {'Key': 'key2', 'Value': 'value2'}]
+
+@pytest.mark.tagging
+@pytest.mark.fails_on_dbstore
+def test_object_copy_replacing_metadata_and_copying_tagging():
+    bucket_name = get_new_bucket()
+    client = get_client()
+    client.put_object(Bucket=bucket_name, Key='foo123bar', Body='foo',
+                      Metadata={'old': 'metadata'},
+                      Tagging='key1=value1&key2=value2')
+
+    copy_source = {'Bucket': bucket_name, 'Key': 'foo123bar'}
+    client.copy_object(Bucket=bucket_name, CopySource=copy_source, Key='bar321foo',
+                       Metadata={'new': 'metadata'},
+                       MetadataDirective='REPLACE',
+                       TaggingDirective='COPY')
+
+    response = client.get_object_tagging(Bucket=bucket_name, Key='bar321foo')
+    assert response['TagSet'] == [{'Key': 'key1', 'Value': 'value1'}, {'Key': 'key2', 'Value': 'value2'}]
+    response = client.get_object(Bucket=bucket_name, Key='bar321foo')
+    assert response['Metadata'] == {'new': 'metadata'}
+
+    client.copy_object(Bucket=bucket_name, CopySource=copy_source, Key='bar321foo-default',
+                       Metadata={'default': 'metadata'},
+                       MetadataDirective='REPLACE')
+
+    response = client.get_object_tagging(Bucket=bucket_name, Key='bar321foo-default')
+    assert response['TagSet'] == [{'Key': 'key1', 'Value': 'value1'}, {'Key': 'key2', 'Value': 'value2'}]
+    response = client.get_object(Bucket=bucket_name, Key='bar321foo-default')
+    assert response['Metadata'] == {'default': 'metadata'}
+
+@pytest.mark.tagging
+@pytest.mark.fails_on_dbstore
+def test_object_copy_replacing_tagging():
+    bucket_name = get_new_bucket()
+    client = get_client()
+    client.put_object(Bucket=bucket_name, Key='foo123bar', Body='foo', Tagging='key1=value1&key2=value2')
+
+    copy_source = {'Bucket': bucket_name, 'Key': 'foo123bar'}
+    client.copy_object(Bucket=bucket_name, CopySource=copy_source, Key='bar321foo', TaggingDirective='REPLACE', Tagging='key3=value3&key4=value4')
+
+    response = client.get_object_tagging(Bucket=bucket_name, Key='bar321foo')
+    assert response['TagSet'] == [{'Key': 'key3', 'Value': 'value3'}, {'Key': 'key4', 'Value': 'value4'}]
+
+@pytest.mark.tagging
+@pytest.mark.fails_on_dbstore
+def test_object_copy_replacing_tagging_with_none():
+    bucket_name = get_new_bucket()
+    client = get_client()
+    client.put_object(Bucket=bucket_name, Key='foo123bar', Body='foo', Tagging='key1=value1&key2=value2')
+
+    copy_source = {'Bucket': bucket_name, 'Key': 'foo123bar'}
+    client.copy_object(Bucket=bucket_name, CopySource=copy_source, Key='bar321foo', TaggingDirective='REPLACE')
+
+    response = client.get_object_tagging(Bucket=bucket_name, Key='bar321foo')
+    assert response['TagSet'] == []
+
+@pytest.mark.tagging
+@pytest.mark.fails_on_dbstore
+def test_object_copy_to_itself_replacing_tagging():
+    bucket_name = get_new_bucket()
+    client = get_client()
+    client.put_object(Bucket=bucket_name, Key='foo123bar', Body='foo', Tagging='key1=value1&key2=value2')
+
+    copy_source = {'Bucket': bucket_name, 'Key': 'foo123bar'}
+    e = assert_raises(ClientError, client.copy_object, Bucket=bucket_name, CopySource=copy_source, Key='foo123bar', TaggingDirective='REPLACE', Tagging='key3=value3&key4=value4')
+    status, error_code = _get_status_and_error_code(e.response)
+    assert status == 400
+    assert error_code == 'InvalidRequest'
+
 def test_object_copy_bucket_not_found():
     bucket_name = get_new_bucket()
     client = get_client()